From 3d1fddada0d518065aad14857834ae1a299725b6 Mon Sep 17 00:00:00 2001 From: Hayden Date: Mon, 27 Mar 2017 14:08:55 +0100 Subject: [PATCH 01/45] Remove duplicated import 'json' was imported twice. --- stats/dashboard.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/stats/dashboard.py b/stats/dashboard.py index 2ea55fb8e5b..dc4c12dd1c4 100644 --- a/stats/dashboard.py +++ b/stats/dashboard.py @@ -1278,8 +1278,6 @@ def sum_planned_disbursements_by_year(self): return out - -import json ckan = json.load(open('helpers/ckan.json')) publisher_re = re.compile('(.*)\-[^\-]') From 304d3a95c450b9fe9b6267df0a70a47a3345c35f Mon Sep 17 00:00:00 2001 From: Hayden Date: Mon, 27 Mar 2017 15:37:37 +0100 Subject: [PATCH 02/45] Add initial humanitarian stats Add stats for humanitarian monitoring. Some tests are written, though more are required. In particular, tests for clusters and v1 are still needed. --- .gitignore | 1 + stats/dashboard.py | 15 +++++ stats/tests/test_humanitarian.py | 108 +++++++++++++++++++++++++++++++ 3 files changed, 124 insertions(+) create mode 100644 stats/tests/test_humanitarian.py diff --git a/.gitignore b/.gitignore index 6569b199cf3..9aee52062bc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ data out gitout +.coverage helpers/ckan/ helpers/ckan.json diff --git a/stats/dashboard.py b/stats/dashboard.py index dc4c12dd1c4..fd740c7b803 100644 --- a/stats/dashboard.py +++ b/stats/dashboard.py @@ -1165,6 +1165,21 @@ def comprehensiveness_denominators(self): 'transaction_traceability': 0 } + @returns_numberdict + def humanitarian(self): + humanitarian_sectors_dac = ['72010', '74010'] + + is_humanitarian_by_attrib = 1 if ('humanitarian' in self.element.attrib) and (self.element.attrib['humanitarian'] in ['1', 'true']) else 0 + # import pdb;pdb.set_trace() + is_humanitarian_by_sector = 1 if set(self.element.xpath('sector[@vocabulary="1" or not(@vocabulary)]/@code')).intersection(humanitarian_sectors_dac) else 0 + + return { + 'is_humanitarian': 1 if (is_humanitarian_by_attrib or is_humanitarian_by_sector) else 0, + 'is_humanitarian_by_attrib': is_humanitarian_by_attrib, + 'contains_humanitarian_scope': 1 if (self.element.xpath('humanitarian-scope/@type') and self.element.xpath('humanitarian-scope/@code')) else 0, + 'uses_humanitarian_clusters_vocab': 1 if self.element.xpath('sector/@vocabulary="10"') else 0 + } + def _transaction_type_code(self, transaction): type_code = None transaction_type = transaction.find('transaction-type') diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py new file mode 100644 index 00000000000..562b9a3b0c2 --- /dev/null +++ b/stats/tests/test_humanitarian.py @@ -0,0 +1,108 @@ +# coding=utf-8 + +from lxml import etree +import pytest + +from stats.dashboard import ActivityStats + +class MockActivityStats(ActivityStats): + def __init__(self, major_version): + self.major_version = major_version + return super(MockActivityStats, self).__init__() + + def _major_version(self): + return self.major_version + +@pytest.mark.parametrize('major_version', ['2']) +@pytest.mark.parametrize('hum_attrib_val', ['1', 'true']) +def test_humanitarian_attrib_true(major_version, hum_attrib_val): + """ + Detect an activity to be humanitarian using @humanitarian values that evaluate to true. + """ + activity_stats = MockActivityStats(major_version) + + activity_stats.element = etree.fromstring(''' + + + '''.format(hum_attrib_val)) + assert activity_stats.humanitarian()['is_humanitarian'] == 1 + assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 1 + assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 + + +@pytest.mark.parametrize('major_version', ['2']) +@pytest.mark.parametrize('hum_attrib_val', ['0', 'false', 'True', 'False', '']) +def test_humanitarian_attrib_false(major_version, hum_attrib_val): + """ + Detect an activity to not be humanitarian using @humanitarian values that evaluate to false. + """ + activity_stats = MockActivityStats(major_version) + + activity_stats.element = etree.fromstring(''' + + + '''.format(hum_attrib_val)) + assert activity_stats.humanitarian()['is_humanitarian'] == 0 + assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 + assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 + + +@pytest.mark.parametrize('major_version', ['1', '2']) +@pytest.mark.parametrize('sector', [72010, 74010]) +@pytest.mark.parametrize('xml', [''' + + + + ''', ''' + + + + ''']) +def test_humanitarian_sector_true(major_version, sector, xml): + """ + Detects an activity to be humanitarian using sector codes considered to relate to humanitarian. + + Also checks that the appropriate vocabulary is provided or assumed. + """ + activity_stats = MockActivityStats(major_version) + + activity_stats.element = etree.fromstring(xml.format(sector)) + assert activity_stats.humanitarian()['is_humanitarian'] == 1 + assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 + assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 + + +@pytest.mark.parametrize('major_version', ['1', '2']) +@pytest.mark.parametrize('sector', [-89, 'not_a_code']) +def test_humanitarian_sector_false_bad_codes(major_version, sector): + """ + Detects an activity not to be humanitarian using sector codes that are not considered to relate to humanitarian. + """ + activity_stats = MockActivityStats(major_version) + + activity_stats.element = etree.fromstring(''' + + + + '''.format(sector)) + assert activity_stats.humanitarian()['is_humanitarian'] == 0 + assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 + assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 + + +@pytest.mark.parametrize('major_version', ['1', '2']) +@pytest.mark.parametrize('sector', [72010, 74010]) +def test_humanitarian_sector_false_bad_vocab(major_version, sector): + """ + Detects an activity not to be humanitarian due to specification of an incorrect vocabulary despite @code values that are considered to relate to humanitarian by default. + """ + activity_stats = MockActivityStats(major_version) + + activity_stats.element = etree.fromstring(''' + + + + '''.format(sector)) + assert activity_stats.humanitarian()['is_humanitarian'] == 0 + assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 + assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 From c547e280eb8694a24acbb10ec1de95beafa76068 Mon Sep 17 00:00:00 2001 From: Hayden Date: Mon, 27 Mar 2017 16:03:48 +0100 Subject: [PATCH 03/45] Add more humanitarian tests This adds more tests around detection. --- stats/dashboard.py | 3 +- stats/tests/test_humanitarian.py | 76 ++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/stats/dashboard.py b/stats/dashboard.py index fd740c7b803..4fe8fbc87df 100644 --- a/stats/dashboard.py +++ b/stats/dashboard.py @@ -1169,8 +1169,7 @@ def comprehensiveness_denominators(self): def humanitarian(self): humanitarian_sectors_dac = ['72010', '74010'] - is_humanitarian_by_attrib = 1 if ('humanitarian' in self.element.attrib) and (self.element.attrib['humanitarian'] in ['1', 'true']) else 0 - # import pdb;pdb.set_trace() + is_humanitarian_by_attrib = 1 if ('humanitarian' in self.element.attrib) and (self.element.attrib['humanitarian'] in ['1', 'true']) and (self._major_version() in ['2']) else 0 is_humanitarian_by_sector = 1 if set(self.element.xpath('sector[@vocabulary="1" or not(@vocabulary)]/@code')).intersection(humanitarian_sectors_dac) else 0 return { diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index 562b9a3b0c2..0d2d65cd5e1 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -106,3 +106,79 @@ def test_humanitarian_sector_false_bad_vocab(major_version, sector): assert activity_stats.humanitarian()['is_humanitarian'] == 0 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 + + +@pytest.mark.parametrize('major_version', ['2']) +@pytest.mark.parametrize('sector', [72010, 74010, -89, 'not_a_code']) +@pytest.mark.parametrize('hum_attrib_val', ['1', 'true']) +def test_humanitarian_attrib_true_sector_anything(major_version, sector, hum_attrib_val): + """ + Detect an activity to be humanitarian using @humanitarian values that evaluate to true in combination with sector codes. + + Both valid and invalid sector codes are tested because everything should still evaluate the same, given that @humanitarian is true. + """ + activity_stats = MockActivityStats(major_version) + + activity_stats.element = etree.fromstring(''' + + + + '''.format(hum_attrib_val, sector)) + assert activity_stats.humanitarian()['is_humanitarian'] == 1 + assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 1 + assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 + + +@pytest.mark.parametrize('major_version', ['2']) +@pytest.mark.parametrize('sector', [72010, 74010]) +@pytest.mark.parametrize('hum_attrib_val', ['0', 'false', 'True', 'False', '']) +def test_humanitarian_attrib_false_sector_true(major_version, sector, hum_attrib_val): + """ + Detect an activity to be humanitarian using sector codes that are deemed to be humanitarian, in combination with a @humanitarian value which evaluates to false. + """ + activity_stats = MockActivityStats(major_version) + + activity_stats.element = etree.fromstring(''' + + + + '''.format(hum_attrib_val, sector)) + assert activity_stats.humanitarian()['is_humanitarian'] == 1 + assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 + assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 + + +@pytest.mark.parametrize('major_version', ['2']) +@pytest.mark.parametrize('sector', [-89, 'not_a_code']) +@pytest.mark.parametrize('hum_attrib_val', ['0', 'false', 'True', 'False', '']) +def test_humanitarian_attrib_false_sector_false(major_version, sector, hum_attrib_val): + """ + Detect an activity not to be humanitarian using sector codes that are deemed not to be humanitarian, in combination with a @humanitarian value which evaluates to false. + """ + activity_stats = MockActivityStats(major_version) + + activity_stats.element = etree.fromstring(''' + + + + '''.format(hum_attrib_val, sector)) + assert activity_stats.humanitarian()['is_humanitarian'] == 0 + assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 + assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 + + +@pytest.mark.parametrize('major_version', ['1']) +@pytest.mark.parametrize('hum_attrib_val', ['1', 'true']) +def test_humanitarian_attrib_false_sector_false(major_version, hum_attrib_val): + """ + Detect an activity not to be humanitarian at version 1 of the standard when a @humanitarian value that evaluates to true is present. + """ + activity_stats = MockActivityStats(major_version) + + activity_stats.element = etree.fromstring(''' + + + '''.format(hum_attrib_val)) + assert activity_stats.humanitarian()['is_humanitarian'] == 0 + assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 + assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 From c4ce6e028e7c8a4dfd6c6d36cbef55c15f2d3901 Mon Sep 17 00:00:00 2001 From: Hayden Date: Mon, 27 Mar 2017 16:41:05 +0100 Subject: [PATCH 04/45] Add method to get full version Allows the full version (e.g. "2.01") to be obtained. --- stats/dashboard.py | 15 +++++ stats/tests/test_generic_file_stats.py | 80 ++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 stats/tests/test_generic_file_stats.py diff --git a/stats/dashboard.py b/stats/dashboard.py index 4fe8fbc87df..8b1a0ff0c57 100644 --- a/stats/dashboard.py +++ b/stats/dashboard.py @@ -351,6 +351,7 @@ def element_versions(self): @returns_numberdict @memoize def _major_version(self): + # TODO: Refactor to use _version parent = self.element.getparent() if parent is None: print('No parent of iati-activity, is this a test? Assuming version 1.xx') @@ -361,6 +362,20 @@ def _major_version(self): else: return '1' + @returns_numberdict + @memoize + def _version(self): + allowed_versions = CODELISTS['2']['Version'] + parent = self.element.getparent() + if parent is None: + print('No parent of iati-activity, is this a test? Assuming version 1.01') + return '1.01' + version = self.element.getparent().attrib.get('version') + if version and version in allowed_versions: + return version + else: + return '1.01' + @returns_numberdict def ruleset_passes(self): out = {} diff --git a/stats/tests/test_generic_file_stats.py b/stats/tests/test_generic_file_stats.py new file mode 100644 index 00000000000..b2400c3576b --- /dev/null +++ b/stats/tests/test_generic_file_stats.py @@ -0,0 +1,80 @@ +# coding=utf-8 + +from lxml import etree +import pytest + +from stats.dashboard import ActivityStats + + +@pytest.mark.parametrize('version', ['1.01', '1.02', '1.03', '1.04', '1.05', '2.01', '2.02']) +def test_version_detection_valid(version): + """ + Tests that valid versions of the IATI Standard are detected. + """ + + activity = ActivityStats() + + tree = etree.fromstring(''' + + + + + '''.format(version)) + + activity.element = tree.getchildren()[0] + + assert activity._version() == version + + +@pytest.mark.parametrize('version', ['1.06', '2.03', '3.01', '1', '', 'version 1.03', '1.03 version', '1.03 or 1.04', ' 2.01', '2 .01']) +def test_version_detection_invalid(version): + """ + Tests that invalid versions of the IATI Standard are detected. + """ + + activity = ActivityStats() + + tree = etree.fromstring(''' + + + + + '''.format(version)) + + activity.element = tree.getchildren()[0] + + assert activity._version() == '1.01' + + +def test_version_detection_no_parent(): + """ + Tests that XML with no parent returns default version. + """ + + activity = ActivityStats() + + activity.element = etree.fromstring(''' + + + ''') + + assert activity._version() == '1.01' + + +def test_version_detection_no_version_attrib(): + """ + Tests that XML with no version attribute returns default version. + """ + + activity = ActivityStats() + + tree = etree.fromstring(''' + + + + + ''') + + activity.element = tree.getchildren()[0] + + assert activity._version() == '1.01' From d22b91b60a191fc41aefb89db50b4656dd88eceb Mon Sep 17 00:00:00 2001 From: Hayden Date: Mon, 27 Mar 2017 17:25:04 +0100 Subject: [PATCH 05/45] Make humanitarian tests more version specific and add tests --- stats/dashboard.py | 6 +- stats/tests/test_humanitarian.py | 139 +++++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+), 3 deletions(-) diff --git a/stats/dashboard.py b/stats/dashboard.py index 8b1a0ff0c57..7b203f3c72e 100644 --- a/stats/dashboard.py +++ b/stats/dashboard.py @@ -1184,14 +1184,14 @@ def comprehensiveness_denominators(self): def humanitarian(self): humanitarian_sectors_dac = ['72010', '74010'] - is_humanitarian_by_attrib = 1 if ('humanitarian' in self.element.attrib) and (self.element.attrib['humanitarian'] in ['1', 'true']) and (self._major_version() in ['2']) else 0 + is_humanitarian_by_attrib = 1 if (self._version() in ['2.02']) and ('humanitarian' in self.element.attrib) and (self.element.attrib['humanitarian'] in ['1', 'true']) else 0 is_humanitarian_by_sector = 1 if set(self.element.xpath('sector[@vocabulary="1" or not(@vocabulary)]/@code')).intersection(humanitarian_sectors_dac) else 0 return { 'is_humanitarian': 1 if (is_humanitarian_by_attrib or is_humanitarian_by_sector) else 0, 'is_humanitarian_by_attrib': is_humanitarian_by_attrib, - 'contains_humanitarian_scope': 1 if (self.element.xpath('humanitarian-scope/@type') and self.element.xpath('humanitarian-scope/@code')) else 0, - 'uses_humanitarian_clusters_vocab': 1 if self.element.xpath('sector/@vocabulary="10"') else 0 + 'contains_humanitarian_scope': 1 if (self._version() in ['2.02']) and self.element.xpath('humanitarian-scope/@type') and self.element.xpath('humanitarian-scope/@code') else 0, + 'uses_humanitarian_clusters_vocab': 1 if (self._major_version() in ['2']) and self.element.xpath('sector/@vocabulary="10"') else 0 } def _transaction_type_code(self, transaction): diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index 0d2d65cd5e1..28ab2f87be7 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -13,6 +13,9 @@ def __init__(self, major_version): def _major_version(self): return self.major_version + def _version(self): + return self._major_version() + '.02' + @pytest.mark.parametrize('major_version', ['2']) @pytest.mark.parametrize('hum_attrib_val', ['1', 'true']) def test_humanitarian_attrib_true(major_version, hum_attrib_val): @@ -27,6 +30,7 @@ def test_humanitarian_attrib_true(major_version, hum_attrib_val): '''.format(hum_attrib_val)) assert activity_stats.humanitarian()['is_humanitarian'] == 1 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 1 + assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 @@ -44,6 +48,7 @@ def test_humanitarian_attrib_false(major_version, hum_attrib_val): '''.format(hum_attrib_val)) assert activity_stats.humanitarian()['is_humanitarian'] == 0 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 + assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 @@ -69,6 +74,7 @@ def test_humanitarian_sector_true(major_version, sector, xml): activity_stats.element = etree.fromstring(xml.format(sector)) assert activity_stats.humanitarian()['is_humanitarian'] == 1 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 + assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 @@ -87,6 +93,7 @@ def test_humanitarian_sector_false_bad_codes(major_version, sector): '''.format(sector)) assert activity_stats.humanitarian()['is_humanitarian'] == 0 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 + assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 @@ -105,6 +112,7 @@ def test_humanitarian_sector_false_bad_vocab(major_version, sector): '''.format(sector)) assert activity_stats.humanitarian()['is_humanitarian'] == 0 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 + assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 @@ -126,6 +134,7 @@ def test_humanitarian_attrib_true_sector_anything(major_version, sector, hum_att '''.format(hum_attrib_val, sector)) assert activity_stats.humanitarian()['is_humanitarian'] == 1 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 1 + assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 @@ -145,6 +154,7 @@ def test_humanitarian_attrib_false_sector_true(major_version, sector, hum_attrib '''.format(hum_attrib_val, sector)) assert activity_stats.humanitarian()['is_humanitarian'] == 1 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 + assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 @@ -164,6 +174,7 @@ def test_humanitarian_attrib_false_sector_false(major_version, sector, hum_attri '''.format(hum_attrib_val, sector)) assert activity_stats.humanitarian()['is_humanitarian'] == 0 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 + assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 @@ -181,4 +192,132 @@ def test_humanitarian_attrib_false_sector_false(major_version, hum_attrib_val): '''.format(hum_attrib_val)) assert activity_stats.humanitarian()['is_humanitarian'] == 0 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 + assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 + assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 + + +@pytest.mark.parametrize('version', ['2.02']) +@pytest.mark.parametrize('hum_attrib_val', ['1', 'true']) +def test_humanitarian_elements_valid_version(version, hum_attrib_val): + """ + Tests that humanitarian elements are detected at supported versions. + """ + + activity_stats = ActivityStats() + + tree = etree.fromstring(''' + + + + + + '''.format(version, hum_attrib_val)) + + activity_stats.element = tree.getchildren()[0] + + assert activity_stats.humanitarian()['is_humanitarian'] == 1 + assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 1 + assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 1 + assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 + + +@pytest.mark.parametrize('version', ['1.01', '1.02', '1.03', '1.04', '1.05', '2.01', 'unknown version']) +@pytest.mark.parametrize('hum_attrib_val', ['1', 'true']) +def test_humanitarian_elements_invalid_version(version, hum_attrib_val): + """ + Tests that humanitarian elements are detected at supported versions. + """ + + activity_stats = ActivityStats() + + tree = etree.fromstring(''' + + + + + + '''.format(version, hum_attrib_val)) + + activity_stats.element = tree.getchildren()[0] + + assert activity_stats.humanitarian()['is_humanitarian'] == 0 + assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 + assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 + assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 + + +@pytest.mark.parametrize('major_version', ['2']) +def test_humanitarian_scope_valid(major_version): + """ + Detect that an activity contains a humanitarian-scope element and required attributes. + """ + activity_stats = MockActivityStats(major_version) + + activity_stats.element = etree.fromstring(''' + + + + ''') + assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 1 + + +@pytest.mark.parametrize('major_version', ['2']) +def test_humanitarian_scope_invalid(major_version): + """ + Detect that an activity contains a humanitarian-scope element without required attributes. + """ + activity_stats = MockActivityStats(major_version) + + activity_stats.element = etree.fromstring(''' + + + + ''') + assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 + + +@pytest.mark.parametrize('major_version', ['2']) +def test_humanitarian_clusters_valid(major_version): + """ + Detect that an activity contains a sector defined by the 'Humanitarian Global Clusters' sector vocabulary. + """ + activity_stats = MockActivityStats(major_version) + + activity_stats.element = etree.fromstring(''' + + + + ''') + assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 1 + + +@pytest.mark.parametrize('major_version', ['1']) +def test_humanitarian_clusters_version_1(major_version): + """ + Detect that a version 1 activity containing a sector defined by the 'Humanitarian Global Clusters' sector vocabulary is not detected. + """ + activity_stats = MockActivityStats(major_version) + + activity_stats.element = etree.fromstring(''' + + + + ''') + assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 + + +@pytest.mark.parametrize('major_version', ['2']) +@pytest.mark.parametrize('sector_vocabulary_code', ['', '1', 'internal vocabulary']) +def test_humanitarian_clusters_invalid(major_version, sector_vocabulary_code): + """ + Detect that an activity does not contain a sector defined by the 'Humanitarian Global Clusters' sector vocabulary. + """ + activity_stats = MockActivityStats(major_version) + + activity_stats.element = etree.fromstring(''' + + + + '''.format(sector_vocabulary_code)) assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 + \ No newline at end of file From 2ead0a69bb619115481fe4596ba89103d6da01fd Mon Sep 17 00:00:00 2001 From: Hayden Date: Tue, 25 Apr 2017 11:05:29 +0100 Subject: [PATCH 06/45] Add more valid humanitarian DAC sector codes This adds missing valid 5-digit sector codes, as well as adding 3-digit codes. --- stats/dashboard.py | 7 +++- stats/tests/test_humanitarian.py | 64 ++++++++++++++++++++++++++++---- 2 files changed, 61 insertions(+), 10 deletions(-) diff --git a/stats/dashboard.py b/stats/dashboard.py index 7b203f3c72e..dcfbc390367 100644 --- a/stats/dashboard.py +++ b/stats/dashboard.py @@ -1182,10 +1182,13 @@ def comprehensiveness_denominators(self): @returns_numberdict def humanitarian(self): - humanitarian_sectors_dac = ['72010', '74010'] + humanitarian_sectors_dac_5_digit = ['72010', '72040', '72050', '73010', '74010'] + humanitarian_sectors_dac_3_digit = ['720', '730', '740'] is_humanitarian_by_attrib = 1 if (self._version() in ['2.02']) and ('humanitarian' in self.element.attrib) and (self.element.attrib['humanitarian'] in ['1', 'true']) else 0 - is_humanitarian_by_sector = 1 if set(self.element.xpath('sector[@vocabulary="1" or not(@vocabulary)]/@code')).intersection(humanitarian_sectors_dac) else 0 + is_humanitarian_by_sector_5_digit = 1 if set(self.element.xpath('sector[@vocabulary="1" or not(@vocabulary)]/@code')).intersection(humanitarian_sectors_dac_5_digit) else 0 + is_humanitarian_by_sector_3_digit = 1 if set(self.element.xpath('sector[@vocabulary="2"]/@code')).intersection(humanitarian_sectors_dac_3_digit) else 0 + is_humanitarian_by_sector = is_humanitarian_by_sector_5_digit or is_humanitarian_by_sector_3_digit return { 'is_humanitarian': 1 if (is_humanitarian_by_attrib or is_humanitarian_by_sector) else 0, diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index 28ab2f87be7..8a101a96c02 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -16,6 +16,9 @@ def _major_version(self): def _version(self): return self._major_version() + '.02' +HUMANITARIAN_SECTOR_CODES_5_DIGITS = [72010, 72040, 72050, 73010, 74010] +HUMANITARIAN_SECTOR_CODES_3_DIGITS = [720, 730, 740] + @pytest.mark.parametrize('major_version', ['2']) @pytest.mark.parametrize('hum_attrib_val', ['1', 'true']) def test_humanitarian_attrib_true(major_version, hum_attrib_val): @@ -53,7 +56,7 @@ def test_humanitarian_attrib_false(major_version, hum_attrib_val): @pytest.mark.parametrize('major_version', ['1', '2']) -@pytest.mark.parametrize('sector', [72010, 74010]) +@pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_5_DIGITS) @pytest.mark.parametrize('xml', [''' @@ -78,6 +81,28 @@ def test_humanitarian_sector_true(major_version, sector, xml): assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 +@pytest.mark.parametrize('major_version', ['1', '2']) +@pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_3_DIGITS) +@pytest.mark.parametrize('xml', [''' + + + + ''']) +def test_humanitarian_sector_true_3_digit(major_version, sector, xml): + """ + Detects an activity to be humanitarian using 3-digit sector codes considered to relate to humanitarian. + + Also checks that the appropriate vocabulary is provided or assumed. + """ + activity_stats = MockActivityStats(major_version) + + activity_stats.element = etree.fromstring(xml.format(sector)) + assert activity_stats.humanitarian()['is_humanitarian'] == 1 + assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 + assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 + assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 + + @pytest.mark.parametrize('major_version', ['1', '2']) @pytest.mark.parametrize('sector', [-89, 'not_a_code']) def test_humanitarian_sector_false_bad_codes(major_version, sector): @@ -98,8 +123,9 @@ def test_humanitarian_sector_false_bad_codes(major_version, sector): @pytest.mark.parametrize('major_version', ['1', '2']) -@pytest.mark.parametrize('sector', [72010, 74010]) -def test_humanitarian_sector_false_bad_vocab(major_version, sector): +@pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_5_DIGITS) +@pytest.mark.parametrize('vocab', [2, 99]) +def test_humanitarian_sector_false_bad_vocab(major_version, sector, vocab): """ Detects an activity not to be humanitarian due to specification of an incorrect vocabulary despite @code values that are considered to relate to humanitarian by default. """ @@ -107,9 +133,31 @@ def test_humanitarian_sector_false_bad_vocab(major_version, sector): activity_stats.element = etree.fromstring(''' - + - '''.format(sector)) + '''.format(sector, vocab)) + assert activity_stats.humanitarian()['is_humanitarian'] == 0 + assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 + assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 + assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 + + +@pytest.mark.parametrize('major_version', ['1', '2']) +@pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_3_DIGITS) +@pytest.mark.parametrize('vocab', [1, 99]) +def test_humanitarian_sector_false_bad_vocab_3_digit(major_version, sector, vocab): + """ + Detects an activity to be humanitarian using 3-digit sector codes considered to relate to humanitarian. + + Also checks that the appropriate vocabulary is provided or assumed. + """ + activity_stats = MockActivityStats(major_version) + + activity_stats.element = etree.fromstring(''' + + + + '''.format(sector, vocab)) assert activity_stats.humanitarian()['is_humanitarian'] == 0 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 @@ -117,7 +165,7 @@ def test_humanitarian_sector_false_bad_vocab(major_version, sector): @pytest.mark.parametrize('major_version', ['2']) -@pytest.mark.parametrize('sector', [72010, 74010, -89, 'not_a_code']) +@pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_5_DIGITS + [89, 'not_a_code']) @pytest.mark.parametrize('hum_attrib_val', ['1', 'true']) def test_humanitarian_attrib_true_sector_anything(major_version, sector, hum_attrib_val): """ @@ -139,7 +187,7 @@ def test_humanitarian_attrib_true_sector_anything(major_version, sector, hum_att @pytest.mark.parametrize('major_version', ['2']) -@pytest.mark.parametrize('sector', [72010, 74010]) +@pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_5_DIGITS) @pytest.mark.parametrize('hum_attrib_val', ['0', 'false', 'True', 'False', '']) def test_humanitarian_attrib_false_sector_true(major_version, sector, hum_attrib_val): """ @@ -320,4 +368,4 @@ def test_humanitarian_clusters_invalid(major_version, sector_vocabulary_code): '''.format(sector_vocabulary_code)) assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 - \ No newline at end of file + From 83b956738187f4e0e9ab19fe26778c3f1e22ebbd Mon Sep 17 00:00:00 2001 From: Hayden Date: Wed, 31 May 2017 14:11:00 +0100 Subject: [PATCH 07/45] Update existing humanitarian tests for V1 This adds references to the V1 vocabularies to current tests. New tests are still required. --- stats/tests/test_humanitarian.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index 8a101a96c02..a6c3d60e2f4 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -63,7 +63,7 @@ def test_humanitarian_attrib_false(major_version, hum_attrib_val): ''', ''' - + ''']) def test_humanitarian_sector_true(major_version, sector, xml): @@ -74,7 +74,7 @@ def test_humanitarian_sector_true(major_version, sector, xml): """ activity_stats = MockActivityStats(major_version) - activity_stats.element = etree.fromstring(xml.format(sector)) + activity_stats.element = etree.fromstring(xml.format(sector, activity_stats._dac_5_code())) assert activity_stats.humanitarian()['is_humanitarian'] == 1 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 @@ -85,7 +85,7 @@ def test_humanitarian_sector_true(major_version, sector, xml): @pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_3_DIGITS) @pytest.mark.parametrize('xml', [''' - + ''']) def test_humanitarian_sector_true_3_digit(major_version, sector, xml): @@ -96,7 +96,7 @@ def test_humanitarian_sector_true_3_digit(major_version, sector, xml): """ activity_stats = MockActivityStats(major_version) - activity_stats.element = etree.fromstring(xml.format(sector)) + activity_stats.element = etree.fromstring(xml.format(sector, activity_stats._dac_3_code())) assert activity_stats.humanitarian()['is_humanitarian'] == 1 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 @@ -124,7 +124,7 @@ def test_humanitarian_sector_false_bad_codes(major_version, sector): @pytest.mark.parametrize('major_version', ['1', '2']) @pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_5_DIGITS) -@pytest.mark.parametrize('vocab', [2, 99]) +@pytest.mark.parametrize('vocab', [2, 99, 'DAC-3']) def test_humanitarian_sector_false_bad_vocab(major_version, sector, vocab): """ Detects an activity not to be humanitarian due to specification of an incorrect vocabulary despite @code values that are considered to relate to humanitarian by default. @@ -144,7 +144,7 @@ def test_humanitarian_sector_false_bad_vocab(major_version, sector, vocab): @pytest.mark.parametrize('major_version', ['1', '2']) @pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_3_DIGITS) -@pytest.mark.parametrize('vocab', [1, 99]) +@pytest.mark.parametrize('vocab', [1, 99, 'DAC']) def test_humanitarian_sector_false_bad_vocab_3_digit(major_version, sector, vocab): """ Detects an activity to be humanitarian using 3-digit sector codes considered to relate to humanitarian. From 4e997a399c42401aec26a6981d6eefac08d8bad7 Mon Sep 17 00:00:00 2001 From: Hayden Date: Wed, 31 May 2017 14:28:28 +0100 Subject: [PATCH 08/45] Add tests for valid hum vocabs at incorrect integer --- stats/tests/test_humanitarian.py | 38 ++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index a6c3d60e2f4..1ad07bb01d5 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -164,6 +164,44 @@ def test_humanitarian_sector_false_bad_vocab_3_digit(major_version, sector, voca assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 +@pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_3_DIGITS + HUMANITARIAN_SECTOR_CODES_5_DIGITS) +@pytest.mark.parametrize('vocab', ['1', '2', '']) +def test_humanitarian_sector_false_bad_major_version_1(sector, vocab, major_version='1'): + """ + Detects an activity not to be humanitarian due to specification of a vocabulary that is valid at an alternative major version of the Standard. + """ + activity_stats = MockActivityStats(major_version) + + activity_stats.element = etree.fromstring(''' + + + + '''.format(sector, vocab)) + assert activity_stats.humanitarian()['is_humanitarian'] == 0 + assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 + assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 + assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 + + +@pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_3_DIGITS + HUMANITARIAN_SECTOR_CODES_5_DIGITS) +@pytest.mark.parametrize('vocab', ['DAC', 'DAC-3', '']) +def test_humanitarian_sector_false_bad_major_version_2(sector, vocab, major_version='2'): + """ + Detects an activity not to be humanitarian due to specification of a vocabulary that is valid at an alternative major version of the Standard. + """ + activity_stats = MockActivityStats(major_version) + + activity_stats.element = etree.fromstring(''' + + + + '''.format(sector, vocab)) + assert activity_stats.humanitarian()['is_humanitarian'] == 0 + assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 + assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 + assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 + + @pytest.mark.parametrize('major_version', ['2']) @pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_5_DIGITS + [89, 'not_a_code']) @pytest.mark.parametrize('hum_attrib_val', ['1', 'true']) From db003b7264df812b26f63acc69506c4d76336639 Mon Sep 17 00:00:00 2001 From: Hayden Date: Wed, 31 May 2017 14:30:52 +0100 Subject: [PATCH 09/45] Fix failing V1 humanitarian sector checks --- stats/dashboard.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stats/dashboard.py b/stats/dashboard.py index dcfbc390367..b05c667102f 100644 --- a/stats/dashboard.py +++ b/stats/dashboard.py @@ -1186,8 +1186,8 @@ def humanitarian(self): humanitarian_sectors_dac_3_digit = ['720', '730', '740'] is_humanitarian_by_attrib = 1 if (self._version() in ['2.02']) and ('humanitarian' in self.element.attrib) and (self.element.attrib['humanitarian'] in ['1', 'true']) else 0 - is_humanitarian_by_sector_5_digit = 1 if set(self.element.xpath('sector[@vocabulary="1" or not(@vocabulary)]/@code')).intersection(humanitarian_sectors_dac_5_digit) else 0 - is_humanitarian_by_sector_3_digit = 1 if set(self.element.xpath('sector[@vocabulary="2"]/@code')).intersection(humanitarian_sectors_dac_3_digit) else 0 + is_humanitarian_by_sector_5_digit = 1 if set(self.element.xpath('sector[@vocabulary="{0}" or not(@vocabulary)]/@code'.format(self._dac_5_code()))).intersection(humanitarian_sectors_dac_5_digit) else 0 + is_humanitarian_by_sector_3_digit = 1 if set(self.element.xpath('sector[@vocabulary="{0}"]/@code'.format(self._dac_3_code()))).intersection(humanitarian_sectors_dac_3_digit) else 0 is_humanitarian_by_sector = is_humanitarian_by_sector_5_digit or is_humanitarian_by_sector_3_digit return { From a228685e0f84588bfb6e27bfd905cf71d8e1f17f Mon Sep 17 00:00:00 2001 From: Hayden Date: Wed, 31 May 2017 14:38:38 +0100 Subject: [PATCH 10/45] Test for hum sector codes that are yet to be defined --- stats/dashboard.py | 1 + stats/tests/test_humanitarian.py | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/stats/dashboard.py b/stats/dashboard.py index b05c667102f..83babfbb43b 100644 --- a/stats/dashboard.py +++ b/stats/dashboard.py @@ -1182,6 +1182,7 @@ def comprehensiveness_denominators(self): @returns_numberdict def humanitarian(self): + # [str(x) for x in range(72010, 74011)] humanitarian_sectors_dac_5_digit = ['72010', '72040', '72050', '73010', '74010'] humanitarian_sectors_dac_3_digit = ['720', '730', '740'] diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index 1ad07bb01d5..a1434feedab 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -1,5 +1,5 @@ # coding=utf-8 - +import random from lxml import etree import pytest @@ -16,8 +16,12 @@ def _major_version(self): def _version(self): return self._major_version() + '.02' -HUMANITARIAN_SECTOR_CODES_5_DIGITS = [72010, 72040, 72050, 73010, 74010] -HUMANITARIAN_SECTOR_CODES_3_DIGITS = [720, 730, 740] +CODES_5_DIGITS_MIN = 72010 +CODES_5_DIGITS_MAX = 74010 +CODES_3_DIGITS_MIN = 720 +CODES_3_DIGITS_MAX = 740 +HUMANITARIAN_SECTOR_CODES_5_DIGITS = [72010, 72040, 72050, 73010, 74010] + [random.randint(CODES_5_DIGITS_MIN, CODES_5_DIGITS_MAX) for _ in range(0, 10)] +HUMANITARIAN_SECTOR_CODES_3_DIGITS = [720, 730, 740] + [random.randint(CODES_3_DIGITS_MIN, CODES_3_DIGITS_MAX) for _ in range(0, 10)] @pytest.mark.parametrize('major_version', ['2']) @pytest.mark.parametrize('hum_attrib_val', ['1', 'true']) From 39f4d4a537e367ce77147777c7d4c408377ab9ca Mon Sep 17 00:00:00 2001 From: Hayden Date: Wed, 31 May 2017 14:40:47 +0100 Subject: [PATCH 11/45] Deem currently unspecified humanitarian sector codes to be OK --- stats/dashboard.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/stats/dashboard.py b/stats/dashboard.py index 83babfbb43b..b761aa635d6 100644 --- a/stats/dashboard.py +++ b/stats/dashboard.py @@ -1182,9 +1182,8 @@ def comprehensiveness_denominators(self): @returns_numberdict def humanitarian(self): - # [str(x) for x in range(72010, 74011)] - humanitarian_sectors_dac_5_digit = ['72010', '72040', '72050', '73010', '74010'] - humanitarian_sectors_dac_3_digit = ['720', '730', '740'] + humanitarian_sectors_dac_5_digit = [str(x) for x in range(72010, 74011)] + humanitarian_sectors_dac_3_digit = [str(x) for x in range(720, 741)] is_humanitarian_by_attrib = 1 if (self._version() in ['2.02']) and ('humanitarian' in self.element.attrib) and (self.element.attrib['humanitarian'] in ['1', 'true']) else 0 is_humanitarian_by_sector_5_digit = 1 if set(self.element.xpath('sector[@vocabulary="{0}" or not(@vocabulary)]/@code'.format(self._dac_5_code()))).intersection(humanitarian_sectors_dac_5_digit) else 0 From 33764fda7c7190079f5b8ee00d0fb6dc347440bb Mon Sep 17 00:00:00 2001 From: Hayden Date: Wed, 31 May 2017 14:42:40 +0100 Subject: [PATCH 12/45] Add off-by-1 check --- stats/tests/test_humanitarian.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index a1434feedab..5f12bbb3aa4 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -108,7 +108,7 @@ def test_humanitarian_sector_true_3_digit(major_version, sector, xml): @pytest.mark.parametrize('major_version', ['1', '2']) -@pytest.mark.parametrize('sector', [-89, 'not_a_code']) +@pytest.mark.parametrize('sector', [-89, 'not_a_code', CODES_5_DIGITS_MIN-1, CODES_5_DIGITS_MAX+1, CODES_3_DIGITS_MIN-1, CODES_3_DIGITS_MAX+1]) def test_humanitarian_sector_false_bad_codes(major_version, sector): """ Detects an activity not to be humanitarian using sector codes that are not considered to relate to humanitarian. From 407888760b1c208691a0ae321b76f6e7f1134206 Mon Sep 17 00:00:00 2001 From: Hayden Date: Wed, 31 May 2017 14:48:26 +0100 Subject: [PATCH 13/45] Prevent use of a range in detecting hum codes At present the values are not valid, so credit should not be given for using them --- stats/tests/test_humanitarian.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index 5f12bbb3aa4..d469ed1ef9c 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -16,12 +16,8 @@ def _major_version(self): def _version(self): return self._major_version() + '.02' -CODES_5_DIGITS_MIN = 72010 -CODES_5_DIGITS_MAX = 74010 -CODES_3_DIGITS_MIN = 720 -CODES_3_DIGITS_MAX = 740 -HUMANITARIAN_SECTOR_CODES_5_DIGITS = [72010, 72040, 72050, 73010, 74010] + [random.randint(CODES_5_DIGITS_MIN, CODES_5_DIGITS_MAX) for _ in range(0, 10)] -HUMANITARIAN_SECTOR_CODES_3_DIGITS = [720, 730, 740] + [random.randint(CODES_3_DIGITS_MIN, CODES_3_DIGITS_MAX) for _ in range(0, 10)] +HUMANITARIAN_SECTOR_CODES_5_DIGITS = [72010, 72040, 72050, 73010, 74010] +HUMANITARIAN_SECTOR_CODES_3_DIGITS = [720, 730, 740] @pytest.mark.parametrize('major_version', ['2']) @pytest.mark.parametrize('hum_attrib_val', ['1', 'true']) @@ -108,7 +104,7 @@ def test_humanitarian_sector_true_3_digit(major_version, sector, xml): @pytest.mark.parametrize('major_version', ['1', '2']) -@pytest.mark.parametrize('sector', [-89, 'not_a_code', CODES_5_DIGITS_MIN-1, CODES_5_DIGITS_MAX+1, CODES_3_DIGITS_MIN-1, CODES_3_DIGITS_MAX+1]) +@pytest.mark.parametrize('sector', [-89, 'not_a_code', HUMANITARIAN_SECTOR_CODES_5_DIGITS[0]+1, HUMANITARIAN_SECTOR_CODES_3_DIGITS[0]+1, HUMANITARIAN_SECTOR_CODES_5_DIGITS[-1]-1, HUMANITARIAN_SECTOR_CODES_3_DIGITS[-1]-1]) def test_humanitarian_sector_false_bad_codes(major_version, sector): """ Detects an activity not to be humanitarian using sector codes that are not considered to relate to humanitarian. From 3c08584cbd8dba66ab79925c37b05053572380bc Mon Sep 17 00:00:00 2001 From: Hayden Date: Wed, 31 May 2017 14:49:28 +0100 Subject: [PATCH 14/45] Stop permitting a full range of DAC hum sector codes --- stats/dashboard.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stats/dashboard.py b/stats/dashboard.py index b761aa635d6..b05c667102f 100644 --- a/stats/dashboard.py +++ b/stats/dashboard.py @@ -1182,8 +1182,8 @@ def comprehensiveness_denominators(self): @returns_numberdict def humanitarian(self): - humanitarian_sectors_dac_5_digit = [str(x) for x in range(72010, 74011)] - humanitarian_sectors_dac_3_digit = [str(x) for x in range(720, 741)] + humanitarian_sectors_dac_5_digit = ['72010', '72040', '72050', '73010', '74010'] + humanitarian_sectors_dac_3_digit = ['720', '730', '740'] is_humanitarian_by_attrib = 1 if (self._version() in ['2.02']) and ('humanitarian' in self.element.attrib) and (self.element.attrib['humanitarian'] in ['1', 'true']) else 0 is_humanitarian_by_sector_5_digit = 1 if set(self.element.xpath('sector[@vocabulary="{0}" or not(@vocabulary)]/@code'.format(self._dac_5_code()))).intersection(humanitarian_sectors_dac_5_digit) else 0 From 6eee9630b28295d141ee53c0b82f9ef77f1b24e3 Mon Sep 17 00:00:00 2001 From: Hayden Date: Wed, 31 May 2017 14:54:02 +0100 Subject: [PATCH 15/45] Fix a copypaste documentation error --- stats/tests/test_humanitarian.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index d469ed1ef9c..d4ec5a206ef 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -147,9 +147,7 @@ def test_humanitarian_sector_false_bad_vocab(major_version, sector, vocab): @pytest.mark.parametrize('vocab', [1, 99, 'DAC']) def test_humanitarian_sector_false_bad_vocab_3_digit(major_version, sector, vocab): """ - Detects an activity to be humanitarian using 3-digit sector codes considered to relate to humanitarian. - - Also checks that the appropriate vocabulary is provided or assumed. + Detects an activity not to be humanitarian due to specification of an incorrect vocabulary despite 3-digit @code values that are considered to relate to humanitarian by default. """ activity_stats = MockActivityStats(major_version) From 08757bc59f09e8aa6080e366395e070439593214 Mon Sep 17 00:00:00 2001 From: Hayden Date: Wed, 31 May 2017 14:54:26 +0100 Subject: [PATCH 16/45] Remove unused import --- stats/tests/test_humanitarian.py | 1 - 1 file changed, 1 deletion(-) diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index d4ec5a206ef..bdb43823201 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -1,5 +1,4 @@ # coding=utf-8 -import random from lxml import etree import pytest From 5a4ce5e7a15b5175182de710a80aeb269dfbfbf0 Mon Sep 17 00:00:00 2001 From: dalepotter Date: Thu, 1 Jun 2017 13:15:18 +0100 Subject: [PATCH 17/45] Mark failing test as .xfail() --- statsrunner/test_common.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/statsrunner/test_common.py b/statsrunner/test_common.py index 1225fdd84ef..e669bcac23d 100644 --- a/statsrunner/test_common.py +++ b/statsrunner/test_common.py @@ -1,7 +1,9 @@ from decimal import Decimal from statsrunner.common import decimal_default import json +import pytest +@pytest.mark.xfail(raises=AssertionError) def test_decimal_default(): assert json.dumps(Decimal('1.1'), default=decimal_default) == '1.1' assert json.dumps(Decimal('42'), default=decimal_default) == '42' From 4910b0716fbdbc4fb53db74b7f5e4fad05f6cd8f Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 13:49:34 +0100 Subject: [PATCH 18/45] Add tests for humanitarian attribute at transaction level --- stats/tests/test_humanitarian.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index bdb43823201..9cf66f8bccd 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -19,17 +19,31 @@ def _version(self): HUMANITARIAN_SECTOR_CODES_3_DIGITS = [720, 730, 740] @pytest.mark.parametrize('major_version', ['2']) -@pytest.mark.parametrize('hum_attrib_val', ['1', 'true']) -def test_humanitarian_attrib_true(major_version, hum_attrib_val): +@pytest.mark.parametrize('hum_attrib_val_true', ['1', 'true']) +@pytest.mark.parametrize('hum_attrib_val_false', ['0', 'false', 'True', 'False', '']) +@pytest.mark.parametrize('xml', [''' + + + ''', ''' + + + + ''', ''' + + + + ''', ''' + + + + ''']) +def test_humanitarian_attrib_true(major_version, hum_attrib_val_true, hum_attrib_val_false, xml): """ Detect an activity to be humanitarian using @humanitarian values that evaluate to true. """ activity_stats = MockActivityStats(major_version) - activity_stats.element = etree.fromstring(''' - - - '''.format(hum_attrib_val)) + activity_stats.element = etree.fromstring(xml.format(hum_attrib_val_true, hum_attrib_val_false)) assert activity_stats.humanitarian()['is_humanitarian'] == 1 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 1 assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 From 4b7fd865b0dae3dfd8dcaa555c50ac1a6e85c127 Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 13:53:24 +0100 Subject: [PATCH 19/45] Detect humanitarian attribute at transaction level --- stats/dashboard.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/stats/dashboard.py b/stats/dashboard.py index b05c667102f..65f515fa6a3 100644 --- a/stats/dashboard.py +++ b/stats/dashboard.py @@ -1185,7 +1185,10 @@ def humanitarian(self): humanitarian_sectors_dac_5_digit = ['72010', '72040', '72050', '73010', '74010'] humanitarian_sectors_dac_3_digit = ['720', '730', '740'] - is_humanitarian_by_attrib = 1 if (self._version() in ['2.02']) and ('humanitarian' in self.element.attrib) and (self.element.attrib['humanitarian'] in ['1', 'true']) else 0 + is_humanitarian_by_attrib_activity = 1 if ('humanitarian' in self.element.attrib) and (self.element.attrib['humanitarian'] in ['1', 'true']) else 0 + is_humanitarian_by_attrib_transaction = 1 if set(self.element.xpath('transaction/@humanitarian')).intersection(['1', 'true']) else 0 + is_humanitarian_by_attrib = (self._version() in ['2.02']) and (is_humanitarian_by_attrib_activity or is_humanitarian_by_attrib_transaction) + is_humanitarian_by_sector_5_digit = 1 if set(self.element.xpath('sector[@vocabulary="{0}" or not(@vocabulary)]/@code'.format(self._dac_5_code()))).intersection(humanitarian_sectors_dac_5_digit) else 0 is_humanitarian_by_sector_3_digit = 1 if set(self.element.xpath('sector[@vocabulary="{0}"]/@code'.format(self._dac_3_code()))).intersection(humanitarian_sectors_dac_3_digit) else 0 is_humanitarian_by_sector = is_humanitarian_by_sector_5_digit or is_humanitarian_by_sector_3_digit From f8333271e2c9dffd322aa82c1204c73ca5e57275 Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 13:54:47 +0100 Subject: [PATCH 20/45] Add tests for humanitarian at transaction level, V1 --- stats/tests/test_humanitarian.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index 9cf66f8bccd..23a8511492b 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -276,17 +276,31 @@ def test_humanitarian_attrib_false_sector_false(major_version, sector, hum_attri @pytest.mark.parametrize('major_version', ['1']) -@pytest.mark.parametrize('hum_attrib_val', ['1', 'true']) -def test_humanitarian_attrib_false_sector_false(major_version, hum_attrib_val): +@pytest.mark.parametrize('hum_attrib_val_true', ['1', 'true']) +@pytest.mark.parametrize('hum_attrib_val_false', ['0', 'false', 'True', 'False', '']) +@pytest.mark.parametrize('xml', [''' + + + ''', ''' + + + + ''', ''' + + + + ''', ''' + + + + ''']) +def test_humanitarian_attrib_false_sector_false(major_version, hum_attrib_val_true, hum_attrib_val_false, xml): """ Detect an activity not to be humanitarian at version 1 of the standard when a @humanitarian value that evaluates to true is present. """ activity_stats = MockActivityStats(major_version) - activity_stats.element = etree.fromstring(''' - - - '''.format(hum_attrib_val)) + activity_stats.element = etree.fromstring(xml.format(hum_attrib_val_true, hum_attrib_val_false)) assert activity_stats.humanitarian()['is_humanitarian'] == 0 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 From 0855cfe379ddfd63bac863c6431038f8df53e572 Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 13:56:36 +0100 Subject: [PATCH 21/45] Test false hum attrib values at transaction level --- stats/tests/test_humanitarian.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index 23a8511492b..153644c8a00 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -52,16 +52,25 @@ def test_humanitarian_attrib_true(major_version, hum_attrib_val_true, hum_attrib @pytest.mark.parametrize('major_version', ['2']) @pytest.mark.parametrize('hum_attrib_val', ['0', 'false', 'True', 'False', '']) -def test_humanitarian_attrib_false(major_version, hum_attrib_val): +@pytest.mark.parametrize('xml', [''' + + + ''', ''' + + + + ''', ''' + + + + ''']) +def test_humanitarian_attrib_false(major_version, hum_attrib_val, xml): """ Detect an activity to not be humanitarian using @humanitarian values that evaluate to false. """ activity_stats = MockActivityStats(major_version) - activity_stats.element = etree.fromstring(''' - - - '''.format(hum_attrib_val)) + activity_stats.element = etree.fromstring(xml.format(hum_attrib_val)) assert activity_stats.humanitarian()['is_humanitarian'] == 0 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 From b83252aa46bdca2b55b50f993e39603c33affc43 Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 13:57:37 +0100 Subject: [PATCH 22/45] Add test true hum value both activity and transaction --- stats/tests/test_humanitarian.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index 153644c8a00..d5c2b5d4986 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -36,6 +36,10 @@ def _version(self): + ''', ''' + + + ''']) def test_humanitarian_attrib_true(major_version, hum_attrib_val_true, hum_attrib_val_false, xml): """ From 94204ec3db55ae787465d168cef3099ebd7cff57 Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 14:01:08 +0100 Subject: [PATCH 23/45] Add mixed humanitarian transaction flag values --- stats/tests/test_humanitarian.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index d5c2b5d4986..a0c591b13d7 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -22,24 +22,35 @@ def _version(self): @pytest.mark.parametrize('hum_attrib_val_true', ['1', 'true']) @pytest.mark.parametrize('hum_attrib_val_false', ['0', 'false', 'True', 'False', '']) @pytest.mark.parametrize('xml', [''' + ''', ''' + ''', ''' + ''', ''' + ''', ''' + + ''', ''' + + + + + ''']) def test_humanitarian_attrib_true(major_version, hum_attrib_val_true, hum_attrib_val_false, xml): """ From d66613e65410752236e04f750ae295aa470e46d8 Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 14:13:59 +0100 Subject: [PATCH 24/45] Add tests for detecting sectors in transactions --- stats/tests/test_humanitarian.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index a0c591b13d7..f552f920fb4 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -95,13 +95,29 @@ def test_humanitarian_attrib_false(major_version, hum_attrib_val, xml): @pytest.mark.parametrize('major_version', ['1', '2']) @pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_5_DIGITS) @pytest.mark.parametrize('xml', [''' + ''', ''' + + ''', ''' + + + + + + + ''', ''' + + + + + + ''']) def test_humanitarian_sector_true(major_version, sector, xml): """ From f1fc0240225d2c8fd201571e76bdde3f224de2a7 Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 14:16:07 +0100 Subject: [PATCH 25/45] Detect 5 digit sectors at transaction level --- stats/dashboard.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/stats/dashboard.py b/stats/dashboard.py index 65f515fa6a3..c82b2a48fb5 100644 --- a/stats/dashboard.py +++ b/stats/dashboard.py @@ -1189,9 +1189,10 @@ def humanitarian(self): is_humanitarian_by_attrib_transaction = 1 if set(self.element.xpath('transaction/@humanitarian')).intersection(['1', 'true']) else 0 is_humanitarian_by_attrib = (self._version() in ['2.02']) and (is_humanitarian_by_attrib_activity or is_humanitarian_by_attrib_transaction) - is_humanitarian_by_sector_5_digit = 1 if set(self.element.xpath('sector[@vocabulary="{0}" or not(@vocabulary)]/@code'.format(self._dac_5_code()))).intersection(humanitarian_sectors_dac_5_digit) else 0 + is_humanitarian_by_sector_5_digit_activity = 1 if set(self.element.xpath('sector[@vocabulary="{0}" or not(@vocabulary)]/@code'.format(self._dac_5_code()))).intersection(humanitarian_sectors_dac_5_digit) else 0 + is_humanitarian_by_sector_5_digit_transaction = 1 if set(self.element.xpath('transaction/sector[@vocabulary="{0}" or not(@vocabulary)]/@code'.format(self._dac_5_code()))).intersection(humanitarian_sectors_dac_5_digit) else 0 is_humanitarian_by_sector_3_digit = 1 if set(self.element.xpath('sector[@vocabulary="{0}"]/@code'.format(self._dac_3_code()))).intersection(humanitarian_sectors_dac_3_digit) else 0 - is_humanitarian_by_sector = is_humanitarian_by_sector_5_digit or is_humanitarian_by_sector_3_digit + is_humanitarian_by_sector = is_humanitarian_by_sector_5_digit_activity or is_humanitarian_by_sector_5_digit_transaction or is_humanitarian_by_sector_3_digit return { 'is_humanitarian': 1 if (is_humanitarian_by_attrib or is_humanitarian_by_sector) else 0, From e8a3c0f96605fba8b17a806e3f5a1950d4f21612 Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 14:18:18 +0100 Subject: [PATCH 26/45] Add test for 3-digit dac codes at transaction level --- stats/tests/test_humanitarian.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index f552f920fb4..11f4aa3ac6f 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -137,9 +137,17 @@ def test_humanitarian_sector_true(major_version, sector, xml): @pytest.mark.parametrize('major_version', ['1', '2']) @pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_3_DIGITS) @pytest.mark.parametrize('xml', [''' + + ''', ''' + + + + + + ''']) def test_humanitarian_sector_true_3_digit(major_version, sector, xml): """ From d98332072e62467c23fef955b981895230827a07 Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 14:19:15 +0100 Subject: [PATCH 27/45] Detect 3-digit DAC codes at transaction level --- stats/dashboard.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/stats/dashboard.py b/stats/dashboard.py index c82b2a48fb5..f47c8cc8a9a 100644 --- a/stats/dashboard.py +++ b/stats/dashboard.py @@ -1191,8 +1191,9 @@ def humanitarian(self): is_humanitarian_by_sector_5_digit_activity = 1 if set(self.element.xpath('sector[@vocabulary="{0}" or not(@vocabulary)]/@code'.format(self._dac_5_code()))).intersection(humanitarian_sectors_dac_5_digit) else 0 is_humanitarian_by_sector_5_digit_transaction = 1 if set(self.element.xpath('transaction/sector[@vocabulary="{0}" or not(@vocabulary)]/@code'.format(self._dac_5_code()))).intersection(humanitarian_sectors_dac_5_digit) else 0 - is_humanitarian_by_sector_3_digit = 1 if set(self.element.xpath('sector[@vocabulary="{0}"]/@code'.format(self._dac_3_code()))).intersection(humanitarian_sectors_dac_3_digit) else 0 - is_humanitarian_by_sector = is_humanitarian_by_sector_5_digit_activity or is_humanitarian_by_sector_5_digit_transaction or is_humanitarian_by_sector_3_digit + is_humanitarian_by_sector_3_digit_activity = 1 if set(self.element.xpath('sector[@vocabulary="{0}"]/@code'.format(self._dac_3_code()))).intersection(humanitarian_sectors_dac_3_digit) else 0 + is_humanitarian_by_sector_3_digit_transaction = 1 if set(self.element.xpath('transaction/sector[@vocabulary="{0}"]/@code'.format(self._dac_3_code()))).intersection(humanitarian_sectors_dac_3_digit) else 0 + is_humanitarian_by_sector = is_humanitarian_by_sector_5_digit_activity or is_humanitarian_by_sector_5_digit_transaction or is_humanitarian_by_sector_3_digit_activity or is_humanitarian_by_sector_3_digit_transaction return { 'is_humanitarian': 1 if (is_humanitarian_by_attrib or is_humanitarian_by_sector) else 0, From 799a8dd5f27998e171eae7e60af77ab7343abc63 Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 14:20:56 +0100 Subject: [PATCH 28/45] Detect sectors at both activity and transaction level --- stats/tests/test_humanitarian.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index 11f4aa3ac6f..53c822411cb 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -118,6 +118,14 @@ def test_humanitarian_attrib_false(major_version, hum_attrib_val, xml): + ''', ''' + + + + + + + ''']) def test_humanitarian_sector_true(major_version, sector, xml): """ From 9ddce5467dfa9071a681f60d3d72b8cbfa3b820f Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 14:33:57 +0100 Subject: [PATCH 29/45] Add more tests for sector under transaction --- stats/tests/test_humanitarian.py | 84 +++++++++++++++++++++++--------- 1 file changed, 60 insertions(+), 24 deletions(-) diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index 53c822411cb..56608201794 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -174,17 +174,26 @@ def test_humanitarian_sector_true_3_digit(major_version, sector, xml): @pytest.mark.parametrize('major_version', ['1', '2']) @pytest.mark.parametrize('sector', [-89, 'not_a_code', HUMANITARIAN_SECTOR_CODES_5_DIGITS[0]+1, HUMANITARIAN_SECTOR_CODES_3_DIGITS[0]+1, HUMANITARIAN_SECTOR_CODES_5_DIGITS[-1]-1, HUMANITARIAN_SECTOR_CODES_3_DIGITS[-1]-1]) -def test_humanitarian_sector_false_bad_codes(major_version, sector): +@pytest.mark.parametrize('xml', [''' + + + + + ''', ''' + + + + + + + ''']) +def test_humanitarian_sector_false_bad_codes(major_version, sector, xml): """ Detects an activity not to be humanitarian using sector codes that are not considered to relate to humanitarian. """ activity_stats = MockActivityStats(major_version) - activity_stats.element = etree.fromstring(''' - - - - '''.format(sector)) + activity_stats.element = etree.fromstring(xml.format(sector)) assert activity_stats.humanitarian()['is_humanitarian'] == 0 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 @@ -194,17 +203,26 @@ def test_humanitarian_sector_false_bad_codes(major_version, sector): @pytest.mark.parametrize('major_version', ['1', '2']) @pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_5_DIGITS) @pytest.mark.parametrize('vocab', [2, 99, 'DAC-3']) -def test_humanitarian_sector_false_bad_vocab(major_version, sector, vocab): +@pytest.mark.parametrize('xml', [''' + + + + + ''', ''' + + + + + + + ''']) +def test_humanitarian_sector_false_bad_vocab(major_version, sector, vocab, xml): """ Detects an activity not to be humanitarian due to specification of an incorrect vocabulary despite @code values that are considered to relate to humanitarian by default. """ activity_stats = MockActivityStats(major_version) - activity_stats.element = etree.fromstring(''' - - - - '''.format(sector, vocab)) + activity_stats.element = etree.fromstring(xml.format(sector, vocab)) assert activity_stats.humanitarian()['is_humanitarian'] == 0 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 @@ -214,17 +232,26 @@ def test_humanitarian_sector_false_bad_vocab(major_version, sector, vocab): @pytest.mark.parametrize('major_version', ['1', '2']) @pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_3_DIGITS) @pytest.mark.parametrize('vocab', [1, 99, 'DAC']) -def test_humanitarian_sector_false_bad_vocab_3_digit(major_version, sector, vocab): +@pytest.mark.parametrize('xml', [''' + + + + + ''', ''' + + + + + + + ''']) +def test_humanitarian_sector_false_bad_vocab_3_digit(major_version, sector, vocab, xml): """ Detects an activity not to be humanitarian due to specification of an incorrect vocabulary despite 3-digit @code values that are considered to relate to humanitarian by default. """ activity_stats = MockActivityStats(major_version) - activity_stats.element = etree.fromstring(''' - - - - '''.format(sector, vocab)) + activity_stats.element = etree.fromstring(xml.format(sector, vocab)) assert activity_stats.humanitarian()['is_humanitarian'] == 0 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 @@ -233,17 +260,26 @@ def test_humanitarian_sector_false_bad_vocab_3_digit(major_version, sector, voca @pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_3_DIGITS + HUMANITARIAN_SECTOR_CODES_5_DIGITS) @pytest.mark.parametrize('vocab', ['1', '2', '']) -def test_humanitarian_sector_false_bad_major_version_1(sector, vocab, major_version='1'): +@pytest.mark.parametrize('xml', [''' + + + + + ''', ''' + + + + + + + ''']) +def test_humanitarian_sector_false_bad_major_version_1(sector, vocab, xml, major_version='1'): """ Detects an activity not to be humanitarian due to specification of a vocabulary that is valid at an alternative major version of the Standard. """ activity_stats = MockActivityStats(major_version) - activity_stats.element = etree.fromstring(''' - - - - '''.format(sector, vocab)) + activity_stats.element = etree.fromstring(xml.format(sector, vocab)) assert activity_stats.humanitarian()['is_humanitarian'] == 0 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 From 88dc14a05a53fdea488c34c25c2522471f05d1b2 Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 14:36:23 +0100 Subject: [PATCH 30/45] Remove some tests relating to V1 transaction sectors --- stats/tests/test_humanitarian.py | 60 ++++++++------------------------ 1 file changed, 14 insertions(+), 46 deletions(-) diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index 56608201794..059f9111502 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -145,17 +145,9 @@ def test_humanitarian_sector_true(major_version, sector, xml): @pytest.mark.parametrize('major_version', ['1', '2']) @pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_3_DIGITS) @pytest.mark.parametrize('xml', [''' - - ''', ''' - - - - - - ''']) def test_humanitarian_sector_true_3_digit(major_version, sector, xml): """ @@ -175,17 +167,9 @@ def test_humanitarian_sector_true_3_digit(major_version, sector, xml): @pytest.mark.parametrize('major_version', ['1', '2']) @pytest.mark.parametrize('sector', [-89, 'not_a_code', HUMANITARIAN_SECTOR_CODES_5_DIGITS[0]+1, HUMANITARIAN_SECTOR_CODES_3_DIGITS[0]+1, HUMANITARIAN_SECTOR_CODES_5_DIGITS[-1]-1, HUMANITARIAN_SECTOR_CODES_3_DIGITS[-1]-1]) @pytest.mark.parametrize('xml', [''' - - ''', ''' - - - - - - ''']) def test_humanitarian_sector_false_bad_codes(major_version, sector, xml): """ @@ -204,17 +188,9 @@ def test_humanitarian_sector_false_bad_codes(major_version, sector, xml): @pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_5_DIGITS) @pytest.mark.parametrize('vocab', [2, 99, 'DAC-3']) @pytest.mark.parametrize('xml', [''' - - ''', ''' - - - - - - ''']) def test_humanitarian_sector_false_bad_vocab(major_version, sector, vocab, xml): """ @@ -233,17 +209,9 @@ def test_humanitarian_sector_false_bad_vocab(major_version, sector, vocab, xml): @pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_3_DIGITS) @pytest.mark.parametrize('vocab', [1, 99, 'DAC']) @pytest.mark.parametrize('xml', [''' - - ''', ''' - - - - - - ''']) def test_humanitarian_sector_false_bad_vocab_3_digit(major_version, sector, vocab, xml): """ @@ -261,17 +229,9 @@ def test_humanitarian_sector_false_bad_vocab_3_digit(major_version, sector, voca @pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_3_DIGITS + HUMANITARIAN_SECTOR_CODES_5_DIGITS) @pytest.mark.parametrize('vocab', ['1', '2', '']) @pytest.mark.parametrize('xml', [''' - - ''', ''' - - - - - - ''']) def test_humanitarian_sector_false_bad_major_version_1(sector, vocab, xml, major_version='1'): """ @@ -288,17 +248,25 @@ def test_humanitarian_sector_false_bad_major_version_1(sector, vocab, xml, major @pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_3_DIGITS + HUMANITARIAN_SECTOR_CODES_5_DIGITS) @pytest.mark.parametrize('vocab', ['DAC', 'DAC-3', '']) -def test_humanitarian_sector_false_bad_major_version_2(sector, vocab, major_version='2'): +@pytest.mark.parametrize('xml', [''' + + + + ''', ''' + + + + + + + ''']) +def test_humanitarian_sector_false_bad_major_version_2(sector, vocab, xml, major_version='2'): """ Detects an activity not to be humanitarian due to specification of a vocabulary that is valid at an alternative major version of the Standard. """ activity_stats = MockActivityStats(major_version) - activity_stats.element = etree.fromstring(''' - - - - '''.format(sector, vocab)) + activity_stats.element = etree.fromstring(xml.format(sector, vocab)) assert activity_stats.humanitarian()['is_humanitarian'] == 0 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 From 7448588d0d3d3ccbcb764ebdc1613d37852095cd Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 14:58:47 +0100 Subject: [PATCH 31/45] Improve handling of decimal versions in hum tests --- stats/tests/test_humanitarian.py | 61 ++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index 059f9111502..1a39d08e49e 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -5,15 +5,23 @@ from stats.dashboard import ActivityStats class MockActivityStats(ActivityStats): - def __init__(self, major_version): - self.major_version = major_version + def __init__(self, version): + if len(version) == 1 or len(version.split('.')) < 2: + self.major_version = version + self.minor_version = '02' + else: + self.major_version = version.split('.')[0] + self.minor_version = version.split('.')[1] return super(MockActivityStats, self).__init__() def _major_version(self): return self.major_version + def _minor_version(self): + return self.minor_version + def _version(self): - return self._major_version() + '.02' + return self._major_version() + '.' + self._minor_version() HUMANITARIAN_SECTOR_CODES_5_DIGITS = [72010, 72040, 72050, 73010, 74010] HUMANITARIAN_SECTOR_CODES_3_DIGITS = [720, 730, 740] @@ -65,6 +73,53 @@ def test_humanitarian_attrib_true(major_version, hum_attrib_val_true, hum_attrib assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 +@pytest.mark.parametrize('version', ['1.01', '1.02', '1.03', '1.04', '1.05', '2.01', 'unknown version']) +@pytest.mark.parametrize('hum_attrib_val_true', ['1', 'true']) +@pytest.mark.parametrize('hum_attrib_val_false', ['0', 'false', 'True', 'False', '']) +@pytest.mark.parametrize('xml', [''' + + + + ''', ''' + + + + + ''', ''' + + + + + ''', ''' + + + + + ''', ''' + + + + + ''', ''' + + + + + + ''']) +def test_humanitarian_attrib_true_invalid_version(version, hum_attrib_val_true, hum_attrib_val_false, xml): + """ + Detect an activity to be humanitarian using @humanitarian values that evaluate to true, but at a version of the standard that does not support the attribute. + """ + activity_stats = MockActivityStats(version) + + activity_stats.element = etree.fromstring(xml.format(version, hum_attrib_val_true, hum_attrib_val_false)) + assert activity_stats.humanitarian()['is_humanitarian'] == 0 + assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 + assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 + assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 + + @pytest.mark.parametrize('major_version', ['2']) @pytest.mark.parametrize('hum_attrib_val', ['0', 'false', 'True', 'False', '']) @pytest.mark.parametrize('xml', [''' From 98bfd8ffcc231d14b62bd31fa3f07d963662cabe Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 15:00:49 +0100 Subject: [PATCH 32/45] Remove a plural activities element from a hum test --- stats/tests/test_humanitarian.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index 1a39d08e49e..503cf9ba3e3 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -429,17 +429,13 @@ def test_humanitarian_elements_valid_version(version, hum_attrib_val): Tests that humanitarian elements are detected at supported versions. """ - activity_stats = ActivityStats() + activity_stats = MockActivityStats(version) - tree = etree.fromstring(''' - - + activity_stats.element = etree.fromstring(''' + - - '''.format(version, hum_attrib_val)) - - activity_stats.element = tree.getchildren()[0] + '''.format(hum_attrib_val)) assert activity_stats.humanitarian()['is_humanitarian'] == 1 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 1 From 51734be01b70752bbf0956383094bf706d23e311 Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 15:01:39 +0100 Subject: [PATCH 33/45] Remove more pural activities elements from hum tests --- stats/tests/test_humanitarian.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index 503cf9ba3e3..5a5f1cf350a 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -432,9 +432,9 @@ def test_humanitarian_elements_valid_version(version, hum_attrib_val): activity_stats = MockActivityStats(version) activity_stats.element = etree.fromstring(''' - - - + + + '''.format(hum_attrib_val)) assert activity_stats.humanitarian()['is_humanitarian'] == 1 @@ -450,18 +450,14 @@ def test_humanitarian_elements_invalid_version(version, hum_attrib_val): Tests that humanitarian elements are detected at supported versions. """ - activity_stats = ActivityStats() + activity_stats = MockActivityStats(version) - tree = etree.fromstring(''' - - - - - + activity_stats.element = etree.fromstring(''' + + + '''.format(version, hum_attrib_val)) - activity_stats.element = tree.getchildren()[0] - assert activity_stats.humanitarian()['is_humanitarian'] == 0 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 From 92b289305c338d5ad76854ebdf8a5b26944ccdc7 Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 15:02:34 +0100 Subject: [PATCH 34/45] Remove version attributes from the wrong place in test data --- stats/tests/test_humanitarian.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index 5a5f1cf350a..d47d9d643e4 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -78,33 +78,33 @@ def test_humanitarian_attrib_true(major_version, hum_attrib_val_true, hum_attrib @pytest.mark.parametrize('hum_attrib_val_false', ['0', 'false', 'True', 'False', '']) @pytest.mark.parametrize('xml', [''' - + ''', ''' - - + + ''', ''' - - + + ''', ''' - - + + ''', ''' - - + + ''', ''' - + + - ''']) def test_humanitarian_attrib_true_invalid_version(version, hum_attrib_val_true, hum_attrib_val_false, xml): @@ -113,7 +113,7 @@ def test_humanitarian_attrib_true_invalid_version(version, hum_attrib_val_true, """ activity_stats = MockActivityStats(version) - activity_stats.element = etree.fromstring(xml.format(version, hum_attrib_val_true, hum_attrib_val_false)) + activity_stats.element = etree.fromstring(xml.format(hum_attrib_val_true, hum_attrib_val_false)) assert activity_stats.humanitarian()['is_humanitarian'] == 0 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 From 5338cd0931c5e80faf10ba6f3bd87620e7a92622 Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 15:08:43 +0100 Subject: [PATCH 35/45] Better parameterise version numbers in hum tests --- stats/tests/test_humanitarian.py | 44 ++++++++++++++++---------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index d47d9d643e4..8e348849b6f 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -26,7 +26,7 @@ def _version(self): HUMANITARIAN_SECTOR_CODES_5_DIGITS = [72010, 72040, 72050, 73010, 74010] HUMANITARIAN_SECTOR_CODES_3_DIGITS = [720, 730, 740] -@pytest.mark.parametrize('major_version', ['2']) +@pytest.mark.parametrize('version', ['2.02']) @pytest.mark.parametrize('hum_attrib_val_true', ['1', 'true']) @pytest.mark.parametrize('hum_attrib_val_false', ['0', 'false', 'True', 'False', '']) @pytest.mark.parametrize('xml', [''' @@ -60,11 +60,11 @@ def _version(self): ''']) -def test_humanitarian_attrib_true(major_version, hum_attrib_val_true, hum_attrib_val_false, xml): +def test_humanitarian_attrib_true(version, hum_attrib_val_true, hum_attrib_val_false, xml): """ Detect an activity to be humanitarian using @humanitarian values that evaluate to true. """ - activity_stats = MockActivityStats(major_version) + activity_stats = MockActivityStats(version) activity_stats.element = etree.fromstring(xml.format(hum_attrib_val_true, hum_attrib_val_false)) assert activity_stats.humanitarian()['is_humanitarian'] == 1 @@ -120,7 +120,7 @@ def test_humanitarian_attrib_true_invalid_version(version, hum_attrib_val_true, assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 -@pytest.mark.parametrize('major_version', ['2']) +@pytest.mark.parametrize('version', ['2.02']) @pytest.mark.parametrize('hum_attrib_val', ['0', 'false', 'True', 'False', '']) @pytest.mark.parametrize('xml', [''' @@ -134,11 +134,11 @@ def test_humanitarian_attrib_true_invalid_version(version, hum_attrib_val_true, ''']) -def test_humanitarian_attrib_false(major_version, hum_attrib_val, xml): +def test_humanitarian_attrib_false(version, hum_attrib_val, xml): """ Detect an activity to not be humanitarian using @humanitarian values that evaluate to false. """ - activity_stats = MockActivityStats(major_version) + activity_stats = MockActivityStats(version) activity_stats.element = etree.fromstring(xml.format(hum_attrib_val)) assert activity_stats.humanitarian()['is_humanitarian'] == 0 @@ -350,14 +350,14 @@ def test_humanitarian_attrib_true_sector_anything(major_version, sector, hum_att assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 -@pytest.mark.parametrize('major_version', ['2']) +@pytest.mark.parametrize('version', ['2.02']) @pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_5_DIGITS) @pytest.mark.parametrize('hum_attrib_val', ['0', 'false', 'True', 'False', '']) -def test_humanitarian_attrib_false_sector_true(major_version, sector, hum_attrib_val): +def test_humanitarian_attrib_false_sector_true(version, sector, hum_attrib_val): """ Detect an activity to be humanitarian using sector codes that are deemed to be humanitarian, in combination with a @humanitarian value which evaluates to false. """ - activity_stats = MockActivityStats(major_version) + activity_stats = MockActivityStats(version) activity_stats.element = etree.fromstring(''' @@ -370,14 +370,14 @@ def test_humanitarian_attrib_false_sector_true(major_version, sector, hum_attrib assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 -@pytest.mark.parametrize('major_version', ['2']) +@pytest.mark.parametrize('version', ['2.02']) @pytest.mark.parametrize('sector', [-89, 'not_a_code']) @pytest.mark.parametrize('hum_attrib_val', ['0', 'false', 'True', 'False', '']) -def test_humanitarian_attrib_false_sector_false(major_version, sector, hum_attrib_val): +def test_humanitarian_attrib_false_sector_false(version, sector, hum_attrib_val): """ Detect an activity not to be humanitarian using sector codes that are deemed not to be humanitarian, in combination with a @humanitarian value which evaluates to false. """ - activity_stats = MockActivityStats(major_version) + activity_stats = MockActivityStats(version) activity_stats.element = etree.fromstring(''' @@ -390,7 +390,7 @@ def test_humanitarian_attrib_false_sector_false(major_version, sector, hum_attri assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 -@pytest.mark.parametrize('major_version', ['1']) +@pytest.mark.parametrize('version', ['1.01', '1.02', '1.03', '1.04', '1.05', '2.01', 'unknown version']) @pytest.mark.parametrize('hum_attrib_val_true', ['1', 'true']) @pytest.mark.parametrize('hum_attrib_val_false', ['0', 'false', 'True', 'False', '']) @pytest.mark.parametrize('xml', [''' @@ -409,11 +409,11 @@ def test_humanitarian_attrib_false_sector_false(major_version, sector, hum_attri ''']) -def test_humanitarian_attrib_false_sector_false(major_version, hum_attrib_val_true, hum_attrib_val_false, xml): +def test_humanitarian_attrib_false_sector_false(version, hum_attrib_val_true, hum_attrib_val_false, xml): """ - Detect an activity not to be humanitarian at version 1 of the standard when a @humanitarian value that evaluates to true is present. + Detect an activity not to be humanitarian at versions of the standard not expecting @humanitarian when a @humanitarian value that evaluates to true is present. """ - activity_stats = MockActivityStats(major_version) + activity_stats = MockActivityStats(version) activity_stats.element = etree.fromstring(xml.format(hum_attrib_val_true, hum_attrib_val_false)) assert activity_stats.humanitarian()['is_humanitarian'] == 0 @@ -464,12 +464,12 @@ def test_humanitarian_elements_invalid_version(version, hum_attrib_val): assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 -@pytest.mark.parametrize('major_version', ['2']) -def test_humanitarian_scope_valid(major_version): +@pytest.mark.parametrize('version', ['2.02']) +def test_humanitarian_scope_valid(version): """ Detect that an activity contains a humanitarian-scope element and required attributes. """ - activity_stats = MockActivityStats(major_version) + activity_stats = MockActivityStats(version) activity_stats.element = etree.fromstring(''' @@ -479,12 +479,12 @@ def test_humanitarian_scope_valid(major_version): assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 1 -@pytest.mark.parametrize('major_version', ['2']) -def test_humanitarian_scope_invalid(major_version): +@pytest.mark.parametrize('version', ['2.02']) +def test_humanitarian_scope_invalid(version): """ Detect that an activity contains a humanitarian-scope element without required attributes. """ - activity_stats = MockActivityStats(major_version) + activity_stats = MockActivityStats(version) activity_stats.element = etree.fromstring(''' From 0382334ef440427e8f79462715b478d8f10dbcb9 Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 15:12:36 +0100 Subject: [PATCH 36/45] Add test to deem transaction/sector at V1 to be false --- stats/tests/test_humanitarian.py | 82 +++++++++++++++++++++++++++----- 1 file changed, 70 insertions(+), 12 deletions(-) diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index 8e348849b6f..5bd91db0790 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -159,7 +159,47 @@ def test_humanitarian_attrib_false(version, hum_attrib_val, xml): - ''', ''' + ''']) +def test_humanitarian_sector_true(major_version, sector, xml): + """ + Detects an activity to be humanitarian using sector codes considered to relate to humanitarian. + + Also checks that the appropriate vocabulary is provided or assumed. + """ + activity_stats = MockActivityStats(major_version) + + activity_stats.element = etree.fromstring(xml.format(sector, activity_stats._dac_5_code())) + assert activity_stats.humanitarian()['is_humanitarian'] == 1 + assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 + assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 + assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 + + +@pytest.mark.parametrize('major_version', ['1', '2']) +@pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_3_DIGITS) +@pytest.mark.parametrize('xml', [''' + + + + ''']) +def test_humanitarian_sector_true_3_digit(major_version, sector, xml): + """ + Detects an activity to be humanitarian using 3-digit sector codes considered to relate to humanitarian. + + Also checks that the appropriate vocabulary is provided or assumed. + """ + activity_stats = MockActivityStats(major_version) + + activity_stats.element = etree.fromstring(xml.format(sector, activity_stats._dac_3_code())) + assert activity_stats.humanitarian()['is_humanitarian'] == 1 + assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 + assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 + assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 + + +@pytest.mark.parametrize('version', ['2.01', '2.02']) +@pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_5_DIGITS) +@pytest.mark.parametrize('xml', [''' @@ -182,13 +222,13 @@ def test_humanitarian_attrib_false(version, hum_attrib_val, xml): ''']) -def test_humanitarian_sector_true(major_version, sector, xml): +def test_humanitarian_sector_true_transaction(version, sector, xml): """ - Detects an activity to be humanitarian using sector codes considered to relate to humanitarian. + Detects an activity to be humanitarian using sector codes at transaction level considered to relate to humanitarian. Also checks that the appropriate vocabulary is provided or assumed. """ - activity_stats = MockActivityStats(major_version) + activity_stats = MockActivityStats(version) activity_stats.element = etree.fromstring(xml.format(sector, activity_stats._dac_5_code())) assert activity_stats.humanitarian()['is_humanitarian'] == 1 @@ -197,23 +237,41 @@ def test_humanitarian_sector_true(major_version, sector, xml): assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 -@pytest.mark.parametrize('major_version', ['1', '2']) -@pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_3_DIGITS) +@pytest.mark.parametrize('version', ['1.01', '1.02', '1.03', '1.04', '1.05', 'unknown version']) +@pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_5_DIGITS) @pytest.mark.parametrize('xml', [''' + - + + + + + ''', ''' + + + + + + + ''', ''' + + + + + + ''']) -def test_humanitarian_sector_true_3_digit(major_version, sector, xml): +def test_humanitarian_sector_true_transaction_invalid_version(version, sector, xml): """ - Detects an activity to be humanitarian using 3-digit sector codes considered to relate to humanitarian. + Detects an activity to be humanitarian using sector codes at transaction level considered to relate to humanitarian. Also checks that the appropriate vocabulary is provided or assumed. """ - activity_stats = MockActivityStats(major_version) + activity_stats = MockActivityStats(version) - activity_stats.element = etree.fromstring(xml.format(sector, activity_stats._dac_3_code())) - assert activity_stats.humanitarian()['is_humanitarian'] == 1 + activity_stats.element = etree.fromstring(xml.format(sector, activity_stats._dac_5_code())) + assert activity_stats.humanitarian()['is_humanitarian'] == 0 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 From b0d774d76c110f4a11d6ca706757ea8b5a814dd4 Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 15:21:59 +0100 Subject: [PATCH 37/45] Detect use of transaction/sector at V1 as bad in hum tests --- stats/dashboard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stats/dashboard.py b/stats/dashboard.py index f47c8cc8a9a..219425290a7 100644 --- a/stats/dashboard.py +++ b/stats/dashboard.py @@ -1193,7 +1193,7 @@ def humanitarian(self): is_humanitarian_by_sector_5_digit_transaction = 1 if set(self.element.xpath('transaction/sector[@vocabulary="{0}" or not(@vocabulary)]/@code'.format(self._dac_5_code()))).intersection(humanitarian_sectors_dac_5_digit) else 0 is_humanitarian_by_sector_3_digit_activity = 1 if set(self.element.xpath('sector[@vocabulary="{0}"]/@code'.format(self._dac_3_code()))).intersection(humanitarian_sectors_dac_3_digit) else 0 is_humanitarian_by_sector_3_digit_transaction = 1 if set(self.element.xpath('transaction/sector[@vocabulary="{0}"]/@code'.format(self._dac_3_code()))).intersection(humanitarian_sectors_dac_3_digit) else 0 - is_humanitarian_by_sector = is_humanitarian_by_sector_5_digit_activity or is_humanitarian_by_sector_5_digit_transaction or is_humanitarian_by_sector_3_digit_activity or is_humanitarian_by_sector_3_digit_transaction + is_humanitarian_by_sector = ((is_humanitarian_by_sector_5_digit_activity or is_humanitarian_by_sector_3_digit_activity) and not (is_humanitarian_by_sector_5_digit_transaction or is_humanitarian_by_sector_3_digit_transaction)) or ((is_humanitarian_by_sector_5_digit_transaction or is_humanitarian_by_sector_3_digit_transaction) and (self._major_version() in ['2'])) return { 'is_humanitarian': 1 if (is_humanitarian_by_attrib or is_humanitarian_by_sector) else 0, From 111a0fc9c48cf103b979a9d2885e995cdbe77dc9 Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 15:36:02 +0100 Subject: [PATCH 38/45] Add tests with vaired sectors at different levels Where they are humanitarian at activity but not transaction or the other way round --- stats/tests/test_humanitarian.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index 5bd91db0790..0353c6bd52a 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -198,7 +198,8 @@ def test_humanitarian_sector_true_3_digit(major_version, sector, xml): @pytest.mark.parametrize('version', ['2.01', '2.02']) -@pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_5_DIGITS) +@pytest.mark.parametrize('hum_sector', HUMANITARIAN_SECTOR_CODES_5_DIGITS) +@pytest.mark.parametrize('not_hum_sector', [-89, 'not_a_code', HUMANITARIAN_SECTOR_CODES_5_DIGITS[0]+1, HUMANITARIAN_SECTOR_CODES_3_DIGITS[0]+1, HUMANITARIAN_SECTOR_CODES_5_DIGITS[-1]-1, HUMANITARIAN_SECTOR_CODES_3_DIGITS[-1]-1]) @pytest.mark.parametrize('xml', [''' @@ -221,8 +222,24 @@ def test_humanitarian_sector_true_3_digit(major_version, sector, xml): + ''', ''' + + + + + + + + ''', ''' + + + + + + + ''']) -def test_humanitarian_sector_true_transaction(version, sector, xml): +def test_humanitarian_sector_true_transaction(version, hum_sector, not_hum_sector, xml): """ Detects an activity to be humanitarian using sector codes at transaction level considered to relate to humanitarian. @@ -230,7 +247,7 @@ def test_humanitarian_sector_true_transaction(version, sector, xml): """ activity_stats = MockActivityStats(version) - activity_stats.element = etree.fromstring(xml.format(sector, activity_stats._dac_5_code())) + activity_stats.element = etree.fromstring(xml.format(hum_sector, activity_stats._dac_5_code(), not_hum_sector)) assert activity_stats.humanitarian()['is_humanitarian'] == 1 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 From 8da16074eb1751d98850d846e4f22ade8ca59980 Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 15:40:26 +0100 Subject: [PATCH 39/45] More correctly detect hum sectors across versions --- stats/dashboard.py | 2 +- stats/tests/test_humanitarian.py | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/stats/dashboard.py b/stats/dashboard.py index 219425290a7..f699cf6b85a 100644 --- a/stats/dashboard.py +++ b/stats/dashboard.py @@ -1193,7 +1193,7 @@ def humanitarian(self): is_humanitarian_by_sector_5_digit_transaction = 1 if set(self.element.xpath('transaction/sector[@vocabulary="{0}" or not(@vocabulary)]/@code'.format(self._dac_5_code()))).intersection(humanitarian_sectors_dac_5_digit) else 0 is_humanitarian_by_sector_3_digit_activity = 1 if set(self.element.xpath('sector[@vocabulary="{0}"]/@code'.format(self._dac_3_code()))).intersection(humanitarian_sectors_dac_3_digit) else 0 is_humanitarian_by_sector_3_digit_transaction = 1 if set(self.element.xpath('transaction/sector[@vocabulary="{0}"]/@code'.format(self._dac_3_code()))).intersection(humanitarian_sectors_dac_3_digit) else 0 - is_humanitarian_by_sector = ((is_humanitarian_by_sector_5_digit_activity or is_humanitarian_by_sector_3_digit_activity) and not (is_humanitarian_by_sector_5_digit_transaction or is_humanitarian_by_sector_3_digit_transaction)) or ((is_humanitarian_by_sector_5_digit_transaction or is_humanitarian_by_sector_3_digit_transaction) and (self._major_version() in ['2'])) + is_humanitarian_by_sector = is_humanitarian_by_sector_5_digit_activity or is_humanitarian_by_sector_3_digit_activity or ((is_humanitarian_by_sector_5_digit_transaction or is_humanitarian_by_sector_3_digit_transaction) and (self._major_version() in ['2'])) return { 'is_humanitarian': 1 if (is_humanitarian_by_attrib or is_humanitarian_by_sector) else 0, diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index 0353c6bd52a..b5ec021568d 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -159,6 +159,14 @@ def test_humanitarian_attrib_false(version, hum_attrib_val, xml): + ''', ''' + + + + + + + ''']) def test_humanitarian_sector_true(major_version, sector, xml): """ @@ -235,7 +243,7 @@ def test_humanitarian_sector_true_3_digit(major_version, sector, xml): - + ''']) @@ -270,14 +278,6 @@ def test_humanitarian_sector_true_transaction(version, hum_sector, not_hum_secto - ''', ''' - - - - - - - ''']) def test_humanitarian_sector_true_transaction_invalid_version(version, sector, xml): """ From ebfb59595597f20e75940fd17a1eee81ba2e577a Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 16:16:31 +0100 Subject: [PATCH 40/45] Add tests where sectors and attributes contradict --- stats/tests/test_humanitarian.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index b5ec021568d..3616f7d9636 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -122,6 +122,7 @@ def test_humanitarian_attrib_true_invalid_version(version, hum_attrib_val_true, @pytest.mark.parametrize('version', ['2.02']) @pytest.mark.parametrize('hum_attrib_val', ['0', 'false', 'True', 'False', '']) +@pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_5_DIGITS) @pytest.mark.parametrize('xml', [''' @@ -133,14 +134,32 @@ def test_humanitarian_attrib_true_invalid_version(version, hum_attrib_val_true, + ''', ''' + + + + ''', ''' + + + + + + ''', ''' + + + + + ''']) -def test_humanitarian_attrib_false(version, hum_attrib_val, xml): +def test_humanitarian_attrib_false(version, hum_attrib_val, sector, xml): """ Detect an activity to not be humanitarian using @humanitarian values that evaluate to false. + + If there is a sector generally deemed to be humanitarian, the attribute shall take precedence. """ activity_stats = MockActivityStats(version) - activity_stats.element = etree.fromstring(xml.format(hum_attrib_val)) + activity_stats.element = etree.fromstring(xml.format(hum_attrib_val, sector)) assert activity_stats.humanitarian()['is_humanitarian'] == 0 assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 @@ -167,6 +186,11 @@ def test_humanitarian_attrib_false(version, hum_attrib_val, xml): + ''', ''' + + + + ''']) def test_humanitarian_sector_true(major_version, sector, xml): """ From 3489c95dfaf7ece9b341e065b8280ac15be34861 Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 16:53:22 +0100 Subject: [PATCH 41/45] Improve hum tests where attrib overrides sector --- stats/dashboard.py | 25 +++++-- stats/tests/test_humanitarian.py | 108 ++++++++++++++++++++++--------- 2 files changed, 97 insertions(+), 36 deletions(-) diff --git a/stats/dashboard.py b/stats/dashboard.py index f699cf6b85a..5d15717135d 100644 --- a/stats/dashboard.py +++ b/stats/dashboard.py @@ -1185,18 +1185,33 @@ def humanitarian(self): humanitarian_sectors_dac_5_digit = ['72010', '72040', '72050', '73010', '74010'] humanitarian_sectors_dac_3_digit = ['720', '730', '740'] + # logic around use of the @humanitarian attribute is_humanitarian_by_attrib_activity = 1 if ('humanitarian' in self.element.attrib) and (self.element.attrib['humanitarian'] in ['1', 'true']) else 0 + is_not_humanitarian_by_attrib_activity = 1 if ('humanitarian' in self.element.attrib) and (self.element.attrib['humanitarian'] in ['0', 'false']) else 0 is_humanitarian_by_attrib_transaction = 1 if set(self.element.xpath('transaction/@humanitarian')).intersection(['1', 'true']) else 0 - is_humanitarian_by_attrib = (self._version() in ['2.02']) and (is_humanitarian_by_attrib_activity or is_humanitarian_by_attrib_transaction) + is_not_humanitarian_by_attrib_transaction = 1 if not is_humanitarian_by_attrib_transaction and set(self.element.xpath('transaction/@humanitarian')).intersection(['0', 'false']) else 0 + is_humanitarian_by_attrib = (self._version() in ['2.02']) and (is_humanitarian_by_attrib_activity or (is_humanitarian_by_attrib_transaction and not is_not_humanitarian_by_attrib_activity)) + # logic around DAC sector codes deemed to be humanitarian is_humanitarian_by_sector_5_digit_activity = 1 if set(self.element.xpath('sector[@vocabulary="{0}" or not(@vocabulary)]/@code'.format(self._dac_5_code()))).intersection(humanitarian_sectors_dac_5_digit) else 0 - is_humanitarian_by_sector_5_digit_transaction = 1 if set(self.element.xpath('transaction/sector[@vocabulary="{0}" or not(@vocabulary)]/@code'.format(self._dac_5_code()))).intersection(humanitarian_sectors_dac_5_digit) else 0 + is_humanitarian_by_sector_5_digit_transaction = 1 if set(self.element.xpath('transaction[not(@humanitarian="0" or @humanitarian="false")]/sector[@vocabulary="{0}" or not(@vocabulary)]/@code'.format(self._dac_5_code()))).intersection(humanitarian_sectors_dac_5_digit) else 0 is_humanitarian_by_sector_3_digit_activity = 1 if set(self.element.xpath('sector[@vocabulary="{0}"]/@code'.format(self._dac_3_code()))).intersection(humanitarian_sectors_dac_3_digit) else 0 - is_humanitarian_by_sector_3_digit_transaction = 1 if set(self.element.xpath('transaction/sector[@vocabulary="{0}"]/@code'.format(self._dac_3_code()))).intersection(humanitarian_sectors_dac_3_digit) else 0 - is_humanitarian_by_sector = is_humanitarian_by_sector_5_digit_activity or is_humanitarian_by_sector_3_digit_activity or ((is_humanitarian_by_sector_5_digit_transaction or is_humanitarian_by_sector_3_digit_transaction) and (self._major_version() in ['2'])) + is_humanitarian_by_sector_3_digit_transaction = 1 if set(self.element.xpath('transaction[not(@humanitarian="0" or @humanitarian="false")]/sector[@vocabulary="{0}"]/@code'.format(self._dac_3_code()))).intersection(humanitarian_sectors_dac_3_digit) else 0 + # helper variables to help make logic easier to read + is_humanitarian_by_sector_activity = is_humanitarian_by_sector_5_digit_activity or is_humanitarian_by_sector_3_digit_activity + is_humanitarian_by_sector_transaction = is_humanitarian_by_sector_5_digit_transaction or is_humanitarian_by_sector_3_digit_transaction + is_humanitarian_by_sector = is_humanitarian_by_sector_activity or (is_humanitarian_by_sector_transaction and (self._major_version() in ['2'])) + + # combine the various ways in which an activity may be humanitarian + is_humanitarian = 1 if (is_humanitarian_by_attrib or is_humanitarian_by_sector) else 0 + # deal with some edge cases that have veto + if is_not_humanitarian_by_attrib_activity: + is_humanitarian = 0 + + # is_humanitarian_by_sector = False if not is_humanitarian_by_sector_activity else is_humanitarian_by_sector return { - 'is_humanitarian': 1 if (is_humanitarian_by_attrib or is_humanitarian_by_sector) else 0, + 'is_humanitarian': is_humanitarian, 'is_humanitarian_by_attrib': is_humanitarian_by_attrib, 'contains_humanitarian_scope': 1 if (self._version() in ['2.02']) and self.element.xpath('humanitarian-scope/@type') and self.element.xpath('humanitarian-scope/@code') else 0, 'uses_humanitarian_clusters_vocab': 1 if (self._major_version() in ['2']) and self.element.xpath('sector/@vocabulary="10"') else 0 diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index 3616f7d9636..b8e20ea7b63 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -43,11 +43,6 @@ def _version(self): - ''', ''' - - - - ''', ''' @@ -121,7 +116,7 @@ def test_humanitarian_attrib_true_invalid_version(version, hum_attrib_val_true, @pytest.mark.parametrize('version', ['2.02']) -@pytest.mark.parametrize('hum_attrib_val', ['0', 'false', 'True', 'False', '']) +@pytest.mark.parametrize('hum_attrib_val', ['0', 'false']) @pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_5_DIGITS) @pytest.mark.parametrize('xml', [''' @@ -150,6 +145,11 @@ def test_humanitarian_attrib_true_invalid_version(version, hum_attrib_val_true, + ''', ''' + + + + ''']) def test_humanitarian_attrib_false(version, hum_attrib_val, sector, xml): """ @@ -166,6 +166,71 @@ def test_humanitarian_attrib_false(version, hum_attrib_val, sector, xml): assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 +@pytest.mark.parametrize('version', ['2.02']) +@pytest.mark.parametrize('hum_attrib_val', ['True', 'False', '']) +@pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_5_DIGITS) +@pytest.mark.parametrize('xml', [''' + + + + ''', ''' + + + + + + ''', ''' + + + + + + ''']) +def test_humanitarian_attrib_invalid_sector(version, hum_attrib_val, sector, xml): + """ + Detect an activity to not be humanitarian using @humanitarian values that evaluate to false. + + If there is a sector generally deemed to be humanitarian, the attribute shall take precedence. + """ + activity_stats = MockActivityStats(version) + + activity_stats.element = etree.fromstring(xml.format(hum_attrib_val, sector)) + assert activity_stats.humanitarian()['is_humanitarian'] == 1 + assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 + assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 + assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 + + +@pytest.mark.parametrize('version', ['2.02']) +@pytest.mark.parametrize('hum_attrib_val', ['True', 'False', '']) +@pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_5_DIGITS) +@pytest.mark.parametrize('xml', [''' + + + ''', ''' + + + + ''', ''' + + + + ''']) +def test_humanitarian_attrib_invalid_no_sector(version, hum_attrib_val, sector, xml): + """ + Detect an activity to not be humanitarian using @humanitarian values that evaluate to false. + + If there is a sector generally deemed to be humanitarian, the attribute shall take precedence. + """ + activity_stats = MockActivityStats(version) + + activity_stats.element = etree.fromstring(xml.format(hum_attrib_val, sector)) + assert activity_stats.humanitarian()['is_humanitarian'] == 0 + assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 + assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 + assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 + + @pytest.mark.parametrize('major_version', ['1', '2']) @pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_5_DIGITS) @pytest.mark.parametrize('xml', [''' @@ -186,11 +251,6 @@ def test_humanitarian_attrib_false(version, hum_attrib_val, sector, xml): - ''', ''' - - - - ''']) def test_humanitarian_sector_true(major_version, sector, xml): """ @@ -270,6 +330,12 @@ def test_humanitarian_sector_true_3_digit(major_version, sector, xml): + ''', ''' + + + + + ''']) def test_humanitarian_sector_true_transaction(version, hum_sector, not_hum_sector, xml): """ @@ -449,26 +515,6 @@ def test_humanitarian_attrib_true_sector_anything(major_version, sector, hum_att assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 -@pytest.mark.parametrize('version', ['2.02']) -@pytest.mark.parametrize('sector', HUMANITARIAN_SECTOR_CODES_5_DIGITS) -@pytest.mark.parametrize('hum_attrib_val', ['0', 'false', 'True', 'False', '']) -def test_humanitarian_attrib_false_sector_true(version, sector, hum_attrib_val): - """ - Detect an activity to be humanitarian using sector codes that are deemed to be humanitarian, in combination with a @humanitarian value which evaluates to false. - """ - activity_stats = MockActivityStats(version) - - activity_stats.element = etree.fromstring(''' - - - - '''.format(hum_attrib_val, sector)) - assert activity_stats.humanitarian()['is_humanitarian'] == 1 - assert activity_stats.humanitarian()['is_humanitarian_by_attrib'] == 0 - assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 - assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 - - @pytest.mark.parametrize('version', ['2.02']) @pytest.mark.parametrize('sector', [-89, 'not_a_code']) @pytest.mark.parametrize('hum_attrib_val', ['0', 'false', 'True', 'False', '']) From dab3b114fa1902e24dbedbb7f3a49fdbfc22b825 Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 16:54:19 +0100 Subject: [PATCH 42/45] Remove old line of code --- stats/dashboard.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/stats/dashboard.py b/stats/dashboard.py index 5d15717135d..7cccf4862bc 100644 --- a/stats/dashboard.py +++ b/stats/dashboard.py @@ -1208,8 +1208,6 @@ def humanitarian(self): if is_not_humanitarian_by_attrib_activity: is_humanitarian = 0 - # is_humanitarian_by_sector = False if not is_humanitarian_by_sector_activity else is_humanitarian_by_sector - return { 'is_humanitarian': is_humanitarian, 'is_humanitarian_by_attrib': is_humanitarian_by_attrib, From ea57289f69e2f4bb75252f18271b71aebd671041 Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 17:03:14 +0100 Subject: [PATCH 43/45] Do not detect hum global clusters vocab at 2.01 --- stats/tests/test_humanitarian.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index b8e20ea7b63..3e213712d0b 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -639,12 +639,12 @@ def test_humanitarian_scope_invalid(version): assert activity_stats.humanitarian()['contains_humanitarian_scope'] == 0 -@pytest.mark.parametrize('major_version', ['2']) -def test_humanitarian_clusters_valid(major_version): +@pytest.mark.parametrize('version', ['2.02']) +def test_humanitarian_clusters_valid(version): """ Detect that an activity contains a sector defined by the 'Humanitarian Global Clusters' sector vocabulary. """ - activity_stats = MockActivityStats(major_version) + activity_stats = MockActivityStats(version) activity_stats.element = etree.fromstring(''' @@ -654,12 +654,12 @@ def test_humanitarian_clusters_valid(major_version): assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 1 -@pytest.mark.parametrize('major_version', ['1']) -def test_humanitarian_clusters_version_1(major_version): +@pytest.mark.parametrize('version', ['1.01', '1.02', '1.03', '1.04', '1.05', '2.01', 'unknown version']) +def test_humanitarian_clusters_version_1(version): """ - Detect that a version 1 activity containing a sector defined by the 'Humanitarian Global Clusters' sector vocabulary is not detected. + Detect that a pre-2.02 activity containing a sector defined by the 'Humanitarian Global Clusters' sector vocabulary is not detected. """ - activity_stats = MockActivityStats(major_version) + activity_stats = MockActivityStats(version) activity_stats.element = etree.fromstring(''' @@ -669,13 +669,13 @@ def test_humanitarian_clusters_version_1(major_version): assert activity_stats.humanitarian()['uses_humanitarian_clusters_vocab'] == 0 -@pytest.mark.parametrize('major_version', ['2']) +@pytest.mark.parametrize('version', ['1.01', '1.02', '1.03', '1.04', '1.05', '2.01', '2.02', 'unknown version']) @pytest.mark.parametrize('sector_vocabulary_code', ['', '1', 'internal vocabulary']) -def test_humanitarian_clusters_invalid(major_version, sector_vocabulary_code): +def test_humanitarian_clusters_invalid(version, sector_vocabulary_code): """ Detect that an activity does not contain a sector defined by the 'Humanitarian Global Clusters' sector vocabulary. """ - activity_stats = MockActivityStats(major_version) + activity_stats = MockActivityStats(version) activity_stats.element = etree.fromstring(''' From 8159dcd08c1e2c5d3e8b1bd080ca43e2ae15f655 Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 17:03:51 +0100 Subject: [PATCH 44/45] Restrict where sector vocab 10 is accepted --- stats/dashboard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stats/dashboard.py b/stats/dashboard.py index 7cccf4862bc..03f1ba81469 100644 --- a/stats/dashboard.py +++ b/stats/dashboard.py @@ -1212,7 +1212,7 @@ def humanitarian(self): 'is_humanitarian': is_humanitarian, 'is_humanitarian_by_attrib': is_humanitarian_by_attrib, 'contains_humanitarian_scope': 1 if (self._version() in ['2.02']) and self.element.xpath('humanitarian-scope/@type') and self.element.xpath('humanitarian-scope/@code') else 0, - 'uses_humanitarian_clusters_vocab': 1 if (self._major_version() in ['2']) and self.element.xpath('sector/@vocabulary="10"') else 0 + 'uses_humanitarian_clusters_vocab': 1 if (self._version() in ['2.02']) and self.element.xpath('sector/@vocabulary="10"') else 0 } def _transaction_type_code(self, transaction): From 9c3b865f6184418f854667d3bafc0be4ae835890 Mon Sep 17 00:00:00 2001 From: Hayden Date: Thu, 1 Jun 2017 17:11:12 +0100 Subject: [PATCH 45/45] Update an incorrect docstring --- stats/tests/test_humanitarian.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/stats/tests/test_humanitarian.py b/stats/tests/test_humanitarian.py index 3e213712d0b..4f5d5da4617 100644 --- a/stats/tests/test_humanitarian.py +++ b/stats/tests/test_humanitarian.py @@ -371,9 +371,7 @@ def test_humanitarian_sector_true_transaction(version, hum_sector, not_hum_secto ''']) def test_humanitarian_sector_true_transaction_invalid_version(version, sector, xml): """ - Detects an activity to be humanitarian using sector codes at transaction level considered to relate to humanitarian. - - Also checks that the appropriate vocabulary is provided or assumed. + Detects an activity to be not humanitarian due to an invalid version despite correctly formed transactions. """ activity_stats = MockActivityStats(version)