Merge a8cf9a5 into ecc2398

stats/dashboard.py

@@ -86,18 +86,40 @@ def convert_to_float(x):
 
 
 # Import codelists
-## In order to test whether or not correct codelist values are being used in the data
-## we need to pull in data about how codelists map to elements
+# In order to test whether or not correct codelist values are being
+# used in the data we need to pull in data about how codelists map
+# to elements
 def get_codelist_mapping(major_version):
-    codelist_mapping_xml = etree.parse('helpers/mapping-{}.xml'.format(major_version))
-    codelist_mappings = [ x.text for x in codelist_mapping_xml.xpath('mapping/path') ]
-    codelist_mappings = [ re.sub('^\/\/iati-activity', './',path) for path in codelist_mappings]
-    codelist_mappings = [ re.sub('^\/\/', './/', path) for path in codelist_mappings ]
-    return codelist_mappings
+    codelist_mapping_xml = etree.parse('helpers/mapping-{}.xml'.format(
+        major_version))
+    codelist_mappings = codelist_mapping_xml.xpath('mapping')
+    codelist_conditions = [x.find('condition').text
+                           if x.find('condition') is not None else None
+                           for x in codelist_mappings]
+    codelist_paths = [x.find('path').text for x in codelist_mappings]
+    codelist_paths = [re.sub(r'^\/\/iati-activity', './', path)
+                      for path in codelist_paths]
+    codelist_paths = [re.sub(r'^\/\/', './/', path)
+                      for path in codelist_paths]
+    codelist_condition_paths = []
+    for path, mapping in zip(codelist_paths, codelist_mappings):
+        condition = mapping.find('condition')
+        if condition is not None:
+            condition = condition.text
+            pref, attr = path.rsplit('/', 1)
+            condition_path = '{0}[{1}]/{2}'.format(pref, condition, attr)
+        else:
+            condition_path = None
+        codelist_condition_paths.append((path, condition_path))
+
+    return codelist_condition_paths
+
+
+codelist_mappings = {major_version: get_codelist_mapping(major_version)
+                     for major_version in ['1', '2']}
 
-codelist_mappings = { major_version: get_codelist_mapping(major_version) for major_version in ['1', '2'] }
 
-CODELISTS = {'1':{}, '2':{}}
+CODELISTS = {'1': {}, '2': {}}
 for major_version in ['1', '2']:
     for codelist_name in ['Version', 'ActivityStatus', 'Currency', 'Sector', 'SectorCategory', 'DocumentCategory', 'AidType', 'BudgetNotProvided']:
         CODELISTS[major_version][codelist_name] = set(c['code'] for c in json.load(open('helpers/codelists/{}/{}.json'.format(major_version, codelist_name)))['data'])
@@ -520,21 +542,25 @@ def elements(self):
     def elements_total(self):
         return element_to_count_dict(self.element, 'iati-activity', defaultdict(int), True)
 
-    @returns_numberdictdict
-    def codelist_values(self):
+    def _codelist_values(self):
         out = defaultdict(lambda: defaultdict(int))
-        for path in codelist_mappings[self._major_version()]:
-            for value in self.element.xpath(path):
+        for path, condition_path in codelist_mappings[self._major_version()]:
+            if condition_path is not None:
+                values = self.element.xpath(condition_path)
+            else:
+                values = self.element.xpath(path)
+            for value in values:
                 out[path][value] += 1
         return out
 
+    @returns_numberdictdict
+    def codelist_values(self):
+        return self._codelist_values()
+
     @returns_numberdictdict
     def codelist_values_by_major_version(self):
-        out = defaultdict(lambda: defaultdict(int))
-        for path in codelist_mappings[self._major_version()]:
-            for value in self.element.xpath(path):
-                out[path][value] += 1
-        return { self._major_version(): out  }
+        out = self._codelist_values()
+        return {self._major_version(): out}
 
     @returns_numberdictdict
     def boolean_values(self):