diff --git a/stats/dashboard.py b/stats/dashboard.py index aa5ee914c1b..b5e7b029068 100644 --- a/stats/dashboard.py +++ b/stats/dashboard.py @@ -348,14 +348,6 @@ def reporting_orgs(self): def participating_orgs(self): return dict([ (x.attrib.get('ref'), 1) for x in self.element.findall('participating-org')]) - @returns_numberdictdict - def participating_orgs_text(self): - return dict([ (x.attrib.get('ref'), {x.text:1}) for x in self.element.findall('participating-org')]) - - @returns_numberdictdict - def participating_orgs_by_role(self): - return dict([ (x.attrib.get('role'), {x.attrib.get('ref'):1}) for x in self.element.findall('participating-org')]) - @returns_numberdict def element_versions(self): return { self.element.attrib.get('version'): 1 } @@ -397,7 +389,6 @@ class ActivityStats(CommonSharedElements): blank = False strict = False # (Setting this to true will ignore values that don't follow the schema) context = '' - comprehensiveness_current_activity_status = None now = datetime.datetime.now() # TODO Add option to set this to date of git commit @returns_numberdict @@ -418,19 +409,6 @@ def _budget_not_provided(self): else: return None - def by_hierarchy(self): - out = {} - for stat in ['activities', 'elements', 'elements_total', - 'forwardlooking_currency_year', 'forwardlooking_activities_current', 'forwardlooking_activities_with_budgets', 'forwardlooking_activities_with_budget_not_provided', - 'comprehensiveness', 'comprehensiveness_with_validation', 'comprehensiveness_denominators', 'comprehensiveness_denominator_default' - ]: - out[stat] = copy.deepcopy(getattr(self, stat)()) - if self.blank: - return defaultdict(lambda: out) - else: - hierarchy = self.element.attrib.get('hierarchy') - return { ('1' if hierarchy is None else hierarchy): out } - @returns_numberdict def currencies(self): currencies = [ x.find('value').get('currency') for x in self.element.findall('transaction') if x.find('value') is not None ] @@ -594,93 +572,10 @@ def receiver_org(self): out[receiver_org.attrib.get('ref')] += 1 return out - @returns_numberdict - def transactions_incoming_funds(self): - """ - Counts the number of activities which contain at least one transaction with incoming funds. - Also counts the number of transactions where the type is incoming funds - """ - # Set default output - out = defaultdict(int) - - # Loop over each tranaction - for transaction in self.element.findall('transaction'): - # If the transaction-type element has a code of 'IF' (v1) or 1 (v2), increment the output counter - if transaction.xpath('transaction-type/@code="{}"'.format(self._incoming_funds_code())): - out['transactions_with_incoming_funds'] += 1 - - # If there is at least one transaction within this activity with an incoming funds transaction, then increment the number of activities with incoming funds - if out['transactions_with_incoming_funds'] > 0: - out['activities_with_incoming_funds'] += 1 - - return out - - @returns_numberdict - def transaction_timing(self): - today = self.now.date() - def months_ago(n): - self.now.date() - datetime.timedelta(days=n*30) - out = { 30:0, 60:0, 90:0, 180:0, 360:0 } - for transaction in self.element.findall('transaction'): - date = transaction_date(transaction) - if date: - days = (today - date).days - if days < -1: - continue - for k in sorted(out.keys()): - if days < k: - out[k] += 1 - return out - - @returns_numberdict - def transaction_months(self): - out = defaultdict(int) - for transaction in self.element.findall('transaction'): - date = transaction_date(transaction) - if date: - out[date.month] += 1 - return out - - @returns_numberdict - def transaction_months_with_year(self): - out = defaultdict(int) - for transaction in self.element.findall('transaction'): - date = transaction_date(transaction) - if date: - out['{}-{}'.format(date.year, str(date.month).zfill(2))] += 1 - return out - - @returns_numberdict - def budget_lengths(self): - out = defaultdict(int) - for budget in self.element.findall('budget'): - period_start = iso_date(budget.find('period-start')) - period_end = iso_date(budget.find('period-end')) - if period_start and period_end: - out[(period_end - period_start).days] += 1 - return out - def _transaction_year(self, transaction): date = transaction_date(transaction) return date.year if date else None - def _spend_currency_year(self, transactions): - out = defaultdict(lambda: defaultdict(Decimal)) - for transaction in transactions: - value = transaction.find('value') - if (transaction.find('transaction-type') is not None and - transaction.find('transaction-type').attrib.get('code') in [self._disbursement_code(), self._expenditure_code()]): - - # Set transaction_value if a value exists for this transaction. Else set to 0 - transaction_value = 0 if value is None else Decimal(value.text) - - out[self._transaction_year(transaction)][get_currency(self, transaction)] += transaction_value - return out - - @returns_numberdictdict - def spend_currency_year(self): - return self._spend_currency_year(self.element.findall('transaction')) - def _is_secondary_reported(self): """Tests if this activity has been secondary reported. Test based on if the secondary-reporter flag is set. @@ -699,23 +594,6 @@ def activities_secondary_reported(self): else: return {} - - @returns_numberdictdict - def forwardlooking_currency_year(self): - # Note this is not currently displayed on the dashboard - # As the forwardlooking page now only displays counts, - # not the sums that this function calculates. - out = defaultdict(lambda: defaultdict(Decimal)) - budgets = self.element.findall('budget') - for budget in budgets: - value = budget.find('value') - - # Set budget_value if a value exists for this budget. Else set to 0 - budget_value = 0 if value is None else Decimal(value.text) - - out[budget_year(budget)][get_currency(self, budget)] += budget_value - return out - def _get_end_date(self): """Gets the end date for the activity. An 'actual end date' is preferred over a 'planned end date' @@ -732,21 +610,6 @@ def _get_end_date(self): else: return None - def _forwardlooking_is_current(self, year): - """Tests if an activity contains i) at least one (actual or planned) end year which is greater - or equal to the year passed to this function, or ii) no (actual or planned) end years at all. - Returns: True or False - """ - # Get list of years for each of the planned-end and actual-end dates - activity_end_years = [ - iso_date(x).year - for x in self.element.xpath('activity-date[@type="{}" or @type="{}"]'.format(self._planned_end_code(), self._actual_end_code())) - if iso_date(x) - ] - # Return boolean. True if activity_end_years is empty, or at least one of the actual/planned - # end years is greater or equal to the year passed to this function - return (not activity_end_years) or any(activity_end_year>=year for activity_end_year in activity_end_years) - def _get_ratio_commitments_disbursements(self, year): """ Calculates the ratio of commitments vs total amount disbursed or expended in or before the input year. Values are converted to USD to improve comparability. @@ -786,39 +649,6 @@ def _get_ratio_commitments_disbursements(self, year): else: return None - def _forwardlooking_exclude_in_calculations(self, year=datetime.date.today().year, date_code_runs=None): - """ Tests if an activity should be excluded from the forward looking calculations. - Activities are excluded if: - i) They end within six months from date_code_runs OR - ii) At least 90% of the commitment transactions has been disbursed or expended - within or before the input year - - This arises from: - https://github.com/IATI/IATI-Dashboard/issues/388 - https://github.com/IATI/IATI-Dashboard/issues/389 - - Input: - year -- The point in time to test the above criteria against - date_code_runs -- a date object for when this code is run - Returns: 0 if not excluded - >0 if excluded - """ - - # Set date_code_runs. Defaults to self.now (as a date object) - date_code_runs = date_code_runs if date_code_runs else self.now.date() - - # If this activity has an end date, check that it will not end within the next six - # months from date_code_runs - if self._get_end_date(): - if (date_code_runs + relativedelta(months=+6)) > self._get_end_date(): - return 1 - - if self._get_ratio_commitments_disbursements(year) >= 0.9 and self._get_ratio_commitments_disbursements(year) is not None: - return 2 - else: - return 0 - - def _is_donor_publisher(self): """Returns True if this activity is deemed to be reported by a donor publisher. Methodology descibed in https://github.com/IATI/IATI-Dashboard/issues/377 @@ -831,127 +661,6 @@ def _is_donor_publisher(self): return ((self.element.xpath('reporting-org/@ref')[0] in self.element.xpath("participating-org[@role='{}']/@ref|participating-org[@role='{}']/@ref".format(self._funding_code(), self._OrganisationRole_Extending_code()))) and (self.element.xpath('reporting-org/@ref')[0] not in self.element.xpath("participating-org[@role='{}']/@ref".format(self._OrganisationRole_Implementing_code())))) - @returns_dict - def forwardlooking_excluded_activities(self): - """Outputs whether this activity is excluded for the purposes of forwardlooking calculations - Returns iati-identifier and...: 0 if not excluded - 1 if excluded - """ - # Set the current year. Defaults to self.now (as a date object) - this_year = datetime.date.today().year - - # Retreive a dictionary with the activity identifier and the result for this and the next two years - return { self.element.find('iati-identifier').text: {year: int(self._forwardlooking_exclude_in_calculations(year)) - for year in range(this_year, this_year+3)} } - - - @returns_numberdict - def forwardlooking_activities_current(self, date_code_runs=None): - """ - The number of current and non-excluded activities for this year and the following 2 years. - - Current activities: http://support.iatistandard.org/entries/52291985-Forward-looking-Activity-level-budgets-numerator - - Note activities excluded according if they meet the logic in _forwardlooking_exclude_in_calculations() - - Note: this is a different definition of 'current' to the older annual - report stats in this file, so does not re-use those functions. - - Input: - date_code_runs -- a date object for when this code is run - Returns: - dictionary containing years with binary value if this activity is current - - """ - - # Set date_code_runs. Defaults to self.now (as a date object) - date_code_runs = date_code_runs if date_code_runs else self.now.date() - - this_year = date_code_runs.year - return { year: int(self._forwardlooking_is_current(year) and not bool(self._forwardlooking_exclude_in_calculations(year=year, date_code_runs=date_code_runs))) - for year in range(this_year, this_year+3) } - - @returns_numberdict - def forwardlooking_activities_with_budgets(self, date_code_runs=None): - """ - The number of current activities with budgets for this year and the following 2 years. - - http://support.iatistandard.org/entries/52292065-Forward-looking-Activity-level-budgets-denominator - - Note activities excluded according if they meet the logic in _forwardlooking_exclude_in_calculations() - - Input: - date_code_runs -- a date object for when this code is run - Returns: - dictionary containing years with binary value if this activity is current and has a budget for the given year - """ - # Set date_code_runs. Defaults to self.now (as a date object) - date_code_runs = date_code_runs if date_code_runs else self.now.date() - - this_year = int(date_code_runs.year) - budget_years = ([ budget_year(budget) for budget in self.element.findall('budget') ]) - return { year: int(self._forwardlooking_is_current(year) and year in budget_years and not bool(self._forwardlooking_exclude_in_calculations(year=year, date_code_runs=date_code_runs))) - for year in range(this_year, this_year+3) } - - @returns_numberdict - def forwardlooking_activities_with_budget_not_provided(self, date_code_runs=None): - """ - Number of activities with the budget_not_provided attribute for this year and the following 2 years. - - Note activities excluded according if they meet the logic in _forwardlooking_exclude_in_calculations() - - Input: - date_code_runs -- a date object for when this code is run - Returns: - dictionary containing years with binary value if this activity is current and has the budget_not_provided attribute - """ - date_code_runs = date_code_runs if date_code_runs else self.now.date() - this_year = int(date_code_runs.year) - bnp = self._budget_not_provided() is not None - return {year: int(self._forwardlooking_is_current(year) and bnp > 0 and not bool(self._forwardlooking_exclude_in_calculations(year=year, date_code_runs=date_code_runs))) - for year in range(this_year, this_year+3)} - - @memoize - def _comprehensiveness_is_current(self): - """ - Tests if this activity should be considered as part of the comprehensiveness calculations. - Logic is based on the activity status code and end dates. - Returns: True or False - """ - - # Get the activity-code value for this activity - activity_status_code = self.element.xpath('activity-status/@code') - - # Get the end dates for this activity as lists - activity_planned_end_dates = [ iso_date(x) for x in self.element.xpath('activity-date[@type="{}"]'.format(self._planned_end_code())) if iso_date(x) ] - activity_actual_end_dates = [ iso_date(x) for x in self.element.xpath('activity-date[@type="{}"]'.format(self._actual_end_code())) if iso_date(x) ] - - # If there is no planned end date AND activity-status/@code is 2 (implementing) or 4 (post-completion), then this is a current activity - if not(activity_planned_end_dates) and activity_status_code: - if activity_status_code[0] == '2' or activity_status_code[0] == '4': - self.comprehensiveness_current_activity_status = 1 - return True - - # If the actual end date is within the last year, then this is a current activity - for actual_end_date in activity_actual_end_dates: - if (actual_end_date>=add_years(self.today, -1)) and (actual_end_date <= self.today): - self.comprehensiveness_current_activity_status = 2 - return True - - # If the planned end date is greater than today, then this is a current activity - for planned_end_date in activity_planned_end_dates: - if planned_end_date>=self.today: - self.comprehensiveness_current_activity_status = 3 - return True - - # If got this far and not met one of the conditions to qualify as a current activity, return false - self.comprehensiveness_current_activity_status = 0 - return False - - @returns_dict - def comprehensiveness_current_activities(self): - """Outputs whether each activity is considered current for the purposes of comprehensiveness calculations""" - return {self.element.find('iati-identifier').text:self.comprehensiveness_current_activity_status} def _is_recipient_language_used(self): """If there is only 1 recipient-country, test if one of the languages for that country is used @@ -985,245 +694,6 @@ def _is_recipient_language_used(self): else: return 0 - - @memoize - def _comprehensiveness_bools(self): - - def is_text_in_element(elementName): - """ Determine if an element with the specified tagname contains any text. - - Keyword arguments: - elementName - The name of the element to be checked - - If text is present return true, else false. - """ - - # Use xpath to return a list of found text within the specified element name - # The precise xpath needed will vary depending on the version - if self._major_version() == '2': - # In v2, textual elements must be contained within child elements - textFound = self.element.xpath('{}/narrative/text()'.format(elementName)) - - elif self._major_version() == '1': - # In v1, free text is allowed without the need for child elements - textFound = self.element.xpath('{}/text()'.format(elementName)) - - else: - # This is not a valid version - textFound = [] - - # Perform logic. If the list is not empty, return true. Otherwise false - return True if textFound else False - - return { - 'version': (self.element.getparent() is not None and - 'version' in self.element.getparent().attrib), - 'reporting-org': (self.element.xpath('reporting-org/@ref') and - is_text_in_element('reporting-org')), - 'iati-identifier': self.element.xpath('iati-identifier/text()'), - 'participating-org': self.element.find('participating-org') is not None, - 'title': is_text_in_element('title'), - 'description': is_text_in_element('description'), - 'activity-status': self.element.find('activity-status') is not None, - 'activity-date': self.element.find('activity-date') is not None, - 'sector': self.element.find('sector') is not None or (self._major_version() != '1' and all_true_and_not_empty( - (transaction.find('sector') is not None) - for transaction in self.element.findall('transaction') - )), - 'country_or_region': ( - self.element.find('recipient-country') is not None or - self.element.find('recipient-region') is not None or - (self._major_version() != '1' and all_true_and_not_empty( - (transaction.find('recipient-country') is not None or - transaction.find('recipient-region') is not None) - for transaction in self.element.findall('transaction') - ))), - 'transaction_commitment': self.element.xpath('transaction[transaction-type/@code="{}" or transaction-type/@code="11"]'.format(self._commitment_code())), - 'transaction_spend': self.element.xpath('transaction[transaction-type/@code="{}" or transaction-type/@code="{}"]'.format(self._disbursement_code(), self._expenditure_code())), - 'transaction_currency': all_true_and_not_empty(x.xpath('value/@value-date') and x.xpath('../@default-currency|./value/@currency') for x in self.element.findall('transaction')), - 'transaction_traceability': all_true_and_not_empty(x.xpath('provider-org/@provider-activity-id') for x in self.element.xpath('transaction[transaction-type/@code="{}"]'.format(self._incoming_funds_code()))) or - self._is_donor_publisher(), - 'budget': self.element.findall('budget'), - 'budget_not_provided': self._budget_not_provided() is not None, - 'contact-info': self.element.findall('contact-info/email'), - 'location': self.element.xpath('location/point/pos|location/name|location/description|location/location-administrative'), - 'location_point_pos': self.element.xpath('location/point/pos'), - 'sector_dac': self._is_sector_dac(), - 'capital-spend': self.element.xpath('capital-spend/@percentage'), - 'document-link': self.element.findall('document-link'), - 'activity-website': self.element.xpath('activity-website' if self._major_version() == '1' else 'document-link[category/@code="A12"]'), - 'recipient_language': self._is_recipient_language_used(), - 'conditions_attached': self.element.xpath('conditions/@attached'), - 'result_indicator': self.element.xpath('result/indicator'), - 'aid_type': ( - all_true_and_not_empty(self.element.xpath('default-aid-type/@code')) or - all_true_and_not_empty([transaction.xpath('aid-type/@code') for transaction in self.element.xpath('transaction')]) - ) - # Alternative: all(map(all_true_and_not_empty, [transaction.xpath('aid-type/@code') for transaction in self.element.xpath('transaction')])) - } - - def _is_sector_dac(self): - """Determine whether an activity has comprehensive DAC sectors against the validation methodology.""" - sector_dac_activity_level = self.element.xpath('sector[@vocabulary="{}" or @vocabulary="{}" or not(@vocabulary)]'.format(self._dac_5_code(), self._dac_3_code())) - - if self._major_version() != '1': - sector_dac_transaction_level = [transaction.xpath('sector[@vocabulary="{}" or @vocabulary="{}" or not(@vocabulary)]'.format(self._dac_5_code(), self._dac_3_code())) for transaction in self.element.xpath('transaction')] - all_transactions_have_dac_sector_codes = all_true_and_not_empty(sector_dac_transaction_level) - else: - all_transactions_have_dac_sector_codes = False - - return sector_dac_activity_level or all_transactions_have_dac_sector_codes - - def _comprehensiveness_with_validation_bools(self): - - def element_ref(element_obj): - """Get the ref attribute of a given element. - - Returns: - Value in the 'ref' attribute or None if none found - """ - return element_obj.attrib.get('ref') if element_obj is not None else None - - bools = copy.copy(self._comprehensiveness_bools()) - reporting_org_ref = element_ref(self.element.find('reporting-org')) - previous_reporting_org_refs = [element_ref(x) for x in self.element.xpath('other-identifier[@type="B1"]') if element_ref(x) is not None] - - def decimal_or_zero(value): - try: - return Decimal(value) - except TypeError: - return 0 - - def empty_or_percentage_sum_is_100(path, by_vocab=False): - elements = self.element.xpath(path) - if not elements: - return True - else: - elements_by_vocab = defaultdict(list) - if by_vocab: - for element in elements: - elements_by_vocab[element.attrib.get('vocabulary')].append(element) - return all( - len(es) == 1 or - sum(decimal_or_zero(x.attrib.get('percentage')) for x in es) == 100 - for es in elements_by_vocab.values()) - else: - return len(elements) == 1 or sum(decimal_or_zero(x.attrib.get('percentage')) for x in elements) == 100 - bools.update({ - 'version': bools['version'] and self.element.getparent().attrib['version'] in CODELISTS[self._major_version()]['Version'], - 'iati-identifier': ( - bools['iati-identifier'] and - ( - # Give v1.xx data an automatic pass on this sub condition: https://github.com/IATI/IATI-Dashboard/issues/399 - (reporting_org_ref and self.element.find('iati-identifier').text.startswith(reporting_org_ref)) or - any([self.element.find('iati-identifier').text.startswith(x) for x in previous_reporting_org_refs]) - if self._major_version() is not '1' else True - )), - 'participating-org': bools['participating-org'] and self._funding_code() in self.element.xpath('participating-org/@role'), - 'activity-status': bools['activity-status'] and all_true_and_not_empty(x in CODELISTS[self._major_version()]['ActivityStatus'] for x in self.element.xpath('activity-status/@code')), - 'activity-date': ( - bools['activity-date'] and - self.element.xpath('activity-date[@type="{}" or @type="{}"]'.format(self._planned_start_code(), self._actual_start_code())) and - all_true_and_not_empty(map(valid_date, self.element.findall('activity-date'))) - ), - 'sector': ( - bools['sector'] and - empty_or_percentage_sum_is_100('sector', by_vocab=True)), - 'country_or_region': ( - bools['country_or_region'] and - empty_or_percentage_sum_is_100('recipient-country|recipient-region')), - 'transaction_commitment': ( - bools['transaction_commitment'] and - all([ valid_value(x.find('value')) for x in bools['transaction_commitment'] ]) and - all_true_and_not_empty(any(valid_date(x) for x in t.xpath('transaction-date|value')) for t in bools['transaction_commitment']) - ), - 'transaction_spend': ( - bools['transaction_spend'] and - all([ valid_value(x.find('value')) for x in bools['transaction_spend'] ]) and - all_true_and_not_empty(any(valid_date(x) for x in t.xpath('transaction-date|value')) for t in bools['transaction_spend']) - ), - 'transaction_currency': all( - all(map(valid_date, t.findall('value'))) and - all(x in CODELISTS[self._major_version()]['Currency'] for x in t.xpath('../@default-currency|./value/@currency')) for t in self.element.findall('transaction') - ), - 'budget': ( - bools['budget'] and - all( - valid_date(budget.find('period-start')) and - valid_date(budget.find('period-end')) and - valid_date(budget.find('value')) and - valid_value(budget.find('value')) - for budget in bools['budget'])), - 'budget_not_provided': ( - bools['budget_not_provided'] and - str(self._budget_not_provided()) in CODELISTS[self._major_version()]['BudgetNotProvided']), - 'location_point_pos': all_true_and_not_empty( - valid_coords(x.text) for x in bools['location_point_pos']), - 'sector_dac': ( - bools['sector_dac'] and - all(x.attrib.get('code') in CODELISTS[self._major_version()]['Sector'] for x in self.element.xpath('sector[@vocabulary="{}" or not(@vocabulary)]'.format(self._dac_5_code()))) and - all(x.attrib.get('code') in CODELISTS[self._major_version()]['SectorCategory'] for x in self.element.xpath('sector[@vocabulary="{}"]'.format(self._dac_3_code()))) - ), - 'document-link': all_true_and_not_empty( - valid_url(x) and x.find('category') is not None and x.find('category').attrib.get('code') in CODELISTS[self._major_version()]['DocumentCategory'] for x in bools['document-link']), - 'activity-website': all_true_and_not_empty(map(valid_url, bools['activity-website'])), - 'aid_type': ( - bools['aid_type'] and - # i) Value in default-aid-type/@code is found in the codelist - (all_true_and_not_empty([code in CODELISTS[self._major_version()]['AidType'] for code in self.element.xpath('default-aid-type/@code')]) - # Or ii) Each transaction has a aid-type/@code which is found in the codelist - or all_true_and_not_empty( - [set(x).intersection(CODELISTS[self._major_version()]['AidType']) - for x in [transaction.xpath('aid-type/@code') for transaction in self.element.xpath('transaction')]] - ) - )) - }) - return bools - - @returns_numberdict - def comprehensiveness(self): - if self._comprehensiveness_is_current(): - return { k:(1 if v and ( - k not in self.comprehensiveness_denominators() - or self.comprehensiveness_denominators()[k] - ) else 0) for k,v in self._comprehensiveness_bools().items() } - else: - return {} - - @returns_numberdict - def comprehensiveness_with_validation(self): - if self._comprehensiveness_is_current(): - return { k:(1 if v and ( - k not in self.comprehensiveness_denominators() - or self.comprehensiveness_denominators()[k] - ) else 0) for k,v in self._comprehensiveness_with_validation_bools().items() } - else: - return {} - - @returns_number - def comprehensiveness_denominator_default(self): - return 1 if self._comprehensiveness_is_current() else 0 - - @returns_numberdict - def comprehensiveness_denominators(self): - if self._comprehensiveness_is_current(): - dates = self.element.xpath('activity-date[@type="{}"]'.format(self._actual_start_code())) + self.element.xpath('activity-date[@type="{}"]'.format(self._planned_start_code())) - if dates: - start_date = iso_date(dates[0]) - else: - start_date = None - return { - 'recipient_language': 1 if len(self.element.findall('recipient-country')) == 1 else 0, - 'transaction_spend': 1 if start_date and start_date < self.today and (self.today - start_date) > datetime.timedelta(days=365) else 0, - 'transaction_traceability': 1 if (self.element.xpath('transaction[transaction-type/@code="{}"]'.format(self._incoming_funds_code()))) or self._is_donor_publisher() else 0, - } - else: - return { - 'recipient_language': 0, - 'transaction_spend': 0, - 'transaction_traceability': 0 - } - @returns_numberdict def humanitarian(self): humanitarian_sectors_dac_5_digit = ['72010', '72040', '72050', '73010', '74010', '74020'] @@ -1275,17 +745,6 @@ def _transaction_type_code(self, transaction): type_code = transaction_type.attrib.get('code') return type_code - @returns_numberdictdict - def transaction_dates(self): - """Generates a dictionary of dates for reported transactions, together - with the number of times they appear. - """ - out = defaultdict(lambda: defaultdict(int)) - for transaction in self.element.findall('transaction'): - date = transaction_date(transaction) - out[self._transaction_type_code(transaction)][unicode(date)] += 1 - return out - @returns_numberdictdict def activity_dates(self): out = defaultdict(lambda: defaultdict(int)) @@ -1295,13 +754,6 @@ def activity_dates(self): out[type_code][unicode(date)] += 1 return out - @returns_numberdictdict - def count_transactions_by_type_by_year(self): - out = defaultdict(lambda: defaultdict(int)) - for transaction in self.element.findall('transaction'): - out[self._transaction_type_code(transaction)][self._transaction_year(transaction)] += 1 - return out - @returns_numberdictdictdict def sum_transactions_by_type_by_year(self): out = defaultdict(lambda: defaultdict(lambda: defaultdict(Decimal))) @@ -1361,12 +813,6 @@ def sum_budgets_by_type_by_year_usd(self): out[budget_type]['USD'][year] += get_USD_value(currency, value, year) return out - @returns_numberdict - def count_planned_disbursements_by_year(self): - out = defaultdict(int) - for pd in self.element.findall('planned-disbursement'): - out[planned_disbursement_year(pd)] += 1 - return out @returns_numberdictdict def sum_planned_disbursements_by_year(self): @@ -1498,24 +944,6 @@ class PublisherStats(object): strict = False # (Setting this to true will ignore values that don't follow the schema) context = '' - @returns_dict - def bottom_hierarchy(self): - def int_or_None(x): - try: - return int(x) - except ValueError: - return None - - hierarchies = self.aggregated['by_hierarchy'].keys() - hierarchies_ints = [ x for x in map(int_or_None, hierarchies) if x is not None ] - if not hierarchies_ints: - return {} - bottom_hierarchy_key = str(max(hierarchies_ints)) - try: - return copy.deepcopy(self.aggregated['by_hierarchy'][bottom_hierarchy_key]) - except KeyError: - return {} - @returns_numberdict def publishers_per_version(self): versions = self.aggregated['versions'].keys() @@ -1547,51 +975,6 @@ def publisher_has_org_file(self): def publisher_unique_identifiers(self): return len(self.aggregated['iati_identifiers']) - @returns_dict - def reference_spend_data(self): - """Lookup the reference spend data (value and currency) for this publisher (obtained by using the - name of the folder), for years 2014 and 2015. - Outputs an empty string for each element where there is no data. - """ - if self.folder in reference_spend_data.keys(): - - # Note that the values may be strings or human-readable numbers (i.e. with commas to seperate thousands) - return { '2014': { 'ref_spend': reference_spend_data[self.folder]['2014_ref_spend'].replace(',','') if is_number(reference_spend_data[self.folder]['2014_ref_spend'].replace(',','')) else '', - 'currency': reference_spend_data[self.folder]['currency'], - 'official_forecast_usd': '' }, - '2015': { 'ref_spend': reference_spend_data[self.folder]['2015_ref_spend'].replace(',','') if is_number(reference_spend_data[self.folder]['2015_ref_spend'].replace(',','')) else '', - 'currency': reference_spend_data[self.folder]['currency'], - 'official_forecast_usd': reference_spend_data[self.folder]['2015_official_forecast'].replace(',','') if is_number(reference_spend_data[self.folder]['2015_official_forecast'].replace(',','')) else '' }, - 'spend_data_error_reported': 1 if reference_spend_data[self.folder]['spend_data_error_reported'] else 0, - 'DAC': 1 if reference_spend_data[self.folder]['DAC'] else 0 - } - else: - return {} - - @returns_dict - def reference_spend_data_usd(self): - """For each year that there is reference spend data for this publisher, convert this - to the USD value for the given year - Outputs an empty string for each element where there is no data. - """ - - output = {} - - # Construct a list of reference spend data related to years 2015 & 2014 only - ref_data_years = [x for x in self.reference_spend_data().items() if is_number(x[0])] - - # Loop over the years - for year, data in ref_data_years: - # Construct output dictionary with USD values - output[year] = {} - output[year]['ref_spend'] = str(get_USD_value(data['currency'], data['ref_spend'], year)) if is_number(data['ref_spend']) else '' - output[year]['official_forecast'] = data['official_forecast_usd'] if is_number(data['official_forecast_usd']) else '' - - # Append the spend error and DAC booleans and return - output['spend_data_error_reported'] = self.reference_spend_data().get('spend_data_error_reported', 0) - output['DAC'] = self.reference_spend_data().get('DAC', 0) - return output - @returns_numberdict def publisher_duplicate_identifiers(self): return {k:v for k,v in self.aggregated['iati_identifiers'].items() if v>1} @@ -1609,30 +992,6 @@ def _timeliness_transactions(self): else: return 'Beyond one year' - @no_aggregation - def timelag(self): - def previous_months_generator(d): - year = d.year - month = d.month - for i in range(0,12): - month -= 1 - if month <= 0: - year -= 1 - month = 12 - yield '{}-{}'.format(year,str(month).zfill(2)) - previous_months = list(previous_months_generator(self.today)) - transaction_months_with_year = self.aggregated['transaction_months_with_year'] - if [ x in transaction_months_with_year for x in previous_months[:3] ].count(True) >= 2: - return 'One month' - elif [ x in transaction_months_with_year for x in previous_months[:3] ].count(True) >= 1: - return 'A quarter' - elif True in [ x in transaction_months_with_year for x in previous_months[:6] ]: - return 'Six months' - elif True in [ x in transaction_months_with_year for x in previous_months[:12] ]: - return 'One year' - else: - return 'More than one year' - def _transaction_alignment(self): transaction_months = self.aggregated['transaction_months'].keys() if len(transaction_months) == 12: @@ -1644,36 +1003,6 @@ def _transaction_alignment(self): else: return '' - @no_aggregation - @memoize - def budget_length_median(self): - budget_lengths = self.aggregated['budget_lengths'] - budgets = sum(budget_lengths.values()) - i = 0 - median = None - for k,v in sorted([ (int(k),v) for k,v in budget_lengths.items()]): - i += v - if i >= (budgets/2.0): - if median: - # Handle the case where the median falls between two frequency bins - median = (median + k) / 2.0 - else: - median = k - if i != (budgets/2.0): - break - return median - - def _budget_alignment(self): - median = self.budget_length_median() - if median is None: - return 'Not known' - elif median < 100: - return 'Quarterly' - elif median < 370: - return 'Annually' - else: - return 'Beyond one year' - @no_aggregation def date_extremes(self): notnone = lambda x: x is not None @@ -1694,24 +1023,6 @@ def date_extremes(self): }, } - @no_aggregation - def most_recent_transaction_date(self): - """Computes the latest non-future transaction data across a dataset - """ - nonfuture_transaction_dates = filter(lambda x: x is not None and x <= self.today, - map(iso_date_match, sum((x.keys() for x in self.aggregated['transaction_dates'].values()), []))) - if nonfuture_transaction_dates: - return unicode(max(nonfuture_transaction_dates)) - - @no_aggregation - def latest_transaction_date(self): - """Computes the latest transaction data across a dataset. Can be in the future - """ - transaction_dates = filter(lambda x: x is not None, - map(iso_date_match, sum((x.keys() for x in self.aggregated['transaction_dates'].values()), []))) - if transaction_dates: - return unicode(max(transaction_dates)) - class OrganisationFileStats(GenericFileStats): """ Stats calculated for an IATI Organisation XML file. """ doc = None diff --git a/stats/tests/test_comprehensiveness.py b/stats/tests/test_comprehensiveness.py deleted file mode 100644 index 54f75a44ce2..00000000000 --- a/stats/tests/test_comprehensiveness.py +++ /dev/null @@ -1,1709 +0,0 @@ -# coding=utf-8 - -from lxml import etree -import datetime -import pytest - -from stats.dashboard import ActivityStats - -class MockActivityStats(ActivityStats): - def __init__(self, major_version): - self.major_version = major_version - return super(MockActivityStats, self).__init__() - - def _major_version(self): - return self.major_version - -@pytest.mark.parametrize('major_version', ['1', '2']) -def test_comprehensiveness_is_current(major_version): - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - - activity_stats.element = etree.fromstring(''' - - - - ''') - assert activity_stats._comprehensiveness_is_current() - - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - - ''') - assert not activity_stats._comprehensiveness_is_current() - - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - ''') - assert not activity_stats._comprehensiveness_is_current() - - - def end_planned_date(datestring): - """ - Create an activity_stats element with a specified 'end-planned' date. - Also sets the current date to 9990-06-01 - - Args: - datestring (str): An ISO date to be used as the 'end-planned' date for the activity_stats element to be returned. - """ - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - - '''.format('end-planned' if major_version == '1' else '3', datestring)) - return activity_stats - - # Any planned end dates before the current date should not be calculated as current - activity_stats = end_planned_date('9989-06-01') - assert not activity_stats._comprehensiveness_is_current() - activity_stats = end_planned_date('9989-12-31') - assert not activity_stats._comprehensiveness_is_current() - activity_stats = end_planned_date('9990-01-01') - assert not activity_stats._comprehensiveness_is_current() - - # Any end dates greater than the current date should be calculated as current - activity_stats = end_planned_date('9990-06-01') - assert activity_stats._comprehensiveness_is_current() - activity_stats = end_planned_date('9990-06-02') - assert activity_stats._comprehensiveness_is_current() - activity_stats = end_planned_date('9991-06-01') - assert activity_stats._comprehensiveness_is_current() - - - def datetype(typestring): - """ - Create an activity_stats element with a specified 'activity-date/@type' value and corresponding - date of 9989-06-01. Also sets the current date to 9990-06-01 - - Keyword arguments: - typestring -- An 'activity-date/@type' value for the activity_stats element to be returned. - """ - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - - '''.format(typestring)) - return activity_stats - - # Ignore start dates in computation to determine if an activity is current - activity_stats = datetype('start-planned' if major_version == '1' else '1') - assert not activity_stats._comprehensiveness_is_current() - activity_stats = datetype('start-actual' if major_version == '1' else '2') - assert not activity_stats._comprehensiveness_is_current() - - # But use all end dates in computation to determine if an activity is current - activity_stats = datetype('end-planned' if major_version == '1' else '3') - assert not activity_stats._comprehensiveness_is_current() - activity_stats = datetype('end-actual' if major_version == '1' else '4') - assert activity_stats._comprehensiveness_is_current() - - # If there are two end dates, 'end-planned' must be in the future, for the activity to be counted as current - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - - - ''' if major_version == '1' else ''' - - - - - ''') - assert not activity_stats._comprehensiveness_is_current() - - # Activity status should take priority over activity date - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - - - '''.format('end-actual' if major_version == '1' else '4')) - assert activity_stats._comprehensiveness_is_current() - - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - - - '''.format('end-actual' if major_version == '1' else '4')) - assert activity_stats._comprehensiveness_is_current() - - -@pytest.mark.parametrize('major_version', ['1', '2']) -def test_comprehensiveness_empty(major_version): - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - - - <description/> - <activity-status code="2"/> - <transaction> - <transaction-type/> - </transaction> - <transaction> - <!-- provider-activity-id only on one transaction should get no points --> - <provider-org provider-activity-id="AAA"/> - <transaction-type code="IF"/> - </transaction> - <transaction> - <transaction-type code="IF"/> - </transaction> - </iati-activity> - ''') - assert activity_stats.comprehensiveness() == { - 'version': 0, - 'reporting-org': 0, - 'iati-identifier': 0, - 'participating-org': 0, - 'title': 0, - 'description': 0, - 'activity-status': 1, - 'activity-date': 0, - 'sector': 0, - 'country_or_region': 0, - 'transaction_commitment': 0, - 'transaction_spend': 0, - 'transaction_currency': 0, - 'transaction_traceability': 0, - 'budget': 0, - 'budget_not_provided': 0, - 'contact-info': 0, - 'location': 0, - 'location_point_pos': 0, - 'sector_dac': 0, - 'capital-spend': 0, - 'document-link': 0, - 'activity-website': 0, - 'recipient_language': 0, - 'conditions_attached': 0, - 'result_indicator': 0, - 'aid_type': 0 - } - - -@pytest.mark.parametrize('major_version', ['1','2']) -def test_comprehensiveness_full(major_version): - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - root = etree.fromstring(''' - <iati-activities version="1.05"> - <iati-activity xml:lang="en"> - <reporting-org ref="AA-AAA">Reporting ORG Name</reporting-org> - <iati-identifier>AA-AAA-1</iati-identifier> - <participating-org/> - <title>A title - A description - - - - - - - - - - - - - - - - - - - - - test@example.org - - - - 31.616944 65.716944 - - - - - - - - - - - - ''' if major_version == '1' else ''' - - - - Reporting ORG Name - - AA-AAA-1 - - - <narrative>A title</narrative> - - - A description - - - - - - - - - - - - - - - - - - - - - - test@example.org - - - - 31.616944 65.716944 - - - - - - - - - - - - - - - ''') - activity_stats.element = root.find('iati-activity') - assert all(type(x) == int for x in activity_stats.comprehensiveness().values()) - assert activity_stats.comprehensiveness() == { - 'version': 1, - 'reporting-org': 1, - 'iati-identifier': 1, - 'participating-org': 1, - 'title': 1, - 'description': 1, - 'activity-status': 1, - 'activity-date': 1, - 'sector': 1, - 'country_or_region': 1, - 'transaction_commitment': 1, - 'transaction_spend': 1, - 'transaction_currency': 1, - 'transaction_traceability': 1, - 'budget': 1, - 'budget_not_provided': 0, - 'contact-info': 1, - 'location': 1, - 'location_point_pos': 1, - 'sector_dac': 1, - 'capital-spend': 1, - 'document-link': 1, - 'activity-website': 1, - 'recipient_language': 1, - 'conditions_attached': 1, - 'result_indicator': 1, - 'aid_type': 1 - } - - # Check recipient-region independently - activity_stats.element = etree.fromstring(''' - - - - - - ''') - comprehensiveness = activity_stats.comprehensiveness() - assert comprehensiveness['country_or_region'] == 1 - - -@pytest.mark.parametrize('major_version', ['1', '2']) -def test_comprehensiveness_other_passes(major_version): - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - root = etree.fromstring(''' - - - - - - - - - - - - - ''' if major_version == '1' else ''' - - - - - - - - - - - - - ''') - activity_stats.element = root.find('iati-activity') - assert all(type(x) == int for x in activity_stats.comprehensiveness().values()) - assert activity_stats.comprehensiveness() == { - 'version': 0, - 'reporting-org': 0, - 'iati-identifier': 0, - 'participating-org': 0, - 'title': 0, - 'description': 0, - 'activity-status': 1, - 'activity-date': 1, - 'sector': 0, - 'country_or_region': 0, - 'transaction_commitment': 0, - 'transaction_spend': 1, - 'transaction_currency': 1, - 'transaction_traceability': 0, - 'budget': 0, - 'budget_not_provided': 0, - 'contact-info': 0, - 'location': 0, - 'location_point_pos': 0, - 'sector_dac': 0, - 'capital-spend': 0, - 'document-link': 0, - 'activity-website': 0, - 'recipient_language': 0, - 'conditions_attached': 0, - 'result_indicator': 0, - 'aid_type': 1 - } - - -@pytest.mark.parametrize('major_version', ['1', '2']) -def test_comprehensiveness_location_other_passes(major_version): - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - - Name - - - ''') - assert activity_stats.comprehensiveness()['location'] == 1 - assert activity_stats.comprehensiveness()['location_point_pos'] == 0 - - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - - Name - - - ''') - assert activity_stats.comprehensiveness()['location'] == 1 - assert activity_stats.comprehensiveness()['location_point_pos'] == 0 - - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - - Name - - - ''') - assert activity_stats.comprehensiveness()['location'] == 1 - assert activity_stats.comprehensiveness()['location_point_pos'] == 0 - - -@pytest.mark.parametrize('major_version', ['1', '2']) -def test_comprehensiveness_recipient_language_passes(major_version): - # Set one and only one recipient-country - # Country code "AI" has valid language code "en" in helpers/transparency_indicator/country_lang_map.csv - - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - Activity title - - General activity description text. Long description of the activity with - no particular structure. - - - - - ''' if major_version == '1' else ''' - - - <narrative xml:lang="en">Activity title</narrative> - - - - General activity description text. Long description of the activity with - no particular structure. - - - - - - ''') - assert activity_stats.comprehensiveness()['recipient_language'] == 1 - - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - Activity title - - General activity description text. Long description of the activity with - no particular structure. - - - - - ''' if major_version == '1' else ''' - - - <narrative xml:lang="en">Activity title</narrative> - <narrative xml:lang="fr">Titre de l'activité</narrative> - - - - General activity description text. Long description of the activity with - no particular structure. - - - Activité générale du texte de description. Longue description de l'activité - sans structure particulière. - - - - - - ''') - assert activity_stats.comprehensiveness()['recipient_language'] == 1 - - -@pytest.mark.parametrize('major_version', ['1', '2']) -def test_comprehensiveness_recipient_language_fails(major_version): - # Set one and only one recipient-country - # Country code "AI" has valid language code "en" in helpers/transparency_indicator/country_lang_map.csv - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - Activity title - - General activity description text. Long description of the activity with - no particular structure. - - - - - ''' if major_version == '1' else ''' - - - <narrative>Activity title</narrative> - - - - General activity description text. Long description of the activity with - no particular structure. - - - - - - ''') - assert activity_stats.comprehensiveness()['recipient_language'] == 0 - - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - Titre de l'activité - - Activité générale du texte de description. Longue description de l'activité - sans structure particulière. - - - - - ''' if major_version == '1' else ''' - - - <narrative xml:lang="fr">Titre de l'activité</narrative> - - - - Activité générale du texte de description. Longue description de l'activité - sans structure particulière. - - - - - - ''') - assert activity_stats.comprehensiveness()['recipient_language'] == 0 - - -@pytest.mark.parametrize('major_version', ['1', '2']) -def test_comprehensiveness_recipient_language_fails_mulitple_countries(major_version): - # Set more than one recipient-country - - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - Activity title - - General activity description text. Long description of the activity with - no particular structure. - - - - - - ''' if major_version == '1' else ''' - - - <narrative xml:lang="en">Activity title</narrative> - - - - General activity description text. Long description of the activity with - no particular structure. - - - - - - - ''') - assert activity_stats.comprehensiveness()['recipient_language'] == 0 - - -@pytest.mark.parametrize('major_version', ['1', '2']) -def test_comprehensiveness_sector_other_passes(major_version): - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - - - ''') - assert activity_stats.comprehensiveness()['sector'] == 1 - assert activity_stats.comprehensiveness()['sector_dac'] == 1 - - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - - - ''') - assert activity_stats.comprehensiveness()['sector'] == 1 - assert activity_stats.comprehensiveness()['sector_dac'] == 0 - - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - - - '''.format('DAC' if major_version == '1' else '1')) - assert activity_stats.comprehensiveness()['sector'] == 1 - assert activity_stats.comprehensiveness()['sector_dac'] == 1 - - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - - - '''.format('DAC-3' if major_version == '1' else '2')) - assert activity_stats.comprehensiveness()['sector'] == 1 - assert activity_stats.comprehensiveness()['sector_dac'] == 1 - - -@pytest.mark.parametrize('major_version', ['1', '2']) -@pytest.mark.parametrize('key', [ - 'version', 'participating-org', 'activity-status', - 'activity-date', 'sector', 'country_or_region', - 'transaction_commitment', 'transaction_currency', - 'budget', - 'location_point_pos', 'sector_dac', 'document-link', 'activity-website', - 'aid_type' - # iati_identifier is exluded as it does not validate for v1.xx data as a special case: https://github.com/IATI/IATI-Dashboard/issues/399 -]) -def test_comprehensiveness_with_validation(key, major_version): - activity_stats_not_valid = MockActivityStats(major_version) - activity_stats_not_valid.today = datetime.date(2014, 1, 1) - root_not_valid = etree.fromstring(''' - - - - AAA-1 - - - - - - - - - - - - - - - - - - - - - - - - - - - 1.5,2 - - - - - - notaurl - - - ''' if major_version == '1' else ''' - - - - AAA-1 - - - - - - - - - - - - - - - - - - - - - - - - - - - 1.5,2 - - - - - - - - - notaurl - - - ''') - activity_stats_not_valid.element = root_not_valid.find('iati-activity') - activity_stats_valid = MockActivityStats(major_version) - activity_stats_valid.today = datetime.date(2014, 1, 1) - root_valid = etree.fromstring(''' - - - - AAA-1 - - - - - - - - - - - - - - - - - 1.0 - - - - - 1.0 - - - - - - 1.0 - - - - 1.5 2 - - - - - - http://example.org/ - - - ''' if major_version == '1' else ''' - - - - AAA-1 - - - - - - - - - - - - - - - - - 1.0 - - - - - 1.0 - - - - - - 1.0 - - - - 1.5 2 - - - - - - - - - - - ''') - activity_stats_valid.element = root_valid.find('iati-activity') - comprehensiveness = activity_stats_not_valid.comprehensiveness() - not_valid = activity_stats_not_valid.comprehensiveness_with_validation() - valid = activity_stats_valid.comprehensiveness_with_validation() - assert comprehensiveness[key] == 1 - assert not_valid[key] == 0 - assert valid[key] == 1 - - -@pytest.mark.parametrize('major_version', ['1', '2']) -def test_comprehensiveness_with_validation_transaction_spend(major_version): - key = 'transaction_spend' - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - root = etree.fromstring(''' - - - - - - - - - - - ''' if major_version == '1' else ''' - - - - - - - - - - - ''') - activity_stats.element = root.find('iati-activity') - activity_stats_valid = MockActivityStats(major_version) - activity_stats_valid.today = datetime.date(9990, 6, 1) - root_valid = etree.fromstring(''' - - - - - - - 1.0 - - - - - ''' if major_version == '1' else ''' - - - - - - - 1.0 - - - - - ''') - activity_stats_valid.element = root_valid.find('iati-activity') - comprehensiveness = activity_stats.comprehensiveness() - print(activity_stats._comprehensiveness_bools()) - not_valid = activity_stats.comprehensiveness_with_validation() - valid = activity_stats_valid.comprehensiveness_with_validation() - assert comprehensiveness[key] == 1 - assert not_valid[key] == 0 - assert valid[key] == 1 - - -@pytest.mark.parametrize('major_version', ['1', '2']) -def test_valid_single_recipient_country(major_version): - activity_stats = MockActivityStats(major_version) - activity_stats.element = etree.fromstring(''' - - - - - ''') - assert activity_stats.comprehensiveness_with_validation()['country_or_region'] == 1 - - activity_stats = MockActivityStats(major_version) - activity_stats.element = etree.fromstring(''' - - - - - ''') - assert activity_stats.comprehensiveness_with_validation()['country_or_region'] == 1 - - -@pytest.mark.parametrize('major_version', ['1', '2']) -def test_aid_type_passes(major_version): - activity_stats = MockActivityStats(major_version) - activity_stats.element = etree.fromstring(''' - - - - - ''') - assert activity_stats.comprehensiveness()['aid_type'] == 1 - - activity_stats = MockActivityStats(major_version) - activity_stats.element = etree.fromstring(''' - - - - - - - - - - ''') - assert activity_stats.comprehensiveness()['aid_type'] == 1 - - -@pytest.mark.parametrize('major_version', ['1', '2']) -def test_aid_type_fails(major_version): - activity_stats = MockActivityStats(major_version) - activity_stats.element = etree.fromstring(''' - - - - - - - - - - ''') - assert activity_stats.comprehensiveness()['aid_type'] == 0 - - -@pytest.mark.parametrize('major_version', ['1', '2']) -def test_aid_type_valid(major_version): - activity_stats = MockActivityStats(major_version) - activity_stats.element = etree.fromstring(''' - - - - - ''') - assert activity_stats.comprehensiveness_with_validation()['aid_type'] == 1 - - activity_stats = MockActivityStats(major_version) - activity_stats.element = etree.fromstring(''' - - - - - - - - - - - ''') - assert activity_stats.comprehensiveness_with_validation()['aid_type'] == 1 - - -@pytest.mark.parametrize('major_version', ['1', '2']) -def test_aid_type_not_valid(major_version): - activity_stats = MockActivityStats(major_version) - activity_stats.element = etree.fromstring(''' - - - - - ''') - assert activity_stats.comprehensiveness_with_validation()['aid_type'] == 0 - - activity_stats = MockActivityStats(major_version) - activity_stats.element = etree.fromstring(''' - - - - - - - - - - ''') - assert activity_stats.comprehensiveness_with_validation()['aid_type'] == 0 - - activity_stats = MockActivityStats(major_version) - activity_stats.element = etree.fromstring(''' - - - - - - - - - - ''') - assert activity_stats.comprehensiveness_with_validation()['aid_type'] == 0 - - -# Note v1.xx data gets an automatic pass for iati_identifier as a special case: https://github.com/IATI/IATI-Dashboard/issues/399 -def test_iati_identifier_valid_v1_passes(): - activity_stats = MockActivityStats('1') - activity_stats.element = etree.fromstring(''' - - Reporting ORG Name - AA-AAA-1 - - - ''') - assert activity_stats.comprehensiveness_with_validation()['iati-identifier'] == 1 - - activity_stats = MockActivityStats('1') - activity_stats.element = etree.fromstring(''' - - Reporting ORG Name - NOT-PREFIXED-WITH-REPORTING-ORG_AA-AAA-1 - - - ''') - assert activity_stats.comprehensiveness_with_validation()['iati-identifier'] == 1 - - -def test_iati_identifier_valid_v2_passes(): - activity_stats = MockActivityStats('2') - activity_stats.element = etree.fromstring(''' - - - Reporting ORG Name - - AA-AAA-1 - - - ''') - assert activity_stats.comprehensiveness_with_validation()['iati-identifier'] == 1 - - activity_stats = MockActivityStats('2') - activity_stats.element = etree.fromstring(''' - - - A new reporting org name (BB-BBB) - - AA-AAA-1 - - - Reporting org name (who previously were known as AA-AAA) - - - - - ''') - assert activity_stats.comprehensiveness_with_validation()['iati-identifier'] == 1 - - -def test_iati_identifier_valid_v2_fails(): - activity_stats = MockActivityStats('2') - activity_stats.element = etree.fromstring(''' - - - Reporting ORG Name - - AA-AAA-1 - - - ''') - assert activity_stats.comprehensiveness_with_validation()['iati-identifier'] == 0 - - activity_stats = MockActivityStats('2') - activity_stats.element = etree.fromstring(''' - - - A new reporting org name (BB-BBB) - - AA-AAA-1 - - - - - ''') - assert activity_stats.comprehensiveness_with_validation()['iati-identifier'] == 0 - - -@pytest.mark.parametrize('major_version', ['1', '2']) -def test_valid_sector_no_vocab(major_version): - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(2014, 1, 1) - root = etree.fromstring(''' - - - - - - - - ''') - activity_stats.element = root.find('iati-activity') - activity_stats_valid = MockActivityStats(major_version) - activity_stats_valid.today = datetime.date(9990, 6, 1) - root_valid = etree.fromstring(''' - - - - - - - - ''') - activity_stats_valid.element = root_valid.find('iati-activity') - assert activity_stats.comprehensiveness()['sector_dac'] == 1 - assert activity_stats.comprehensiveness_with_validation()['sector_dac'] == 0 - assert activity_stats_valid.comprehensiveness_with_validation()['sector_dac'] == 1 - - -@pytest.mark.parametrize('major_version', ['1', '2']) -def test_valid_location(major_version): - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - - - +1.5 -2 - - - - ''') - assert activity_stats.comprehensiveness_with_validation()['location_point_pos'] == 1 - - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - - - x1.5 -2 - - - - ''') - assert activity_stats.comprehensiveness_with_validation()['location_point_pos'] == 0 - - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - - - 1,5 2,5 - - - - ''') - assert activity_stats.comprehensiveness_with_validation()['location_point_pos'] == 0 - - -@pytest.mark.parametrize('major_version', ['1', '2']) -def test_comprehensiveness_transaction_level_elements(major_version): - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - - - - - - ''') - comprehensiveness = activity_stats.comprehensiveness() - assert comprehensiveness['sector'] == (0 if major_version == '1' else 1) - assert comprehensiveness['country_or_region'] == (0 if major_version == '1' else 1) - - # Check recipient-region too - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - - - - - ''') - comprehensiveness = activity_stats.comprehensiveness() - assert comprehensiveness['country_or_region'] == (0 if major_version == '1' else 1) - - # If is only at transaction level, but not for all transactions, we should get 0 - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - - - - - - - - ''') - comprehensiveness = activity_stats.comprehensiveness() - assert comprehensiveness['sector'] == 0 - assert comprehensiveness['country_or_region'] == 0 - - -@pytest.mark.parametrize('major_version', ['1', '2']) -@pytest.mark.parametrize('key', ['sector', 'country_or_region']) -def test_comprehensiveness_with_validation_transaction_level_elements(key, major_version): - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(2014, 1, 1) - root = etree.fromstring(''' - - - - - - - - - - ''') - activity_stats.element = root.find('iati-activity') - activity_stats_valid = MockActivityStats(major_version) - root_valid = etree.fromstring(''' - - - - - - - - - - ''') - activity_stats_valid.element = root_valid.find('iati-activity') - comprehensiveness = activity_stats.comprehensiveness() - not_valid = activity_stats.comprehensiveness_with_validation() - valid = activity_stats_valid.comprehensiveness_with_validation() - assert comprehensiveness[key] == 1 - assert not_valid[key] == 0 - assert valid[key] == (0 if major_version == '1' else 1) - - - -## Denominator - - -@pytest.mark.parametrize('major_version', ['1', '2']) -def test_comprehensivness_denominator_default(major_version): - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats._comprehensiveness_is_current = lambda: True - assert activity_stats.comprehensiveness_denominator_default() == 1 - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats._comprehensiveness_is_current = lambda: False - assert activity_stats.comprehensiveness_denominator_default() == 0 - - -@pytest.mark.parametrize('major_version', ['1', '2']) -def test_comprehensivness_denominator_empty(major_version): - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - ''') - assert activity_stats.comprehensiveness_denominators() == { - 'recipient_language': 0, - 'transaction_spend': 0, - 'transaction_traceability': 0 - } - - -@pytest.mark.parametrize('major_version', ['1', '2']) -@pytest.mark.parametrize('key', [ - 'transaction_spend', 'transaction_traceability' ]) -def test_transaction_exclusions(key, major_version): - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - - - - - - - - ''') - assert activity_stats.comprehensiveness_denominators()[key] == 0 - - # Broken activity-date/@iso-date - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - - - - - - - - ''') - assert activity_stats.comprehensiveness_denominators()[key] == 0 - - -@pytest.mark.parametrize('major_version', ['1', '2']) -@pytest.mark.parametrize('key', [ - 'transaction_spend', 'transaction_traceability' ]) -def test_transaction_non_exclusions(key, major_version): - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - - - - - - - - - ''' if major_version == '1' else ''' - - - - - - - - - - - ''') - assert activity_stats.comprehensiveness_denominators()[key] == 1 - - -@pytest.mark.parametrize('major_version', ['1', '2']) -def test_comprehensivness_denominator_recipient_language_true(major_version): - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - - - ''') - assert activity_stats.comprehensiveness_denominators()['recipient_language'] == 1 - - -@pytest.mark.parametrize('major_version', ['1', '2']) -def test_comprehensivness_denominator_recipient_language_false(major_version): - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - activity_stats.element = etree.fromstring(''' - - - - - - ''') - assert activity_stats.comprehensiveness_denominators()['recipient_language'] == 0 - - -@pytest.mark.parametrize('major_version', ['2']) -def test_comprehensiveness_dac_sector_codes_v2(major_version): - """Check that DAC sector codes in transactions in version 2 are also included.""" - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - root = etree.fromstring(''' - - - - Reporting ORG Name - - AA-AAA-1 - - - <narrative>A title</narrative> - - - A description - - - - - - - - - - - - - - - - - - - - - - - - test@example.org - - - - 31.616944 65.716944 - - - - - - - - - - - - - - - ''') - activity_stats.element = root.find('iati-activity') - assert all(type(x) == int for x in activity_stats.comprehensiveness().values()) - assert activity_stats.comprehensiveness() == { - 'version': 1, - 'reporting-org': 1, - 'iati-identifier': 1, - 'participating-org': 1, - 'title': 1, - 'description': 1, - 'activity-status': 1, - 'activity-date': 1, - 'sector': 1, - 'country_or_region': 1, - 'transaction_commitment': 1, - 'transaction_spend': 1, - 'transaction_currency': 1, - 'transaction_traceability': 1, - 'budget': 1, - 'budget_not_provided': 0, - 'contact-info': 1, - 'location': 1, - 'location_point_pos': 1, - 'sector_dac': 1, - 'capital-spend': 1, - 'document-link': 1, - 'activity-website': 1, - 'recipient_language': 1, - 'conditions_attached': 1, - 'result_indicator': 1, - 'aid_type': 1 - } - -@pytest.mark.parametrize('major_version', ['2']) -def test_comprehensiveness_dac_sector_codes_v2_incomplete(major_version): - """Check that DAC sector codes in transactions in version 2 return False when sector not included in every transaction.""" - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - root = etree.fromstring(''' - - - - Reporting ORG Name - - AA-AAA-1 - - - <narrative>A title</narrative> - - - A description - - - - - - - - - - - - - - - - - - - - - - - test@example.org - - - - 31.616944 65.716944 - - - - - - - - - - - - - - - ''') - activity_stats.element = root.find('iati-activity') - assert all(type(x) == int for x in activity_stats.comprehensiveness().values()) - assert activity_stats.comprehensiveness() == { - 'version': 1, - 'reporting-org': 1, - 'iati-identifier': 1, - 'participating-org': 1, - 'title': 1, - 'description': 1, - 'activity-status': 1, - 'activity-date': 1, - 'sector': 0, - 'country_or_region': 1, - 'transaction_commitment': 1, - 'transaction_spend': 1, - 'transaction_currency': 1, - 'transaction_traceability': 1, - 'budget': 1, - 'budget_not_provided': 0, - 'contact-info': 1, - 'location': 1, - 'location_point_pos': 1, - 'sector_dac': 0, - 'capital-spend': 1, - 'document-link': 1, - 'activity-website': 1, - 'recipient_language': 1, - 'conditions_attached': 1, - 'result_indicator': 1, - 'aid_type': 1 - } - - -@pytest.mark.parametrize('major_version', ['1']) -def test_comprehensiveness_v1_returns_false(major_version): - """Check that V1 activity returns false when no valid sector element entered at activity level.""" - activity_stats = MockActivityStats(major_version) - activity_stats.today = datetime.date(9990, 6, 1) - root = etree.fromstring(''' - - - Reporting ORG Name - AA-AAA-1 - - A title - A description - - - - - - - - - - - - - - - - - - - - - - - test@example.org - - - - 31.616944 65.716944 - - - - - - - - - - - - ''') - activity_stats.element = root.find('iati-activity') - assert all(type(x) == int for x in activity_stats.comprehensiveness().values()) - assert activity_stats.comprehensiveness() == { - 'version': 1, - 'reporting-org': 1, - 'iati-identifier': 1, - 'participating-org': 1, - 'title': 1, - 'description': 1, - 'activity-status': 1, - 'activity-date': 1, - 'sector': 0, - 'country_or_region': 1, - 'transaction_commitment': 1, - 'transaction_spend': 1, - 'transaction_currency': 1, - 'transaction_traceability': 1, - 'budget': 1, - 'budget_not_provided': 0, - 'contact-info': 1, - 'location': 1, - 'location_point_pos': 1, - 'sector_dac': 0, - 'capital-spend': 1, - 'document-link': 1, - 'activity-website': 1, - 'recipient_language': 1, - 'conditions_attached': 1, - 'result_indicator': 1, - 'aid_type': 1 - } diff --git a/stats/tests/test_forward_looking.py b/stats/tests/test_forward_looking.py deleted file mode 100644 index cea9557d2eb..00000000000 --- a/stats/tests/test_forward_looking.py +++ /dev/null @@ -1,292 +0,0 @@ -import datetime -from lxml import etree -import pytest - -from stats.common import budget_year -from stats.dashboard import ActivityStats -from test_comprehensiveness import MockActivityStats - - -def test_forwardlooking_is_current(): - activity_stats = ActivityStats() - - # If there are no end dates, the activity is current - activity_stats.element = etree.fromstring(''' - ''') - assert activity_stats._forwardlooking_is_current(9990) - activity_stats.element = etree.fromstring(''' - - - ''') - assert activity_stats._forwardlooking_is_current(9990) - - # If there is an end date before the given year, it's not current - activity_stats.element = etree.fromstring(''' - - ''') - assert not activity_stats._forwardlooking_is_current(9990) - activity_stats.element = etree.fromstring(''' - - ''') - assert not activity_stats._forwardlooking_is_current(9990) - - # If there is an end date on or after the given year, it is current - activity_stats.element = etree.fromstring(''' - - ''') - assert activity_stats._forwardlooking_is_current(9990) - activity_stats.element = etree.fromstring(''' - - ''') - assert activity_stats._forwardlooking_is_current(9990) - activity_stats.element = etree.fromstring(''' - - - ''') - assert activity_stats._forwardlooking_is_current(9990) - - -def wrap_activity(activity): - return etree.fromstring('{}'.format(activity)).find('iati-activity') - - -def test_forwardlooking_is_current_2xx(): - activity_stats = ActivityStats() - - # If there are no end dates, the activity is current - activity_stats.element = wrap_activity(''' - ''') - assert activity_stats._forwardlooking_is_current(9990) - activity_stats.element = wrap_activity(''' - - - ''') - assert activity_stats._forwardlooking_is_current(9990) - - # If there is an end date before the given year, it's not current - activity_stats.element = wrap_activity(''' - - ''') - assert not activity_stats._forwardlooking_is_current(9990) - activity_stats.element = wrap_activity(''' - - ''') - assert not activity_stats._forwardlooking_is_current(9990) - - # If there is an end date on or after the given year, it is current - activity_stats.element = wrap_activity(''' - - ''') - assert activity_stats._forwardlooking_is_current(9990) - activity_stats.element = wrap_activity(''' - - ''') - assert activity_stats._forwardlooking_is_current(9990) - activity_stats.element = wrap_activity(''' - - - ''') - assert activity_stats._forwardlooking_is_current(9990) - - -@pytest.mark.parametrize('major_version', ['1', '2']) -@pytest.mark.parametrize('year', [9980, 9981, 9982]) -def test_forwardlooking_activities_with_budgets_true(major_version, year): - date_code_runs = datetime.date(year, 1, 1) - activity_stats = MockActivityStats(major_version) - # Activity with budgets for each year of operation - activity_stats.element = etree.fromstring(''' - - - - - - - 1000 - - - - - 1000 - - - - - 1000 - - - ''' if major_version == '1' else ''' - - - - - - - 1000 - - - - - 1000 - - - - - 1000 - - - ''') - assert activity_stats.forwardlooking_activities_with_budgets(date_code_runs)[year] - - -@pytest.mark.parametrize('major_version', ['1', '2']) -@pytest.mark.parametrize('year', [9980, 9981, 9982]) -def test_forwardlooking_activities_with_budgets_false(major_version, year): - date_code_runs = datetime.date(year, 1, 1) - activity_stats = MockActivityStats(major_version) - # Activity with no budgets for any year of operation - activity_stats.element = etree.fromstring(''' - - - - - ''' if major_version == '1' else ''' - - - - - ''') - assert activity_stats.forwardlooking_activities_with_budgets(date_code_runs)[year] == 0 - - date_code_runs = datetime.date(year, 1, 1) - # Activity ends within six months, regardless that a budget is declared for the full year - activity_stats.element = etree.fromstring(''' - - - - - - - 1000 - - - ''' if major_version == '1' else ''' - - - - - - - 1000 - - - ''') - assert activity_stats.forwardlooking_activities_with_budgets(date_code_runs)[year] == 0 - - -@pytest.mark.parametrize('major_version', ['1', '2']) -def test_forwardlooking_activities_with_budgets_ends_in_six_months(major_version): - activity_stats = MockActivityStats(major_version) - # Activity ends in one year and six months of 9980-01-01, regardless that a budget is declared for both full years - date_code_runs = datetime.date(9980, 1, 1) - activity_stats.element = etree.fromstring(''' - - - - - - - 1000 - - - - - 1000 - - - ''' if major_version == '1' else ''' - - - - - - - 1000 - - - - - 1000 - - - ''') - assert activity_stats.forwardlooking_activities_with_budgets(date_code_runs)[9980] == 1 - assert activity_stats.forwardlooking_activities_with_budgets(date_code_runs)[9981] == 1 - assert activity_stats.forwardlooking_activities_with_budgets(date_code_runs)[9982] == 0 - - -@pytest.mark.parametrize('major_version', ['1', '2']) -def test_get_ratio_commitments_disbursements(major_version): - # Using expected data - activity_stats = MockActivityStats(major_version) - activity_stats.element = etree.fromstring(''' - - - - 100 - - - - - 50 - - - - - 50 - - - - ''' if major_version == '1' else ''' - - - - 100 - - - - - 50 - - - - - 50 - - - - ''') - assert activity_stats._get_ratio_commitments_disbursements(2012) == 0.5 - - # Missing transaction date, value and currency - activity_stats = MockActivityStats(major_version) - activity_stats.element = etree.fromstring(''' - - - - - - - - - ''' if major_version == '1' else ''' - - - - - - - - - ''') - assert activity_stats._get_ratio_commitments_disbursements(2012) == None diff --git a/stats/tests/test_hierarchy.py b/stats/tests/test_hierarchy.py deleted file mode 100644 index e970a758272..00000000000 --- a/stats/tests/test_hierarchy.py +++ /dev/null @@ -1,78 +0,0 @@ -from lxml import etree - -from stats.dashboard import ActivityStats, PublisherStats - -def test_hierarchies(): - activity_stats = ActivityStats() - activity_stats.element = etree.fromstring(''' - - - ''') - assert activity_stats.hierarchies() == {None: 1} - - activity_stats = ActivityStats() - activity_stats.element = etree.fromstring(''' - - - ''') - assert activity_stats.hierarchies() == {'3': 1} - - -def test_by_hierarchy(): - # Unlike the hierarchies dict, by_hierarchy should treat activities without - # a hierarchy attribute as hierarchy 1 - activity_stats = ActivityStats() - activity_stats.element = etree.fromstring(''' - - - ''') - assert activity_stats.by_hierarchy().keys() == [ '1' ] - - activity_stats = ActivityStats() - activity_stats.element = etree.fromstring(''' - - - ''') - assert activity_stats.by_hierarchy().keys() == [ '3' ] - - -def test_bottom_hierarchy(): - publisher_stats = PublisherStats() - - publisher_stats.aggregated = { - 'by_hierarchy': { - } - } - assert publisher_stats.bottom_hierarchy() == {} - - publisher_stats.aggregated = { - 'by_hierarchy': { - '1': { 'testkey': 'v1' }, - '2': { 'testkey': 'v2' }, - '3': { 'testkey': 'v3' } - } - } - assert publisher_stats.bottom_hierarchy() == { 'testkey': 'v3' } - - -def test_bottom_hierarchy_non_integer(): - """ Hierachy values that are not integers should be ignored """ - - publisher_stats = PublisherStats() - - publisher_stats.aggregated = { - 'by_hierarchy': { - 'notaninteger': { 'testkey': 'v_notaninteger' }, - } - } - assert publisher_stats.bottom_hierarchy() == {} - - publisher_stats.aggregated = { - 'by_hierarchy': { - 'notaninteger': { 'testkey': 'v_notaninteger' }, - '2': { 'testkey': 'v2' }, - '3': { 'testkey': 'v3' } - } - } - assert publisher_stats.bottom_hierarchy() == { 'testkey': 'v3' } -