Skip to content

Commit

Permalink
Merge 651e61a into cc8cb34
Browse files Browse the repository at this point in the history
  • Loading branch information
@BibianaC
BibianaC committed Apr 8, 2019
2 parents cc8cb34 + 651e61a commit e7b22a2
Show file tree
Hide file tree
Showing 10 changed files with 816 additions and 38 deletions.
432 changes: 432 additions & 0 deletions cove_iati/fixtures/basic_iati_org_valid_repeat_identifiers.xml
View file

Large diffs are not rendered by default.

262 changes: 262 additions & 0 deletions cove_iati/fixtures/iati_openag_tag_repeat_identifiers.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,262 @@
<iati-activities version="2.03">
<iati-activity>
<iati-identifier>AA-AAA-123123-AA123</iati-identifier>
<reporting-org ref="NO-ORGIDS-10000" type="40"><narrative>Reporting Organisation A</narrative></reporting-org>
<title><narrative>A title</narrative></title>
<description><narrative>A description</narrative></description>
<participating-org role="1" ref="TZ-BRLA-20000"/>
<activity-status code="3"/>
<activity-date type="start-actual" iso-date="2017-01-01"/>
<recipient-country code="TZ" percentage="100"/>
<tag vocabulary-uri="http://aims.fao.org/aos/agrovoc/" code="c_4397">
<narrative xml:lang="en">Livestock</narrative>
</tag>
<transaction>
<transaction-type code="2"/>
<transaction-date iso-date="2017-01-01"/>
<value value-date="2017-01-01">20</value>
</transaction>
<transaction>
<transaction-type code="3"/>
<transaction-date iso-date="2017-01-01"/>
<value value-date="2017-01-01">20</value>
</transaction>
<transaction>
<transaction-type code="2"/>
<transaction-date iso-date="2017-01-01"/>
<value value-date="2017-01-01">20</value>
</transaction>
<transaction>
<transaction-type code="3"/>
<transaction-date iso-date="2017-01-01"/>
<value value-date="2017-01-01" >40</value>
</transaction>
</iati-activity>
<iati-activity>
<iati-identifier>BB-BBB-123123-BB123</iati-identifier>
<reporting-org ref="TZ-BRLA-30000" type="40"><narrative>Reporting Organisation B</narrative></reporting-org>
<title><narrative>A title</narrative></title>
<description><narrative>A description</narrative></description>
<participating-org role="1" ref="NO-ORGIDS-40000"/>
<activity-status code="3"/><activity-date type="2" iso-date="2017-01-01"/>
<recipient-country code="TZ" percentage="100"/>
<location>
<location-id vocabulary="G1" />
<name>
<narrative>Ilala District, Dar es Salaam, Tanzania TZ</narrative>
</name>
<description>
<narrative>Ilala District is one of three districts in Dar es Salaam</narrative>
</description>
<administrative vocabulary="G1" level="2" code="159239"/>
<point srsName="http://www.opengis.net/def/crs/EPSG/0/4326">
<pos>-6.91805, 39.16254</pos>
</point>
<exactness code="1"/>
<location-class code="1"/>
<feature-designation code="ADM2"/>
</location>
<tag vocabulary="98" vocabulary-uri="http://bad.org" code="c_4397">
<narrative xml:lang="en">Livestock</narrative>
</tag>
<transaction>
<transaction-type code="2"/>
<transaction-date iso-date="2017-01-01"/>
<value value-date="2017-01-01">10</value>
</transaction>
<transaction>
<transaction-type code="2"/>
<transaction-date iso-date="2017-01-01"/>
<value value-date="2017-01-01">20</value>
</transaction>
<transaction>
<transaction-type code="2"/>
<transaction-date iso-date="2017-01-01"/>
<value value-date="2017-01-01">30</value>
</transaction>
<transaction>
<transaction-type code="3"/>
<transaction-date iso-date="2017-01-01"/>
<value value-date="2017-01-01">40</value>
</transaction>
</iati-activity>
<iati-activity>
<iati-identifier>CC-CCC-789789-CC789</iati-identifier>
<reporting-org ref="TZ-BRLA-50000" type="40"><narrative>Reporting Organisation C</narrative></reporting-org>
<title><narrative>A title</narrative></title>
<description><narrative>A description</narrative></description>
<participating-org role="1" ref="TZ-BRLA-60000"/>
<activity-status code="3"/>
<activity-date type="start-actual" iso-date="2017-01-01"/>
<recipient-country code="AF" percentage="30"/>
<location>
<location-reach code="1"/>
<location-id code="159239" />
<name>
<narrative>Ilala District, Dar es Salaam, Tanzania TZ</narrative>
</name>
<description>
<narrative>Ilala District is one of three districts in Dar es Salaam</narrative>
</description>
<administrative vocabulary="G1" level="2" code="159239"/>
<point srsName="http://www.opengis.net/def/crs/EPSG/0/4326">
<pos>-6.91805, 39.16254</pos>
</point>
<exactness code="1"/>
<location-class code="1"/>
<feature-designation code="ADM2"/>
</location>
<tag vocabulary="01" vocabulary-uri="http://aims.fao.org/aos/agrovoc/" code="c_4397">
<narrative xml:lang="en">Livestock</narrative>
</tag>
<transaction>
<transaction-type code="2"/>
<transaction-date iso-date="2017-01-01"/>
<value value-date="2017-01-01">10</value>
</transaction>
<transaction>
<transaction-type code="3"/>
<transaction-date iso-date="2017-01-01"/>
<value value-date="2017-01-01">20</value>
</transaction>
<transaction>
<transaction-type code="2"/>
<transaction-date iso-date="2017-01-01"/>
<value value-date="2017-01-01">30</value>
</transaction>
<transaction>
<transaction-type code="3"/>
<transaction-date iso-date="2017-01-01"/>
<value value-date="2017-01-01">40</value>
</transaction>
</iati-activity>
<iati-activity>
<iati-identifier>DD-DDD-789789-DD789</iati-identifier>
<reporting-org ref="TZ-BRLA-70000" type="40"><narrative>Reporting Organisation D</narrative></reporting-org>
<title><narrative>A title</narrative></title>
<description><narrative>A description</narrative></description>
<participating-org role="1" ref="TZ-BRLA-80000"/>
<activity-status code="3"/><activity-date type="start-actual" iso-date="2017-01-01"/>
<recipient-country code="AF" percentage="30"/>
<location>
<location-reach code="1"/>
<location-id vocabulary="G1" code="159239" />
<name>
<narrative>Ilala District, Dar es Salaam, Tanzania TZ</narrative>
</name>
<description>
<narrative>Ilala District is one of three districts in Dar es Salaam</narrative>
</description>
<administrative vocabulary="G1" level="2" code="159239"/>
<point srsName="http://www.opengis.net/def/crs/EPSG/0/4326">
<pos>-6.91805, 39.16254</pos>
</point>
<exactness code="1"/>
<location-class code="1"/>
<feature-designation code="ADM2"/>
</location>
<tag vocabulary="98" vocabulary-uri="http://aims.fao.org/aos/agrovoc/">
<narrative xml:lang="en">Livestock</narrative>
</tag>
<transaction>
<transaction-type code="2"/>
<transaction-date iso-date="2017-01-01"/>
<value value-date="2012-01-01">10</value>
</transaction>
<transaction>
<transaction-type code="3"/>
<transaction-date iso-date="2017-01-01"/>
<value value-date="2017-01-01">20</value>
</transaction>
<transaction>
<transaction-type code="2"/>
<transaction-date iso-date="2017-01-01"/>
<value value-date="2017-01-01">30</value>
</transaction>
<transaction>
<transaction-type code="3"/>
<transaction-date iso-date="2017-01-01"/>
<value value-date="2017-01-01">40</value>
</transaction>
</iati-activity>
<iati-activity>
<iati-identifier>DD-DDD-789789-DD789</iati-identifier>
<reporting-org ref="TZ-BRLA-90000" type="40"><narrative>Reporting Organisation D</narrative></reporting-org>
<title><narrative>A title</narrative></title>
<description><narrative>A description</narrative></description>
<participating-org role="1" ref="TZ-BRLA-100000"/>
<activity-status code="3"/><activity-date type="start-actual" iso-date="2017-01-01"/>
<recipient-country code="AF" percentage="30"/>
<location>
<location-reach code="1"/>
<name>
<narrative>Ilala District, Dar es Salaam, Tanzania TZ</narrative>
</name>
<description>
<narrative>Ilala District is one of three districts in Dar es Salaam</narrative>
</description>
<administrative vocabulary="G1" level="2" code="159239"/>
<point srsName="http://www.opengis.net/def/crs/EPSG/0/4326">
<pos>-6.91805, 39.16254</pos>
</point>
<exactness code="1"/>
<location-class code="1"/>
<feature-designation code="ADM2"/>
</location>
<transaction>
<transaction-type code="2"/>
<transaction-date iso-date="2017-01-01"/>
<value value-date="2012-01-01">10</value>
</transaction>
<transaction>
<transaction-type code="3"/>
<transaction-date iso-date="2017-01-01"/>
<value value-date="2017-01-01">20</value>
</transaction>
<transaction>
<transaction-type code="2"/>
<transaction-date iso-date="2017-01-01"/>
<value value-date="2017-01-01">30</value>
</transaction>
<transaction>
<transaction-type code="3"/>
<transaction-date iso-date="2017-01-01"/>
<value value-date="2017-01-01">40</value>
</transaction>
</iati-activity>
<iati-activity>
<iati-identifier>FF-FFA-123123-FF123</iati-identifier>
<reporting-org ref="NO-ORGIDS-10000" type="40"><narrative>Reporting Organisation A</narrative></reporting-org>
<title><narrative>A title</narrative></title>
<description><narrative>A description</narrative></description>
<participating-org role="1" ref="TZ-BRLA-20000"/>
<activity-status code="3"/>
<activity-date type="start-actual" iso-date="2017-01-01"/>
<recipient-country code="TZ" percentage="100"/>
<location>
<location-reach code="1"/>
<location-id vocabulary="G1" code="159239" />
<name>
<narrative>Ilala District, Dar es Salaam, Tanzania TZ</narrative>
</name>
<description>
<narrative>Ilala District is one of three districts in Dar es Salaam</narrative>
</description>
<administrative vocabulary="G1" level="2" code="159239"/>
<point srsName="http://www.opengis.net/def/crs/EPSG/0/4326">
<pos>-6.91805, 39.16254</pos>
</point>
<exactness code="1"/>
<location-class code="1"/>
<feature-designation code="ADM2"/>
</location>
<tag vocabulary="1" code="c_4397">
<narrative xml:lang="en">Livestock</narrative>
</tag>
<transaction>
<transaction-type code="2"/>
<transaction-date iso-date="2017-01-01"/>
<value value-date="2017-01-01">20</value>
</transaction>
</iati-activity>
</iati-activities>
5 changes: 3 additions & 2 deletions cove_iati/lib/api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import json
import shutil

from .iati import common_checks_context_iati, get_file_type
from .iati import common_checks_context_iati, get_file_type, get_tree
from .schema import SchemaIATI
from libcove.lib.converters import convert_spreadsheet
from libcove.config import LibCoveConfig
Expand Down Expand Up @@ -51,8 +51,9 @@ def iati_json_output(output_dir, file, openag=False, orgids=False):
else:
data_file = file

tree = get_tree(data_file)
context = context_api_transform(
common_checks_context_iati(context, output_dir, data_file, file_type,
common_checks_context_iati(context, output_dir, data_file, file_type, tree,
api=True, openag=openag, orgids=orgids)
)

Expand Down
43 changes: 31 additions & 12 deletions cove_iati/lib/iati.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,24 +5,17 @@
import defusedxml.lxml as etree
import lxml.etree
from bdd_tester import bdd_tester
from django.utils.translation import ugettext_lazy as _
from django.utils.html import format_html

from .schema import SchemaIATI
from django.utils.translation import ugettext_lazy as _
from libcove.lib.exceptions import CoveInputDataError
from cove_iati.lib.exceptions import UnrecognisedFileTypeXML
from libcove.lib.tools import ignore_errors
from cove_iati.lib.process_codelists import invalid_embedded_codelist_values

from cove_iati.lib.exceptions import UnrecognisedFileTypeXML
from cove_iati.lib.process_codelists import invalid_embedded_codelist_values
from .schema import SchemaIATI

def common_checks_context_iati(context, upload_dir, data_file, file_type, api=False, openag=False, orgids=False):
'''TODO: this function is trying to do too many things. Separate some
of its logic into smaller functions doing one single thing each.
'''
schema_iati = SchemaIATI()
cell_source_map = {}
validation_errors_path = os.path.join(upload_dir, 'validation_errors-3.json')

def get_tree(data_file):
with open(data_file, 'rb') as fp:
try:
tree = etree.parse(fp)
Expand All @@ -46,6 +39,16 @@ def common_checks_context_iati(context, upload_dir, data_file, file_type, api=Fa
'</span> <strong>Error message:</strong> {}', err)),
'error': format(err)
})
return tree


def common_checks_context_iati(context, upload_dir, data_file, file_type, tree, api=False, openag=False, orgids=False):
'''TODO: this function is trying to do too many things. Separate some
of its logic into smaller functions doing one single thing each.
'''
schema_iati = SchemaIATI()
cell_source_map = {}
validation_errors_path = os.path.join(upload_dir, 'validation_errors-3.json')

if tree.getroot().tag == 'iati-organisations':
schema_path = schema_iati.organisation_schema
Expand Down Expand Up @@ -421,3 +424,19 @@ def get_file_type(file):
return 'csv'
else:
raise UnrecognisedFileTypeXML


def iati_identifier_count(tree):
root = tree.getroot()
identifiers = root.xpath('/iati-activities/iati-activity/iati-identifier/text()')
unique_identifiers = list(set(identifiers))

return len(unique_identifiers)


def organisation_identifier_count(tree):
root = tree.getroot()
identifiers = root.xpath('/iati-organisations/iati-organisation/organisation-identifier/text()')
unique_identifiers = list(set(identifiers))

return len(unique_identifiers)
14 changes: 13 additions & 1 deletion cove_iati/templates/cove_iati/explore.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,20 @@ <h3 class="panel-title panel-title-explore">
{% else %}
{% trans "uploaded " %}
{% endif %}
{% trans "on " %} {{created_date}}.
{% trans "on " %} {{created_date}} at {{created_time}}.
</p>
<br>
<ul class="left-space">
<li>{% blocktrans %}Name of the file: {{file_name}}{% endblocktrans %}</li>
{% if iati_identifiers_count %}
<li>{% blocktrans %}Number of activities: {{iati_identifiers_count}}{% endblocktrans %}</li>
{% endif %}
{% if organisation_identifier_count %}
<li>
{% blocktrans %}Number of organisations: {{organisation_identifier_count}}{% endblocktrans %}
</li>
{% endif %}
</ul>
{% if conversion == 'flattenable' %}
<form method="post">
<button name="flatten" value="true" type="submit" class="btn btn-success btn-sm pull-right">{% trans "Convert to Spreadsheet" %}</button>
Expand Down
Loading

0 comments on commit e7b22a2

Please sign in to comment.