Merge 8fba599 into a57ad8d

cove/lib/common.py

@@ -33,6 +33,8 @@
 
 
 class CustomJsonrefLoader(jsonref.JsonLoader):
+    '''This ref loader is only for use with the jsonref library
+    and NOT jsonschema.'''
     def __init__(self, **kwargs):
         self.schema_url = kwargs.pop('schema_url', None)
         super().__init__(**kwargs)
@@ -48,19 +50,30 @@ def get_remote_json(self, uri, **kwargs):
 
 
 class CustomRefResolver(RefResolver):
+    '''This RefResolver is only for use with the jsonschema library'''
     def __init__(self, *args, **kw):
+        # this is the name of the json file that you want replaced i.e release-schema.json
+        self.file_schema_name = kw.pop('file_schema_name', '')
+        # the path on the disk of the file you want to replace the ref
         self.schema_file = kw.pop('schema_file', None)
+        # the url of the path to the schema. i.e http://standard.open-contracting.org/schema/1__1__1/
+        # the name of the schema file is appended to this to make the full url.
+        # this is ignored when you supply a file
         self.schema_url = kw.pop('schema_url', '')
         super().__init__(*args, **kw)
 
     def resolve_remote(self, uri):
-        schema_name = self.schema_file or uri.split('/')[-1]
-        uri = urljoin(self.schema_url, schema_name)
+        schema_name = uri.split('/')[-1]
+        if self.schema_file and self.file_schema_name == schema_name:
+            uri = self.schema_file
+        else:
+            uri = urljoin(self.schema_url, schema_name)
+
         document = self.store.get(uri)
 
         if document:
             return document
-        if self.schema_url.startswith("http"):
+        if uri.startswith("http"):
             return super().resolve_remote(uri)
         else:
             with open(uri) as schema_file:
@@ -87,7 +100,7 @@ def release_pkg_schema_str(self):
 
     @property
     def _release_schema_obj(self):
-        return json.loads(self.release_schema_str)
+        return json.loads(self.release_schema_str, object_pairs_hook=OrderedDict)
 
     @property
     def _release_pkg_schema_obj(self):
@@ -302,7 +315,9 @@ def get_schema_validation_errors(json_data, schema_obj, schema_name, cell_src_ma
         format_checker.checkers.update(extra_checkers)
 
     if getattr(schema_obj, 'extended', None):
-        resolver = CustomRefResolver('', pkg_schema_obj, schema_file=schema_obj.extended_schema_file)
+        resolver = CustomRefResolver('', pkg_schema_obj, schema_url=schema_obj.schema_host,
+                                     schema_file=schema_obj.extended_schema_file,
+                                     file_schema_name=schema_obj.release_schema_name)
     else:
         resolver = CustomRefResolver('', pkg_schema_obj, schema_url=schema_obj.schema_host)
 

cove/lib/converters.py

@@ -112,7 +112,7 @@ def convert_spreadsheet(upload_dir, upload_url, file_name, file_type, schema_url
     return context
 
 
-def convert_json(upload_dir, upload_url, file_name, schema_url, replace=False, request=None, flatten=False, cache=True):
+def convert_json(upload_dir, upload_url, file_name, schema_url=None, replace=False, request=None, flatten=False, cache=True, xml=False):
     context = {}
     converted_path = os.path.join(upload_dir, 'flattened')
 
@@ -124,6 +124,10 @@ def convert_json(upload_dir, upload_url, file_name, schema_url, replace=False, r
         schema=schema_url
     )
 
+    if xml:
+        flatten_kwargs['xml'] = True
+        flatten_kwargs['id_name'] = config.get('id_name', 'id')
+
     try:
         conversion_warning_cache_path = os.path.join(upload_dir, 'conversion_warning_messages.json')
         conversion_exists = os.path.exists(converted_path + '.xlsx')

cove_iati/templates/cove_iati/explore.html

@@ -34,6 +34,12 @@ <h3 class="panel-title panel-title-explore">
              {% endif %}
              {% trans "on " %} {{created_date}}.
            </p>
+          {% if conversion == 'flattenable' %}
+            <form method="post">
+              <button name="flatten" value="true" type="submit" class="btn btn-success btn-sm pull-right">{% trans "Convert to Spreadsheet" %}</button>
+              {% csrf_token %}
+            </form>
+          {% endif %}
         </div>
       </div>
     </div>

cove_iati/views.py

@@ -4,7 +4,7 @@
 from django import forms
 from django.utils.translation import ugettext_lazy as _
 
-from cove.lib.converters import convert_spreadsheet
+from cove.lib.converters import convert_spreadsheet, convert_json
 from cove.lib.exceptions import cove_web_input_error
 from cove.input.models import SuppliedData
 from cove.input.views import data_input
@@ -66,6 +66,8 @@ def explore_iati(request, pk):
         sort_iati_xml_file(context['converted_path'], context['converted_path'])
     else:
         data_file = db_data.original_file.file.name
+        context.update(convert_json(db_data.upload_dir(), db_data.upload_url(), db_data.original_file.file.name,
+                       request=request, flatten=request.POST.get('flatten'), xml=True))
 
     context = common_checks_context_iati(context, db_data.upload_dir(), data_file, file_type)
     context['first_render'] = not db_data.rendered

cove_ocds/fixtures/SOURCES

@@ -19,8 +19,8 @@ tenders_releases_2_releases_with_deprecated_fields.json
 Source: Same data as tenders_releases_2_releases.json, but edited such that it contain deprecated fields.
 License: http://data.gc.ca/eng/open-government-licence-canada
 
-tenders_releases_1_release_with_extensions_version_1_1.json
-Source: Same data as tenders_releases_2_releases.json, but edited such that it contains extensions, and is missing some fields.
+tenders_releases_1_release_with_extensions_1_1.json
+Source: Same data as tenders_releases_2_releases.json, but edited such that it contains extensions, and is missing some fields, and has some additional fields added.
 License: http://data.gc.ca/eng/open-government-licence-canada
 
 tenders_releases_1_release_with_extensions_new_layout.json
@@ -35,6 +35,14 @@ tenders_releases_1_release_with_all_invalid_extensions.json
 Source: Same data as tenders_releases_2_releases.json, but edited such that it contains valid and invalid extensions.
 License: http://data.gc.ca/eng/open-government-licence-canada
 
+tenders_1_release_with_extensions_1_1_missing_party_scale.json
+Source: Same data as tenders_releases_2_releases.json, but edited such that it contains valid extensions, and with some additional fields added.
+License: http://data.gc.ca/eng/open-government-licence-canada
+
+cove_ocds/fixtures/tenders_records_1_record_with_invalid_extensions.json
+Source: Same data as tenders_releases_2_releases.json, but edited to be a record, and with some additional fields added.
+License: http://data.gc.ca/eng/open-government-licence-canada
+
 ocds_release_nulls.json
 Adapted from://github.com/open-contracting/sample-data/blob/master/blank-template/release-template-1__0__0.json 
 

cove_ocds/fixtures/tenders_1_release_with_extensions_1_1_missing_party_scale.json

@@ -0,0 +1,82 @@
+{
+    "publisher": {
+        "scheme": null, 
+        "name": "Buyandsell.gc.ca", 
+        "uri": "https://buyandsell.gc.ca", 
+        "uid": null
+    },
+    "extensions": [
+        "https://raw.githubusercontent.com/open-contracting/ocds_metrics_extension/master/extension.json",
+        "https://raw.githubusercontent.com/open-contracting/ocds_extension_parties/master/extension.json"
+    ],
+    "version": "1.1",
+    "releases": [
+        {
+            "date": "2014-03-25T00:00:00.00Z", 
+            "language": "English", 
+            "id": "Buyandsell.gc.ca-2014-11-07-89f689cd-e784-4374-bb17-94144679d46f", 
+            "ocid": "PW-14-00627094", 
+            "initiationType": "tender", 
+            "parties": [{
+                "id": "GB-COH-0000",
+                "details": {
+                    "scale": "sme"
+                }
+            }],
+            "tender": {
+                "documents": [
+                    {
+                        "url": "https://buyandsell.gc.ca/cds/public/2014/03/25/3a76d19e61825db7939e14afaa2002e3/14-2015_itt.pdf", 
+                        "id": "https://buyandsell.gc.ca/cds/public/2014/03/25/3a76d19e61825db7939e14afaa2002e3/14-2015_itt.pdf"
+                    }, 
+                    {
+                        "url": "https://buyandsell.gc.ca/cds/public/2014/03/25/a5d07c1166142b8b2f4183d7a5363fec/14-2015_itt_-_f.pdf", 
+                        "id": "https://buyandsell.gc.ca/cds/public/2014/03/25/a5d07c1166142b8b2f4183d7a5363fec/14-2015_itt_-_f.pdf"
+                    }, 
+                    {
+                        "url": "https://buyandsell.gc.ca/cds/public/2014/04/04/ea5d0f03cfc6ea201b5b6df8794dbb3c/14-2015_itt_amd1.pdf", 
+                        "id": "https://buyandsell.gc.ca/cds/public/2014/04/04/ea5d0f03cfc6ea201b5b6df8794dbb3c/14-2015_itt_amd1.pdf"
+                    }, 
+                    {
+                        "url": "https://buyandsell.gc.ca/cds/public/2014/04/04/ac9afb3b664a28d519062f2b2a4cbceb/14-2015_itt_amd1-f.pdf", 
+                        "id": "https://buyandsell.gc.ca/cds/public/2014/04/04/ac9afb3b664a28d519062f2b2a4cbceb/14-2015_itt_amd1-f.pdf"
+                    }, 
+                    {
+                        "url": "https://buyandsell.gc.ca/cds/public/2014/04/11/f3f3993b4f6f260e057d0cb95f43741d/14-2015_itt_amd2.pdf", 
+                        "id": "https://buyandsell.gc.ca/cds/public/2014/04/11/f3f3993b4f6f260e057d0cb95f43741d/14-2015_itt_amd2.pdf"
+                    }, 
+                    {
+                        "url": "https://buyandsell.gc.ca/cds/public/2014/04/11/69359ba194d3f01f4f13cb9673b85071/14-2015_itt_amd2-f.pdf", 
+                        "id": "https://buyandsell.gc.ca/cds/public/2014/04/11/69359ba194d3f01f4f13cb9673b85071/14-2015_itt_amd2-f.pdf"
+                    }
+                ], 
+                "tenderPeriod": {
+                    "endDate": "2014-04-15T14:00:00.00Z"
+                }, 
+                "items": [
+                    {
+                        "id": "1", 
+                        "classification": {
+                            "scheme": "GSIN", 
+                            "description": "F029: Other Animal Care and Control Services"
+                        }, 
+                        "description": "Canine Control of the Canada Geese (14-2015)"
+                    }
+                ], 
+                "awardCriteriaDetails": null, 
+                "procuringEntity": {
+                    "name": "Agriculture & Agrifood Canada",
+                    "name_fr": "Agriculture & Agrifood Canada"
+                }, 
+                "id": "PW-14-00627094", 
+                "methodRationale": "Open"
+            }, 
+            "tag": ["tender"], 
+            "buyer": {
+            }
+        }
+    ], 
+    "uri": "https://github.com/open-contracting/sample-data/blob/master/buyandsell/ocds_data/tender_releases.json.zip", 
+    "license": "http://data.gc.ca/eng/open-government-licence-canada", 
+    "publishedDate": "2014-11-07T00:00:00.00Z"
+}

cove_ocds/fixtures/tenders_records_1_record_with_invalid_extensions.json

@@ -5,6 +5,7 @@
         "uri": "https://buyandsell.gc.ca", 
         "uid": null
     },
+    "version": "1.1",
     "extensions": [
         "https://raw.githubusercontent.com/open-contracting/ocds_metrics_extension/master/extension.json",
         "https://raw.githubusercontent.com/open-contracting/ocds_extension_parties/master/extension.json",
@@ -15,12 +16,18 @@
     ],
     "records": [
         {
-            "compiledRelease": {
+            "releases": [{
                 "date": "2014-03-25T00:00:00.00Z", 
                 "language": "English", 
                 "id": "Buyandsell.gc.ca-2014-11-07-89f689cd-e784-4374-bb17-94144679d46f", 
                 "ocid": "PW-14-00627094", 
                 "initiationType": "tender", 
+                "parties": [{
+                    "id": "GB-COH-0000",
+                    "details": {
+                        "scale": "mooo"
+                    }
+                }],
                 "tender": {
                     "documents": [
                         {
@@ -47,7 +54,73 @@
                             "url": "https://buyandsell.gc.ca/cds/public/2014/04/11/69359ba194d3f01f4f13cb9673b85071/14-2015_itt_amd2-f.pdf", 
                             "id": "https://buyandsell.gc.ca/cds/public/2014/04/11/69359ba194d3f01f4f13cb9673b85071/14-2015_itt_amd2-f.pdf"
                         }
+                    ],
+                    "tenderPeriod": {
+                        "endDate": "2014-04-15T14:00:00.00Z"
+                    }, 
+                    "items": [
+                        {
+                            "id": "1", 
+                            "classification": {
+                                "scheme": "GSIN", 
+                                "description": "F029: Other Animal Care and Control Services"
+                            }, 
+                            "description": "Canine Control of the Canada Geese (14-2015)"
+                        }
                     ], 
+                    "awardCriteriaDetails": null, 
+                    "procuringEntity": {
+                        "name": "Agriculture & Agrifood Canada",
+                        "name_fr": "Agriculture & Agrifood Canada"
+                    }, 
+                    "id": "PW-14-00627094", 
+                    "methodRationale": "Open",
+                    "targets": [{"id": "an id", "titles": "not a title"}]
+                }, 
+                "tag": ["tender"], 
+                "buyer": {
+                    "name": "Agriculture & Agrifood Canada"
+                }
+            }],
+            "compiledRelease": {
+                "date": "2014-03-25T00:00:00.00Z", 
+                "language": "English", 
+                "id": "Buyandsell.gc.ca-2014-11-07-89f689cd-e784-4374-bb17-94144679d46f", 
+                "ocid": "PW-14-00627094", 
+                "initiationType": "tender", 
+                "parties": [{
+                    "id": "GB-COH-0000",
+                    "details": {
+                        "scale": "mooo"
+                    }
+                }],
+                "tender": {
+                    "documents": [
+                        {
+                            "url": "https://buyandsell.gc.ca/cds/public/2014/03/25/3a76d19e61825db7939e14afaa2002e3/14-2015_itt.pdf", 
+                            "id": "https://buyandsell.gc.ca/cds/public/2014/03/25/3a76d19e61825db7939e14afaa2002e3/14-2015_itt.pdf"
+                        }, 
+                        {
+                            "url": "https://buyandsell.gc.ca/cds/public/2014/03/25/a5d07c1166142b8b2f4183d7a5363fec/14-2015_itt_-_f.pdf", 
+                            "id": "https://buyandsell.gc.ca/cds/public/2014/03/25/a5d07c1166142b8b2f4183d7a5363fec/14-2015_itt_-_f.pdf"
+                        }, 
+                        {
+                            "url": "https://buyandsell.gc.ca/cds/public/2014/04/04/ea5d0f03cfc6ea201b5b6df8794dbb3c/14-2015_itt_amd1.pdf", 
+                            "id": "https://buyandsell.gc.ca/cds/public/2014/04/04/ea5d0f03cfc6ea201b5b6df8794dbb3c/14-2015_itt_amd1.pdf"
+                        }, 
+                        {
+                            "url": "https://buyandsell.gc.ca/cds/public/2014/04/04/ac9afb3b664a28d519062f2b2a4cbceb/14-2015_itt_amd1-f.pdf", 
+                            "id": "https://buyandsell.gc.ca/cds/public/2014/04/04/ac9afb3b664a28d519062f2b2a4cbceb/14-2015_itt_amd1-f.pdf"
+                        }, 
+                        {
+                            "url": "https://buyandsell.gc.ca/cds/public/2014/04/11/f3f3993b4f6f260e057d0cb95f43741d/14-2015_itt_amd2.pdf", 
+                            "id": "https://buyandsell.gc.ca/cds/public/2014/04/11/f3f3993b4f6f260e057d0cb95f43741d/14-2015_itt_amd2.pdf"
+                        }, 
+                        {
+                            "url": "https://buyandsell.gc.ca/cds/public/2014/04/11/69359ba194d3f01f4f13cb9673b85071/14-2015_itt_amd2-f.pdf", 
+                            "id": "https://buyandsell.gc.ca/cds/public/2014/04/11/69359ba194d3f01f4f13cb9673b85071/14-2015_itt_amd2-f.pdf"
+                        }
+                    ],
                     "tenderPeriod": {
                         "endDate": "2014-04-15T14:00:00.00Z"
                     }, 
@@ -67,7 +140,8 @@
                         "name_fr": "Agriculture & Agrifood Canada"
                     }, 
                     "id": "PW-14-00627094", 
-                    "methodRationale": "Open"
+                    "methodRationale": "Open",
+                    "targets": [{"id": "an id", "titles": "not a title"}]
                 }, 
                 "tag": ["tender"], 
                 "buyer": {

cove_ocds/fixtures/tenders_releases_1_release_with_extensions_version_1_1.json → cove_ocds/fixtures/tenders_releases_1_release_with_extensions_1_1.json

@@ -18,6 +18,12 @@
             "id": "Buyandsell.gc.ca-2014-11-07-89f689cd-e784-4374-bb17-94144679d46f", 
             "ocid": "PW-14-00627094", 
             "initiationType": "tender", 
+            "parties": {
+                "id": "GB-COH-0000",
+                "details": {
+                    "scale": "sme"
+                }
+            },
             "tender": {
                 "documents": [
                     {
@@ -44,7 +50,7 @@
                         "url": "https://buyandsell.gc.ca/cds/public/2014/04/11/69359ba194d3f01f4f13cb9673b85071/14-2015_itt_amd2-f.pdf", 
                         "id": "https://buyandsell.gc.ca/cds/public/2014/04/11/69359ba194d3f01f4f13cb9673b85071/14-2015_itt_amd2-f.pdf"
                     }
-                ], 
+                ],
                 "tenderPeriod": {
                     "endDate": "2014-04-15T14:00:00.00Z"
                 }, 

cove_ocds/lib/schema.py

@@ -101,17 +101,16 @@ def get_release_schema_obj(self, deref=False):
         return release_schema_obj
 
     def get_release_pkg_schema_obj(self, deref=False):
-        package_schema_obj = self._release_pkg_schema_obj
+        package_schema_obj = deepcopy(self._release_pkg_schema_obj)
         if deref:
-            deref_release_schema_obj = self.get_release_schema_obj(deref=True)
             if self.extended:
-                package_schema_obj = deepcopy(self._release_pkg_schema_obj)
+                deref_release_schema_obj = self.get_release_schema_obj(deref=True)
                 package_schema_obj['properties']['releases']['items'] = {}
                 release_pkg_schema_str = json.dumps(package_schema_obj)
                 package_schema_obj = self.deref_schema(release_pkg_schema_str)
                 package_schema_obj['properties']['releases']['items'].update(deref_release_schema_obj)
             else:
-                package_schema_obj = self.deref_schema(self.release_pkg_schema_str)
+                return self.deref_schema(self.release_pkg_schema_str)
         return package_schema_obj
 
     def apply_extensions(self, schema_obj):
@@ -204,8 +203,13 @@ def _record_pkg_schema_obj(self):
 
     def get_record_pkg_schema_obj(self, deref=False):
         if deref:
-            return self.deref_schema(self.record_pkg_schema_str)
-        return self._record_pkg_schema_obj
+            deref_package_schema = self.deref_schema(self.record_pkg_schema_str)
+            if self.extended:
+                deref_release_schema_obj = self.get_release_schema_obj(deref=True)
+                deref_package_schema['properties']['records']['items']['properties']['compiledRelease'] = deref_release_schema_obj
+                deref_package_schema['properties']['records']['items']['properties']['releases']['oneOf'][1] = deref_release_schema_obj
+            return deref_package_schema
+        return deepcopy(self._record_pkg_schema_obj)
 
     def get_record_pkg_schema_fields(self):
         return set(schema_dict_fields_generator(self.get_record_pkg_schema_obj(deref=True)))