Merge pull request #98 from datosgobar/89-push-dataset-to-ckan

89 push dataset to ckan
datosgobar · Feb 8, 2018 · 1d2c2e8 · 1d2c2e8
2 parents 88ee5e6 + 639374b
commit 1d2c2e8
Show file tree

Hide file tree

Showing 11 changed files with 267 additions and 310 deletions.
diff --git a/docs/MANUAL.md b/docs/MANUAL.md
@@ -94,6 +94,10 @@ Toma los siguientes parámetros:
   - **apikey**: La apikey de un usuario del portal de destino con los permisos para crear el dataset bajo la
   organización pasada como parámetro y los grupos nuevos en caso de que el dataset tenga un super theme que no
   estuviera presente anteriormente en el catálogo de destino.
+
+  **Advertencia**: La función `push_dataset_to_ckan()` sólo garantiza consistencia con los estándares de CKAN. Para
+  mantener una consistencia más estricta dentro del catálogo a federar, es necesario validar los datos antes de pasarlos
+  a la función. 
 
 ## Uso
 

diff --git a/pydatajson/ckan_utils.py b/pydatajson/ckan_utils.py
@@ -29,7 +29,7 @@ def map_dataset_to_package(dataset, catalog_id):
     append_attribute_to_extra(package, dataset, 'accrualPeriodicity')
 
     distributions = dataset['distribution']
-    package['resources'] = map_distributions_to_resources(distributions, package['id'])
+    package['resources'] = map_distributions_to_resources(distributions, catalog_id)
 
     super_themes = dataset['superTheme']
     package['groups'] = [{'name': re.sub(r'[^a-z-_]+', '', super_theme.lower())} for super_theme in super_themes]
@@ -77,12 +77,12 @@ def convert_iso_string_to_utc(date_string):
     return utc_date_time.isoformat()
 
 
-def map_distributions_to_resources(distributions, package_id):
+def map_distributions_to_resources(distributions, catalog_id):
     resources = []
     for distribution in distributions:
         resource = dict()
 #       Obligatorios
-        resource['id'] = package_id+'_'+distribution['identifier']
+        resource['id'] = catalog_id + '_' + distribution['identifier']
         resource['name'] = distribution['title']
         resource['url'] = distribution['downloadURL']
         resource['created'] = convert_iso_string_to_utc(distribution['issued'])

diff --git a/pydatajson/federation.py b/pydatajson/federation.py
@@ -23,8 +23,6 @@ def push_dataset_to_ckan(catalog, catalog_id, owner_org, dataset_origin_identifi
         Returns:
             str: El id del dataset en el catálogo de destino.
     """
-    if not catalog.is_valid_catalog():
-        raise ValueError('The catalog is invalid')
     dataset = catalog.get_dataset(dataset_origin_identifier)
     ckan_portal = RemoteCKAN(portal_url, apikey=apikey)
 

diff --git a/tests/cassetes/ckan_integration/tearDown.yaml b/tests/cassetes/ckan_integration/tearDown.yaml
@@ -12,13 +12,13 @@ interactions:
     uri: http://181.209.63.239/api/action/package_list
   response:
     body: {string: !!python/unicode '{"help": "http://181.209.63.239/api/3/action/help_show?name=package_list",
-        "success": true, "result": ["dataset-de-prueba", "sistemadecontratacioneselectrnicas"]}'}
+        "success": true, "result": ["dataset-de-prueba", "fruta", "sistemadecontratacioneselectrnicas"]}'}
     headers:
       cache-control: [no-cache]
       connection: [keep-alive]
-      content-length: ['162']
+      content-length: ['171']
       content-type: [application/json;charset=utf-8]
-      date: ['Wed, 07 Feb 2018 15:48:50 GMT']
+      date: ['Thu, 08 Feb 2018 18:34:02 GMT']
       pragma: [no-cache]
       server: [nginx/1.11.9]
     status: {code: 200, message: OK}
@@ -41,7 +41,7 @@ interactions:
       connection: [keep-alive]
       content-length: ['108']
       content-type: [application/json;charset=utf-8]
-      date: ['Wed, 07 Feb 2018 15:48:50 GMT']
+      date: ['Thu, 08 Feb 2018 18:34:02 GMT']
       pragma: [no-cache]
       server: [nginx/1.11.9]
     status: {code: 200, message: OK}
@@ -58,13 +58,13 @@ interactions:
     uri: http://181.209.63.239/api/action/package_list
   response:
     body: {string: !!python/unicode '{"help": "http://181.209.63.239/api/3/action/help_show?name=package_list",
-        "success": true, "result": ["dataset-de-prueba", "sistemadecontratacioneselectrnicas"]}'}
+        "success": true, "result": ["dataset-de-prueba", "fruta", "sistemadecontratacioneselectrnicas"]}'}
     headers:
       cache-control: [no-cache]
       connection: [keep-alive]
-      content-length: ['162']
+      content-length: ['171']
       content-type: [application/json;charset=utf-8]
-      date: ['Wed, 07 Feb 2018 15:48:51 GMT']
+      date: ['Thu, 08 Feb 2018 18:34:03 GMT']
       pragma: [no-cache]
       server: [nginx/1.11.9]
     status: {code: 200, message: OK}
@@ -87,7 +87,7 @@ interactions:
       connection: [keep-alive]
       content-length: ['108']
       content-type: [application/json;charset=utf-8]
-      date: ['Wed, 07 Feb 2018 15:48:51 GMT']
+      date: ['Thu, 08 Feb 2018 18:34:03 GMT']
       pragma: [no-cache]
       server: [nginx/1.11.9]
     status: {code: 200, message: OK}
@@ -104,13 +104,13 @@ interactions:
     uri: http://181.209.63.239/api/action/package_list
   response:
     body: {string: !!python/unicode '{"help": "http://181.209.63.239/api/3/action/help_show?name=package_list",
-        "success": true, "result": ["dataset-de-prueba", "sistemadecontratacioneselectrnicas"]}'}
+        "success": true, "result": ["dataset-de-prueba", "fruta", "sistemadecontratacioneselectrnicas"]}'}
     headers:
       cache-control: [no-cache]
       connection: [keep-alive]
-      content-length: ['162']
+      content-length: ['171']
       content-type: [application/json;charset=utf-8]
-      date: ['Wed, 07 Feb 2018 15:48:52 GMT']
+      date: ['Thu, 08 Feb 2018 18:34:04 GMT']
       pragma: [no-cache]
       server: [nginx/1.11.9]
     status: {code: 200, message: OK}
@@ -133,30 +133,7 @@ interactions:
       connection: [keep-alive]
       content-length: ['108']
       content-type: [application/json;charset=utf-8]
-      date: ['Wed, 07 Feb 2018 15:48:52 GMT']
-      pragma: [no-cache]
-      server: [nginx/1.11.9]
-    status: {code: 200, message: OK}
-- request:
-    body: !!python/unicode '{}'
-    headers:
-      Accept: ['*/*']
-      Accept-Encoding: ['gzip, deflate']
-      Connection: [keep-alive]
-      Content-Length: ['2']
-      Content-Type: [application/json]
-      User-Agent: ['ckanapi/4.0 (+https://github.com/ckan/ckanapi)']
-    method: POST
-    uri: http://181.209.63.239/api/action/package_list
-  response:
-    body: {string: !!python/unicode '{"help": "http://181.209.63.239/api/3/action/help_show?name=package_list",
-        "success": true, "result": ["dataset-de-prueba"]}'}
-    headers:
-      cache-control: [no-cache]
-      connection: [keep-alive]
-      content-length: ['124']
-      content-type: [application/json;charset=utf-8]
-      date: ['Wed, 07 Feb 2018 15:48:52 GMT']
+      date: ['Thu, 08 Feb 2018 18:34:04 GMT']
       pragma: [no-cache]
       server: [nginx/1.11.9]
     status: {code: 200, message: OK}
@@ -174,13 +151,13 @@ interactions:
   response:
     body: {string: !!python/unicode '{"help": "http://181.209.63.239/api/3/action/help_show?name=package_list",
         "success": true, "result": ["basededatoslegislativosinfoleg", "dataset-de-prueba",
-        "sistemadecontratacioneselectrnicas"]}'}
+        "fruta", "sistemadecontratacioneselectrnicas"]}'}
     headers:
       cache-control: [no-cache]
       connection: [keep-alive]
-      content-length: ['196']
+      content-length: ['205']
       content-type: [application/json;charset=utf-8]
-      date: ['Wed, 07 Feb 2018 15:48:55 GMT']
+      date: ['Thu, 08 Feb 2018 18:34:06 GMT']
       pragma: [no-cache]
       server: [nginx/1.11.9]
     status: {code: 200, message: OK}
@@ -203,7 +180,7 @@ interactions:
       connection: [keep-alive]
       content-length: ['108']
       content-type: [application/json;charset=utf-8]
-      date: ['Wed, 07 Feb 2018 15:48:56 GMT']
+      date: ['Thu, 08 Feb 2018 18:34:07 GMT']
       pragma: [no-cache]
       server: [nginx/1.11.9]
     status: {code: 200, message: OK}
@@ -226,7 +203,7 @@ interactions:
       connection: [keep-alive]
       content-length: ['108']
       content-type: [application/json;charset=utf-8]
-      date: ['Wed, 07 Feb 2018 15:48:56 GMT']
+      date: ['Thu, 08 Feb 2018 18:34:07 GMT']
       pragma: [no-cache]
       server: [nginx/1.11.9]
     status: {code: 200, message: OK}

diff --git a/tests/cassetes/ckan_integration/test_dataset_is_created_correctly.yaml b/tests/cassetes/ckan_integration/test_dataset_is_created_correctly.yaml
@@ -18,7 +18,7 @@ interactions:
       connection: [keep-alive]
       content-length: ['127']
       content-type: [application/json;charset=utf-8]
-      date: ['Wed, 07 Feb 2018 15:48:49 GMT']
+      date: ['Thu, 08 Feb 2018 18:34:01 GMT']
       pragma: [no-cache]
       server: [nginx/1.11.9]
     status: {code: 200, message: OK}
@@ -110,7 +110,7 @@ interactions:
       connection: [keep-alive]
       content-length: ['5727']
       content-type: [application/json;charset=utf-8]
-      date: ['Wed, 07 Feb 2018 15:48:49 GMT']
+      date: ['Thu, 08 Feb 2018 18:34:01 GMT']
       pragma: [no-cache]
       server: [nginx/1.11.9]
     status: {code: 200, message: OK}
@@ -158,13 +158,13 @@ interactions:
       {\"type\": \"string\", \"description\": \"Clase de la convocatoria.\", \"title\":
       \"clase_convocatoria\"}, {\"type\": \"string\", \"description\": \"Objeto/objetivo
       de la convocatoria\", \"title\": \"objeto_convocatoria\"}]", "created": "2016-04-14T22:48:05.433640",
-      "last_modified": "2016-04-19T22:48:05.433640", "size": 5120, "id": "datosargentina_99db6631-d1c9-470b-a73e-c62daa32c777_1.1",
+      "last_modified": "2016-04-19T22:48:05.433640", "size": 5120, "id": "datosargentina_1.1",
       "name": "Convocatorias abiertas durante el a\u00f1o 2015"}]}'
     headers:
       Accept: ['*/*']
       Accept-Encoding: ['gzip, deflate']
       Connection: [keep-alive]
-      Content-Length: ['4033']
+      Content-Length: ['3996']
       Content-Type: [application/json]
       User-Agent: ['ckanapi/4.0 (+https://github.com/ckan/ckanapi)']
     method: POST
@@ -178,7 +178,7 @@ interactions:
       connection: [keep-alive]
       content-length: ['195']
       content-type: [application/json;charset=utf-8]
-      date: ['Wed, 07 Feb 2018 15:48:49 GMT']
+      date: ['Thu, 08 Feb 2018 18:34:01 GMT']
       pragma: [no-cache]
       server: [nginx/1.11.9]
     status: {code: 404, message: Not Found}
@@ -226,13 +226,13 @@ interactions:
       {\"type\": \"string\", \"description\": \"Clase de la convocatoria.\", \"title\":
       \"clase_convocatoria\"}, {\"type\": \"string\", \"description\": \"Objeto/objetivo
       de la convocatoria\", \"title\": \"objeto_convocatoria\"}]", "created": "2016-04-14T22:48:05.433640",
-      "last_modified": "2016-04-19T22:48:05.433640", "size": 5120, "id": "datosargentina_99db6631-d1c9-470b-a73e-c62daa32c777_1.1",
+      "last_modified": "2016-04-19T22:48:05.433640", "size": 5120, "id": "datosargentina_1.1",
       "name": "Convocatorias abiertas durante el a\u00f1o 2015"}]}'
     headers:
       Accept: ['*/*']
       Accept-Encoding: ['gzip, deflate']
       Connection: [keep-alive]
-      Content-Length: ['4033']
+      Content-Length: ['3996']
       Content-Type: [application/json]
       User-Agent: ['ckanapi/4.0 (+https://github.com/ckan/ckanapi)']
     method: POST
@@ -244,7 +244,7 @@ interactions:
         Administrativa. Oficina Nacional de Contrataciones. Direcci\u00f3n de Compras
         Electr\u00f3nicas.", "relationships_as_object": [], "private": false, "maintainer_email":
         "onc-compraselectronicas@modernizacion.gob.ar", "num_tags": 4, "id": "datosargentina_99db6631-d1c9-470b-a73e-c62daa32c777",
-        "metadata_created": "2018-02-07T15:48:49.919946", "metadata_modified": "2018-02-07T15:48:49.943976",
+        "metadata_created": "2018-02-08T18:34:01.823992", "metadata_modified": "2018-02-08T18:34:01.844425",
         "author": "Ministerio de Modernizaci\u00f3n. Secretar\u00eda de Modernizaci\u00f3n
         Administrativa. Oficina Nacional de Contrataciones", "author_email": "onc@modernizacion.gob.ar",
         "state": "active", "version": null, "creator_user_id": "824dc36b-38b0-448d-bcb2-dda225089ba2",
@@ -272,15 +272,15 @@ interactions:
         \"clase_convocatoria\"}, {\"type\": \"string\", \"description\": \"Objeto/objetivo
         de la convocatoria\", \"title\": \"objeto_convocatoria\"}]", "package_id":
         "datosargentina_99db6631-d1c9-470b-a73e-c62daa32c777", "webstore_last_updated":
-        null, "datastore_active": false, "id": "datosargentina_99db6631-d1c9-470b-a73e-c62daa32c777_1.1",
-        "size": "5120", "state": "active", "accessURL": "http://datos.gob.ar/dataset/sistema-de-contrataciones-electronicas-argentina-compra/archivo/fa3603b3-0af7-43cc-9da9-90a512217d8a",
+        null, "datastore_active": false, "id": "datosargentina_1.1", "size": "5120",
+        "state": "active", "accessURL": "http://datos.gob.ar/dataset/sistema-de-contrataciones-electronicas-argentina-compra/archivo/fa3603b3-0af7-43cc-9da9-90a512217d8a",
         "hash": "", "description": "Listado de las convocatorias abiertas durante
         el a\u00f1o 2015 en el sistema de contrataciones electr\u00f3nicas", "format":
         "CSV", "last_modified": "2016-04-19T22:48:05.433640", "url_type": null, "mimetype":
         "text/csv", "cache_url": null, "name": "Convocatorias abiertas durante el
         a\u00f1o 2015", "created": "2016-04-14T22:48:05.433640", "url": "http://186.33.211.253/dataset/99db6631-d1c9-470b-a73e-c62daa32c420/resource/4b7447cb-31ff-4352-96c3-589d212e1cc9/download/convocatorias-abiertas-anio-2015.csv",
         "webstore_url": null, "mimetype_inner": null, "position": 0, "revision_id":
-        "5e629b7f-cd66-4178-b0e4-c2bb996494b4", "resource_type": null}], "num_resources":
+        "b9d77885-f722-452d-9f9a-fe4d23308ccd", "resource_type": null}], "num_resources":
         1, "tags": [{"vocabulary_id": null, "state": "active", "display_name": "bienes",
         "id": "7cacc193-5776-46b3-8f31-ccc52042248f", "name": "bienes"}, {"vocabulary_id":
         null, "state": "active", "display_name": "compras", "id": "27e042d0-87b8-416e-9745-8b504dc182e0",
@@ -289,7 +289,7 @@ interactions:
         {"vocabulary_id": null, "state": "active", "display_name": "convocatorias",
         "id": "a7cf30d4-d35a-433a-914f-47727d61b6f7", "name": "convocatorias"}], "groups":
         [{"display_name": "econ", "description": "", "image_display_url": "", "title":
-        "", "id": "bde6ea2b-8d4c-487a-a971-3ba906bebab3", "name": "econ"}], "license_id":
+        "", "id": "ddf819de-dfd5-474d-9335-9a2d74c28f64", "name": "econ"}], "license_id":
         "notspecified", "relationships_as_subject": [], "organization": {"description":
         "", "created": "2018-02-07T15:37:27.733652", "title": "oficina-de-muestra",
         "name": "oficina-de-muestra", "is_organization": true, "state": "active",
@@ -304,13 +304,13 @@ interactions:
         "[\"spa\"]"}, {"key": "modified", "value": "2016-04-19T19:48:05.433640-03:00"},
         {"key": "spatial", "value": "ARG"}, {"key": "super_theme", "value": "[\"econ\"]"},
         {"key": "temporal", "value": "2015-01-01/2015-12-31"}], "title": "Sistema
-        de contrataciones electr\u00f3nicas", "revision_id": "5e629b7f-cd66-4178-b0e4-c2bb996494b4"}}'}
+        de contrataciones electr\u00f3nicas", "revision_id": "b9d77885-f722-452d-9f9a-fe4d23308ccd"}}'}
     headers:
       cache-control: [no-cache]
       connection: [keep-alive]
-      content-length: ['5865']
+      content-length: ['5828']
       content-type: [application/json;charset=utf-8]
-      date: ['Wed, 07 Feb 2018 15:48:50 GMT']
+      date: ['Thu, 08 Feb 2018 18:34:02 GMT']
       pragma: [no-cache]
       server: [nginx/1.11.9]
     status: {code: 200, message: OK}