Merge branch 'master' into 90-mover-superTheme-a-theme

docs/MANUAL.md

@@ -96,6 +96,9 @@ Toma los siguientes parámetros:
   organización pasada como parámetro.
   - **demote_superThemes** (opcional, default: True):Si está en true, los ids de los themes del dataset, se escriben
   como groups de CKAN.
+  - **demote_themes** (opcional, default: True): Si está en true, los labels de los themes del dataset, se escriben como
+  tags de CKAN; sino,se pasan como grupo.
+
 
   Retorna el id en el nodo de destino del dataset federado.
 

pydatajson/ckan_utils.py

@@ -14,7 +14,8 @@ def append_attribute_to_extra(package, dataset, attribute, serialize=False):
         package['extras'].append({'key': attribute, 'value': value})
 
 
-def map_dataset_to_package(dataset, catalog_id, demote_superThemes=True):
+def map_dataset_to_package(dataset, catalog_id, owner_org, theme_taxonomy,
+                           demote_superThemes=True, demote_themes=True):
     package = dict()
     package['extras'] = []
 #   Obligatorios
@@ -24,6 +25,7 @@ def map_dataset_to_package(dataset, catalog_id, demote_superThemes=True):
     package['private'] = False
     package['notes'] = dataset['description']
     package['author'] = dataset['publisher']['name']
+    package['owner_org'] = owner_org
 
     append_attribute_to_extra(package, dataset, 'issued')
     append_attribute_to_extra(package, dataset, 'accrualPeriodicity')
@@ -46,7 +48,6 @@ def map_dataset_to_package(dataset, catalog_id, demote_superThemes=True):
     append_attribute_to_extra(package, dataset, 'language', serialize=True)
 
     spatial = dataset.get('spatial')
-
     if spatial:
         serializable = type(spatial) is list
         append_attribute_to_extra(package, dataset, 'spatial', serializable)
@@ -60,6 +61,16 @@ def map_dataset_to_package(dataset, catalog_id, demote_superThemes=True):
     if keywords:
         package['tags'] = [{'name': keyword} for keyword in keywords]
 
+    # Move themes to keywords
+    themes = dataset.get('theme', [])
+    if themes and demote_themes:
+        package['tags'] = package.get('tags', [])
+        for theme in themes:
+            label = next(x['label'] for x in theme_taxonomy if x['id'] == theme)
+            package['tags'].append({'name': label})
+    else:
+        package['groups'] += [{'name': title_to_name(theme, decode=False)} for theme in themes]
+
     return package
 
 

pydatajson/federation.py

@@ -10,7 +10,7 @@
 
 
 def push_dataset_to_ckan(catalog, catalog_id, owner_org, dataset_origin_identifier, portal_url, apikey,
-                         demote_superThemes=True):
+                         demote_superThemes=True, demote_themes=True):
     """Escribe la metadata de un dataset en el portal pasado por parámetro.
 
         Args:
@@ -20,15 +20,19 @@ def push_dataset_to_ckan(catalog, catalog_id, owner_org, dataset_origin_identifi
             dataset_origin_identifier (str): El id del dataset que se va a federar.
             portal_url (str): La URL del portal CKAN de destino.
             apikey (str): La apikey de un usuario con los permisos que le permitan crear o actualizar el dataset.
+            demote_superThemes(bool): Si está en true, los ids de los super themes del dataset, se propagan como grupo.
+            demote_themes(bool): Si está en true, los labels de los themes del dataset, pasan a ser tags. Sino,
+            se pasan como grupo.
 
         Returns:
             str: El id del dataset en el catálogo de destino.
     """
     dataset = catalog.get_dataset(dataset_origin_identifier)
     ckan_portal = RemoteCKAN(portal_url, apikey=apikey)
+    theme_taxonomy = catalog.themes
 
-    package = map_dataset_to_package(dataset, catalog_id, demote_superThemes)
-    package['owner_org'] = owner_org
+    package = map_dataset_to_package(dataset, catalog_id, owner_org, theme_taxonomy,
+                                     demote_superThemes, demote_themes)
 
     # Get license id
     if dataset.get('license'):
@@ -43,15 +47,6 @@ def push_dataset_to_ckan(catalog, catalog_id, owner_org, dataset_origin_identifi
     else:
         package['license_id'] = 'notspecified'
 
-    # Move themes to keywords
-    themes = dataset.get('theme')
-    if themes:
-        package['tags'] = package.get('tags') or []
-        theme_taxonomy = catalog.themes
-        for theme in themes:
-            label = next(x['label'] for x in theme_taxonomy if x['id'] == theme)
-            package['tags'].append({'name': label})
-
     try:
         pushed_package = ckan_portal.call_action(
             'package_update', data_dict=package)

tests/test_ckan_utils.py

@@ -23,16 +23,17 @@ def setUpClass(cls):
         cls.distributions = cls.dataset['distribution']
 
     def test_replicated_plain_attributes_are_corrext(self):
-        package = map_dataset_to_package(self.dataset, self.catalog_id)
+        package = map_dataset_to_package(self.dataset, self.catalog_id, 'owner', self.catalog.themes)
         plain_replicated_attributes = [('title', 'title'),
                                        ('notes', 'description'),
                                        ('url', 'landingPage')]
         for fst, snd in plain_replicated_attributes:
             self.assertEqual(self.dataset.get(snd), package.get(fst))
+        self.assertEqual('owner', package['owner_org'])
         self.assertEqual(self.catalog_id+'_'+self.dataset_id, package['id'])
 
     def test_dataset_nested_replicated_attributes_stay_the_same(self):
-        package = map_dataset_to_package(self.dataset, self.catalog_id)
+        package = map_dataset_to_package(self.dataset, self.catalog_id, 'owner', self.catalog.themes)
         contact_point_nested = [('maintainer', 'fn'),
                                 ('maintainer_email', 'hasEmail')]
         for fst, snd in contact_point_nested:
@@ -43,7 +44,7 @@ def test_dataset_nested_replicated_attributes_stay_the_same(self):
             self.assertEqual(self.dataset.get('publisher').get(snd), package.get(fst))
 
     def test_dataset_array_attributes_are_correct(self):
-        package = map_dataset_to_package(self.dataset, self.catalog_id)
+        package = map_dataset_to_package(self.dataset, self.catalog_id, 'owner', self.catalog.themes)
         groups = [group['name'] for group in package.get('groups', [])]
         super_themes = [title_to_name(s_theme.lower()) for s_theme in self.dataset.get('superTheme')]
         try:
@@ -53,29 +54,60 @@ def test_dataset_array_attributes_are_correct(self):
 
         tags = [tag['name'] for tag in package['tags']]
         keywords = self.dataset.get('keyword', [])
+
+        themes = self.dataset.get('theme', [])
+        theme_labels = []
+        for theme in themes:
+            label = next(x['label'] for x in self.catalog.themes if x['id'] == theme)
+            theme_labels.append(label)
+
+        try:
+            self.assertItemsEqual(keywords + theme_labels, tags)
+        except AttributeError:
+            self.assertCountEqual(keywords + theme_labels, tags)
+
+    def test_themes_are_preserved_if_not_demoted(self):
+        package = map_dataset_to_package(self.dataset, self.catalog_id, 'owner',
+                                         self.catalog.themes, demote_themes=False)
+        groups = [group['name'] for group in package.get('groups', [])]
+        super_themes = [title_to_name(s_theme.lower()) for s_theme in self.dataset.get('superTheme')]
+        themes = self.dataset.get('theme', [])
+        tags = [tag['name'] for tag in package['tags']]
+        keywords = self.dataset.get('keyword', [])
+
+        try:
+            self.assertItemsEqual(super_themes + themes, groups)
+        except AttributeError:
+            self.assertCountEqual(super_themes + themes, groups)
         try:
             self.assertItemsEqual(keywords, tags)
         except AttributeError:
             self.assertCountEqual(keywords, tags)
 
     def test_superThemes_dont_impact_groups_if_not_demoted(self):
-        package = map_dataset_to_package(self.dataset, self.catalog_id, demote_superThemes=False)
+        package = map_dataset_to_package(self.dataset, self.catalog_id,'owner',
+                                         self.catalog.themes, demote_superThemes=False)
 
         groups = [group['name'] for group in package.get('groups', [])]
         tags = [tag['name'] for tag in package['tags']]
         keywords = self.dataset.get('keyword', [])
+        themes = self.dataset.get('theme', [])
+        theme_labels = []
+        for theme in themes:
+            label = next(x['label'] for x in self.catalog.themes if x['id'] == theme)
+            theme_labels.append(label)
 
         try:
             self.assertItemsEqual([], groups)
         except AttributeError:
             self.assertCountEqual([], groups)
         try:
-            self.assertItemsEqual(keywords, tags)
+            self.assertItemsEqual(keywords + theme_labels, tags)
         except AttributeError:
-            self.assertCountEqual(keywords, tags)
+            self.assertCountEqual(keywords + theme_labels, tags)
 
     def test_dataset_extra_attributes_are_correct(self):
-        package = map_dataset_to_package(self.dataset, self.catalog_id)
+        package = map_dataset_to_package(self.dataset, self.catalog_id, 'owner', self.catalog.themes)
 #       extras are included in dataset
         if package['extras']:
             for extra in package['extras']:
@@ -94,7 +126,7 @@ def test_dataset_extra_attributes_are_correct(self):
                     self.assertEqual(dataset_value, extra_value)
 
     def test_dataset_extra_attributes_are_complete(self):
-        package = map_dataset_to_package(self.dataset, self.catalog_id)
+        package = map_dataset_to_package(self.dataset, self.catalog_id, 'owner', self.catalog.themes)
 #       dataset attributes are included in extras
         extra_attrs = ['issued', 'modified', 'accrualPeriodicity', 'temporal', 'language', 'spatial']
         for key in extra_attrs: