Merge pull request #125 from datosgobar/90-mover-superTheme-a-theme

90 mover super theme a theme

docs/MANUAL.md

@@ -94,8 +94,11 @@ Toma los siguientes parámetros:
   - **portal_url**: URL del portal de CKAN de destino.
   - **apikey**: La apikey de un usuario del portal de destino con los permisos para crear el dataset bajo la
   organización pasada como parámetro.
+  - **demote_superThemes** (opcional, default: True):Si está en true, los ids de los themes del dataset, se escriben
+  como groups de CKAN.
   - **demote_themes** (opcional, default: True): Si está en true, los labels de los themes del dataset, se escriben como
   tags de CKAN; sino,se pasan como grupo.
+
 
   Retorna el id en el nodo de destino del dataset federado.
 

pydatajson/ckan_utils.py

@@ -14,7 +14,8 @@ def append_attribute_to_extra(package, dataset, attribute, serialize=False):
         package['extras'].append({'key': attribute, 'value': value})
 
 
-def map_dataset_to_package(dataset, catalog_id, owner_org, theme_taxonomy, demote_themes=True):
+def map_dataset_to_package(dataset, catalog_id, owner_org, theme_taxonomy,
+                           demote_superThemes=True, demote_themes=True):
     package = dict()
     package['extras'] = []
 #   Obligatorios
@@ -33,8 +34,10 @@ def map_dataset_to_package(dataset, catalog_id, owner_org, theme_taxonomy, demot
     package['resources'] = map_distributions_to_resources(distributions, catalog_id)
 
     super_themes = dataset['superTheme']
-    package['groups'] = [{'name': title_to_name(super_theme, decode=False)} for super_theme in super_themes]
     append_attribute_to_extra(package, dataset, 'superTheme', serialize=True)
+    if demote_superThemes:
+        package['groups'] = [{'name': title_to_name(super_theme, decode=False)} for super_theme in super_themes]
+
 
 #   Recomendados y opcionales
     package['url'] = dataset.get('landingPage')
@@ -66,7 +69,8 @@ def map_dataset_to_package(dataset, catalog_id, owner_org, theme_taxonomy, demot
             label = next(x['label'] for x in theme_taxonomy if x['id'] == theme)
             package['tags'].append({'name': label})
     else:
-        package['groups'] += [{'name': title_to_name(theme, decode=False)} for theme in themes]
+        package['groups'] = package.get('groups', []) + [{'name': title_to_name(theme, decode=False)}
+                                                         for theme in themes]
 
     return package
 

pydatajson/federation.py

@@ -10,7 +10,7 @@
 
 
 def push_dataset_to_ckan(catalog, catalog_id, owner_org, dataset_origin_identifier, portal_url, apikey,
-                         demote_themes=True):
+                         demote_superThemes=True, demote_themes=True):
     """Escribe la metadata de un dataset en el portal pasado por parámetro.
 
         Args:
@@ -20,6 +20,7 @@ def push_dataset_to_ckan(catalog, catalog_id, owner_org, dataset_origin_identifi
             dataset_origin_identifier (str): El id del dataset que se va a federar.
             portal_url (str): La URL del portal CKAN de destino.
             apikey (str): La apikey de un usuario con los permisos que le permitan crear o actualizar el dataset.
+            demote_superThemes(bool): Si está en true, los ids de los super themes del dataset, se propagan como grupo.
             demote_themes(bool): Si está en true, los labels de los themes del dataset, pasan a ser tags. Sino,
             se pasan como grupo.
 
@@ -30,7 +31,8 @@ def push_dataset_to_ckan(catalog, catalog_id, owner_org, dataset_origin_identifi
     ckan_portal = RemoteCKAN(portal_url, apikey=apikey)
     theme_taxonomy = catalog.themes
 
-    package = map_dataset_to_package(dataset, catalog_id, owner_org, theme_taxonomy, demote_themes=demote_themes)
+    package = map_dataset_to_package(dataset, catalog_id, owner_org, theme_taxonomy,
+                                     demote_superThemes, demote_themes)
 
     # Get license id
     if dataset.get('license'):

tests/test_ckan_utils.py

@@ -67,8 +67,8 @@ def test_dataset_array_attributes_are_correct(self):
             self.assertCountEqual(keywords + theme_labels, tags)
 
     def test_themes_are_preserved_if_not_demoted(self):
-        package = map_dataset_to_package(self.dataset, self.catalog_id, 'owner', self.catalog.themes,
-                                         demote_themes=False)
+        package = map_dataset_to_package(self.dataset, self.catalog_id, 'owner',
+                                         self.catalog.themes, demote_themes=False)
         groups = [group['name'] for group in package.get('groups', [])]
         super_themes = [title_to_name(s_theme.lower()) for s_theme in self.dataset.get('superTheme')]
         themes = self.dataset.get('theme', [])
@@ -84,15 +84,48 @@ def test_themes_are_preserved_if_not_demoted(self):
         except AttributeError:
             self.assertCountEqual(keywords, tags)
 
+    def test_superThemes_dont_impact_groups_if_not_demoted(self):
+        package = map_dataset_to_package(self.dataset, self.catalog_id, 'owner',
+                                         self.catalog.themes, demote_superThemes=False)
+        groups = [group['name'] for group in package.get('groups', [])]
+        tags = [tag['name'] for tag in package['tags']]
+        keywords = self.dataset.get('keyword', [])
+        themes = self.dataset.get('theme', [])
+        theme_labels = []
+        for theme in themes:
+            label = next(x['label'] for x in self.catalog.themes if x['id'] == theme)
+            theme_labels.append(label)
+        try:
+            self.assertItemsEqual([], groups)
+        except AttributeError:
+            self.assertCountEqual([], groups)
+        try:
+            self.assertItemsEqual(keywords + theme_labels, tags)
+        except AttributeError:
+            self.assertCountEqual(keywords + theme_labels, tags)
+
+    def test_preserve_themes_and_superThemes(self):
+        package = map_dataset_to_package(self.dataset, self.catalog_id, 'owner',
+                                         self.catalog.themes, False, False)
+        groups = [group['name'] for group in package.get('groups', [])]
+        tags = [tag['name'] for tag in package['tags']]
+        keywords = self.dataset.get('keyword', [])
+        themes = self.dataset.get('theme', [])
+        try:
+            self.assertItemsEqual(themes, groups)
+        except AttributeError:
+            self.assertCountEqual(themes, groups)
+        try:
+            self.assertItemsEqual(keywords, tags)
+        except AttributeError:
+            self.assertCountEqual(keywords, tags)
+
     def test_dataset_extra_attributes_are_correct(self):
         package = map_dataset_to_package(self.dataset, self.catalog_id, 'owner', self.catalog.themes)
 #       extras are included in dataset
         if package['extras']:
             for extra in package['extras']:
-                if extra['key'] == 'super_theme':
-                    dataset_value = self.dataset['superTheme']
-                else:
-                    dataset_value = self.dataset[extra['key']]
+                dataset_value = self.dataset[extra['key']]
                 if type(dataset_value) is list:
                     extra_value = json.loads(extra['value'])
                     try:
@@ -106,7 +139,7 @@ def test_dataset_extra_attributes_are_correct(self):
     def test_dataset_extra_attributes_are_complete(self):
         package = map_dataset_to_package(self.dataset, self.catalog_id, 'owner', self.catalog.themes)
 #       dataset attributes are included in extras
-        extra_attrs = ['issued', 'modified', 'accrualPeriodicity', 'temporal', 'language', 'spatial']
+        extra_attrs = ['issued', 'modified', 'accrualPeriodicity', 'temporal', 'language', 'spatial', 'superTheme']
         for key in extra_attrs:
             value = self.dataset.get(key)
             if value:
@@ -115,8 +148,6 @@ def test_dataset_extra_attributes_are_complete(self):
                 resulting_dict = {'key': key, 'value': value}
                 self.assertTrue(resulting_dict in package['extras'])
 
-        self.assertTrue({'key': 'super_theme', 'value': json.dumps(self.dataset['superTheme'])})
-
     def test_resources_replicated_attributes_stay_the_same(self):
         resources = map_distributions_to_resources(self.distributions, self.catalog_id+'_'+self.dataset_id)
         for resource in resources: