Merge branch 'fix_multi_org_names' of 'https://github.com/zhquan/Grim…

…oireELK'

Merges #1019
Closes #1019

grimoire_elk/enriched/enrich.py

@@ -21,7 +21,6 @@
 #   Miguel Ángel Fernández <mafesan@bitergia.com>
 #
 
-import copy
 import json
 import functools
 import logging
@@ -119,7 +118,6 @@ class Enrich(ElasticItems):
 
     def __init__(self, db_sortinghat=None, db_projects_map=None, json_projects_map=None,
                  db_user='', db_password='', db_host='', insecure=True):
-
         perceval_backend = None
         super().__init__(perceval_backend, insecure=insecure)
 
@@ -765,7 +763,8 @@ def get_item_no_sh_fields(self, identity, rol):
             rol + "_gender": self.unknown_gender,
             rol + "_gender_acc": None,
             rol + "_org_name": self.unaffiliated_group,
-            rol + "_bot": False
+            rol + "_bot": False,
+            rol + MULTI_ORG_NAMES: [self.unaffiliated_group]
         }
 
     def get_item_sh_fields(self, identity=None, item_date=None, sh_id=None,
@@ -876,28 +875,7 @@ def get_item_sh_from_id(self, eitem, roles=None):
     def get_item_sh_meta_fields(self, eitem, roles=None, suffixes=None, non_authored_prefix=None):
         """Get the SH meta fields from the data in the enriched item."""
 
-        def add_non_authored_fields(eitem_sh, rol, non_authored_prefix, author_uuid):
-            new_eitem_sh = {}
-            # Add the non_authored_* field
-            for item in eitem_sh:
-                if rol in item and non_authored_prefix not in item:
-                    new_eitem_sh[non_authored_prefix + item] = copy.deepcopy(eitem_sh[item])
-
-            uuids_field = non_authored_prefix + rol + '_uuids'
-            # Check if author_uuid is in uuids_field
-            if author_uuid not in new_eitem_sh[uuids_field]:
-                new_eitem_sh.update(eitem_sh)
-                return new_eitem_sh
-
-            # Remove the author in non_authored
-            remove_indices = [i for i, uuid in enumerate(new_eitem_sh[uuids_field]) if uuid == author_uuid]
-            for item in new_eitem_sh:
-                new_eitem_sh[item] = [i for index, i in enumerate(new_eitem_sh[item]) if index not in remove_indices]
-            new_eitem_sh.update(eitem_sh)
-
-            return new_eitem_sh
-
-        eitem_sh = {}  # Item enriched
+        eitem_meta_sh = {}  # Item enriched
 
         date = str_to_datetime(eitem[self.get_field_date()])
 
@@ -910,16 +888,39 @@ def add_non_authored_fields(eitem_sh, rol, non_authored_prefix, author_uuid):
                 continue
 
             for sh_uuid in sh_uuids:
-                position = sh_uuids.index(sh_uuid)
-                new_eitem = self.get_item_sh_fields(sh_id=sh_uuid, item_date=date, rol=rol)
+                sh_fields = self.get_item_sh_fields(sh_id=sh_uuid, item_date=date, rol=rol)
 
-                for suffix in suffixes:
-                    eitem_sh[rol + suffix] = eitem[rol + suffix]
-                    eitem_sh[rol + suffix][position] = new_eitem[rol + suffix[:-1]]
-            if non_authored_prefix:
-                eitem_sh = add_non_authored_fields(eitem_sh, rol, non_authored_prefix, eitem['author_uuid'])
+                self.add_meta_fields(eitem, eitem_meta_sh, sh_fields, rol, sh_uuid, suffixes, non_authored_prefix)
 
-        return eitem_sh
+        return eitem_meta_sh
+
+    def add_meta_fields(self, eitem, meta_eitem, sh_fields, rol, uuid, suffixes, non_authored_prefix):
+        def add_non_authored_fields(author_uuid, uuid, new_eitem, new_list, non_authored_field):
+            if author_uuid == uuid:
+                non_authored = []
+            else:
+                non_authored = new_list
+            new_eitem[non_authored_field] = non_authored
+
+        for suffix in suffixes:
+            field = rol + suffix[:-1]
+            if suffix == "_org_names":
+                field = rol + "_multi" + suffix
+
+            new_list = sh_fields[field]
+            if type(new_list) != list:
+                new_list = [new_list]
+
+            try:
+                meta_eitem[rol + suffix] += new_list
+            except KeyError:
+                meta_eitem[rol + suffix] = new_list
+
+            if non_authored_prefix:
+                non_authored_field = non_authored_prefix + rol + suffix
+                add_non_authored_fields(eitem['author_uuid'], uuid, meta_eitem, new_list,
+                                        non_authored_field)
+        return meta_eitem
 
     def get_users_data(self, item):
         """ If user fields are inside the global item dict """

grimoire_elk/enriched/git.py

@@ -364,6 +364,7 @@ def __add_commit_meta_fields(self, eitem, commit):
         if 'message' not in commit:
             return
 
+        meta_eitem = {}
         for line in commit['message'].split('\n'):
             m = self.AUTHOR_REGEX.match(line)
             if not m:
@@ -384,12 +385,10 @@ def __add_commit_meta_fields(self, eitem, commit):
             else:
                 sh_fields = self.get_item_no_sh_fields(identity, rol=meta_field)
 
-            for suffix in self.meta_fields_suffixes:
-                eitem[meta_field + suffix].append(sh_fields[meta_field + suffix[:-1]])
-                # Add non_authored_* if it is not the author of the commit
-                if sh_fields[meta_field + '_uuid'] != eitem['author_uuid']:
-                    eitem[self.meta_non_authored_prefix + meta_field + suffix].append(
-                        sh_fields[meta_field + suffix[:-1]])
+            uuid = sh_fields[meta_field + '_uuid']
+            self.add_meta_fields(eitem, meta_eitem, sh_fields, meta_field, uuid, self.meta_fields_suffixes,
+                                 self.meta_non_authored_prefix)
+        eitem.update(meta_eitem)
 
     def __add_pair_programming_metrics(self, commit, eitem):