Improve query time get_builds (#290)

At the same way of get_deployment, we are doing one query for each job so if we have 2000 jobs then we have 2000 queries.

As we can't send all the jobs to ask information of them in one single query because it's too big and it returns error, I've used chunks like: #276 and we have reduced the time of the queries:

From:
INFO     cibyl.orchestrator   Took 551.80s to query system osp_jenkins using source: 'elasticsearch' of type: 'elasticsearch' using method get_builds
To:
INFO     cibyl.orchestrator   Took 117.57s to query system osp_jenkins using source elasticsearch of type elasticsearch using method get_builds

cibyl/sources/elasticsearch/api.py

@@ -133,48 +133,84 @@ def get_builds(self: object, **kwargs: Argument):
         """
         jobs_found = self.get_jobs(**kwargs)
 
-        for job_name, job in jobs_found.items():
-            query_body = QueryTemplate('job_name',
-                                       [job_name],
-                                       query_type='match').get
+        query_body = {
+            "query": {
+              "bool": {
+                "should": []
+              }
+            }
+        }
 
-            builds = self.__query_get_hits(
-                query=query_body,
-                index='jenkins_builds'
+        def append_job_match_to_query(job_name: str):
+            query_body['query']['bool']['should'].append(
+                {
+                  "match": {
+                    "job_name": f"{job_name}"
+                  }
+                }
             )
 
-            build_statuses = []
-            if 'build_status' in kwargs:
-                build_statuses = [status.upper()
-                                  for status in
-                                  kwargs.get('build_status').value]
-
-            build_id_argument = None
-            if 'builds' in kwargs:
-                build_id_argument = kwargs.get('builds').value
+        for _, job in jobs_found.items():
 
-            for build in builds:
+            chunked_list_of_jobs = []
+            chunk_size_for_search = 400
 
-                build_result = None
-                if not build['_source']['build_result'] and \
-                        build['_source']['current_build_result']:
-                    build_result = build['_source']['current_build_result']
-                else:
-                    build_result = build["_source"]['build_result']
-
-                if 'build_status' in kwargs and \
-                        build['_source']['build_result'] not in build_statuses:
-                    continue
+            for chunk_max_value in range(
+                0,
+                len(list(jobs_found.keys())),
+                chunk_size_for_search
+            ):
+                chunked_list_of_jobs.append(
+                    list(
+                        jobs_found.keys()
+                    )[chunk_max_value:chunk_max_value + chunk_size_for_search]
+                )
 
-                build_id = str(build['_source']['build_id'])
+        builds = []
+        for jobs_list in chunked_list_of_jobs:
+            for job in jobs_list:
+                append_job_match_to_query(job)
 
-                if build_id_argument and \
-                        build_id not in build_id_argument:
-                    continue
+            builds_results = self.__query_get_hits(
+                query=query_body,
+                index='jenkins_builds'
+            )
+            if builds_results:
+                for build in builds_results:
+                    builds.append(build)
+
+            query_body['query']['bool']['should'].clear()
+
+        build_statuses = []
+        if 'build_status' in kwargs:
+            build_statuses = [status.upper()
+                              for status in
+                              kwargs.get('build_status').value]
+
+        build_id_argument = None
+        if 'builds' in kwargs:
+            build_id_argument = kwargs.get('builds').value
+
+        for build in builds:
+            build_result = None
+            if not build['_source']['build_result'] and \
+                    build['_source']['current_build_result']:
+                build_result = build['_source']['current_build_result']
+            else:
+                build_result = build["_source"]['build_result']
+
+            if 'build_status' in kwargs and \
+                    build['_source']['build_result'] not in build_statuses:
+                continue
 
-                job.add_build(Build(build_id,
-                                    build_result,
-                                    build['_source']['time_duration']))
+            build_id = str(build['_source']['build_id'])
+            if build_id_argument and \
+                    build_id not in build_id_argument:
+                continue
+            job_name = build['_source']['job_name']
+            jobs_found[job_name].add_build(Build(build_id,
+                                           build_result,
+                                           build['_source']['time_duration']))
 
         if 'last_build' in kwargs:
             return self.get_last_build(jobs_found)

tests/unit/sources/elasticsearch/test_api.py

@@ -174,7 +174,7 @@ def test_get_builds(self: object, mock_query_hits: object) -> None:
         mock_query_hits.return_value = self.build_hits
 
         builds = self.es_api.get_builds()['test'].builds
-        self.assertEqual(len(builds), 2)
+        self.assertEqual(len(builds), 1)
 
         build = builds['1']
         self.assertEqual(build.build_id.value, '1')
@@ -202,7 +202,7 @@ def test_get_builds_by_status(self: object,
         type(status_argument).value = build_status
 
         builds = self.es_api.get_builds(build_status=status_argument)
-        builds_values = builds['test'].builds
+        builds_values = builds['test2'].builds
         build = builds_values['2']
         self.assertEqual(build.build_id.value, '2')
         self.assertEqual(build.status.value, "FAIL")
@@ -220,7 +220,6 @@ def test_get_tests(self: object,
         with self.assertRaises(MissingArgument):
             self.es_api.get_tests()
 
-        #
         builds_kwargs = MagicMock()
         builds_value = PropertyMock(return_value=[])
         type(builds_kwargs).value = builds_value
@@ -251,8 +250,8 @@ def test_get_tests(self: object,
         )
 
         self.assertEqual(
-            len(tests['test'].builds['2'].tests),
-            0
+            len(tests['test'].builds['1'].tests),
+            1
         )
         self.assertTrue('it_is_just_a_test' in
                         tests['test'].builds['1'].tests)
@@ -271,11 +270,6 @@ def test_get_tests(self: object,
             1
         )
 
-        self.assertEqual(
-            len(tests['test'].builds['2'].tests),
-            0
-        )
-
     @patch('cibyl.sources.elasticsearch.api.ElasticSearchClient')
     def test_setup(self, mock_client):
         """Test setup method of ElasticSearch"""