webapp time out - Graphs in search_features_profiles

IssueID #2746: webapp time out - Graphs in search_features_profiles
IssueID #2602: Graphs in search_features_profiles

- Added a lightweight ionosphere_show_graphs function to ionosphere_backend to
  and changed from using ionosphere_metric_data to using this lightweight
  function which does not process matched graphs, only the images required by
  the show graaphs context.  This stops webapp time outs (2746)

Modified:
skyline/webapp/ionosphere_backend.py
skyline/webapp/webapp.py

skyline/webapp/ionosphere_backend.py

@@ -4452,3 +4452,63 @@ def get_metrics_with_features_profiles_to_validate():
     logger.info('%s :: metrics with features profiles to validate - %s' % (
         function_str, str(metrics_with_features_profiles_to_validate)))
     return (metrics_with_features_profiles_to_validate, fail_msg, trace)
+
+
+# @added 20181205 - Bug #2746: webapp time out - Graphs in search_features_profiles
+#                   Feature #2602: Graphs in search_features_profiles
+def ionosphere_show_graphs(requested_timestamp, data_for_metric, fp_id):
+    """
+    Get a list of all graphs
+    """
+
+    base_name = data_for_metric.replace(settings.FULL_NAMESPACE, '', 1)
+    log_context = 'features profile data show graphs'
+    logger.info('%s requested for %s at %s' % (
+        log_context, str(base_name), str(requested_timestamp)))
+    images = []
+    timeseries_dir = base_name.replace('.', '/')
+    metric_data_dir = '%s/%s/%s' % (
+        settings.IONOSPHERE_PROFILES_FOLDER, timeseries_dir,
+        str(requested_timestamp))
+
+    td_files = listdir(metric_data_dir)
+    for i_file in td_files:
+        metric_file = path.join(metric_data_dir, i_file)
+        if i_file.endswith('.png'):
+            # @modified 20170106 - Feature #1842: Ionosphere - Graphite now graphs
+            # Exclude any graphite_now png files from the images lists
+            append_image = True
+            if '.graphite_now.' in i_file:
+                append_image = False
+            # @added 20170107 - Feature #1852: Ionosphere - features_profile matched graphite graphs
+            # Exclude any matched.fp-id images
+            if '.matched.fp_id' in i_file:
+                append_image = False
+            # @added 20170308 - Feature #1960: ionosphere_layers
+            #                   Feature #1852: Ionosphere - features_profile matched graphite graphs
+            # Exclude any matched.fp-id images
+            if '.matched.layers.fp_id' in i_file:
+                append_image = False
+            if append_image:
+                images.append(str(metric_file))
+
+    graphite_now_images = []
+    graphite_now = int(time.time())
+    graph_resolutions = []
+    graph_resolutions = [int(settings.TARGET_HOURS), 24, 168, 720]
+    # @modified 20170107 - Feature #1842: Ionosphere - Graphite now graphs
+    # Exclude if matches TARGET_HOURS - unique only
+    _graph_resolutions = sorted(set(graph_resolutions))
+    graph_resolutions = _graph_resolutions
+
+    for target_hours in graph_resolutions:
+        graph_image = False
+        try:
+            graph_image_file = '%s/%s.graphite_now.%sh.png' % (metric_data_dir, base_name, str(target_hours))
+            if path.isfile(graph_image_file):
+                graphite_now_images.append(graph_image_file)
+        except:
+            logger.error(traceback.format_exc())
+            logger.error('error :: failed to get Graphite graph at %s hours for %s' % (str(target_hours), base_name))
+
+    return (images, graphite_now_images)

skyline/webapp/webapp.py

@@ -123,7 +123,12 @@
     # @added 20180812 - Feature #2430: Ionosphere validate learnt features profiles page
     get_features_profiles_to_validate,
     # @added 20180815 - Feature #2430: Ionosphere validate learnt features profiles page
-    get_metrics_with_features_profiles_to_validate,)
+    get_metrics_with_features_profiles_to_validate,
+    # @added 20181205 - Bug #2746: webapp time out - Graphs in search_features_profiles
+    #                   Feature #2602: Graphs in search_features_profiles
+    ionosphere_show_graphs,)
+
+#from utilites import alerts_matcher
 
 # @added 20170114 - Feature #1854: Ionosphere learn
 # Decoupled the create_features_profile from ionosphere_backend and moved to
@@ -1380,6 +1385,7 @@ def ionosphere():
             # @modified 20180717 - Task #2446: Optimize Ionosphere
             # Added missing search_success variable
             features_profiles, fps_count, mc, cc, gc, full_duration_list, enabled_list, tsfresh_version_list, generation_list, search_success, fail_msg, trace = ionosphere_search(False, True)
+
             return render_template(
                 'ionosphere.html', fp_search=fp_search_req,
                 fp_search_results=fp_search_req,
@@ -1405,7 +1411,7 @@ def ionosphere():
             if not search_success:
                 return internal_error(fail_msg, trace)
 
-            # @added 20180917 - Feature #2602: Graaphs in search_features_profiles
+            # @added 20180917 - Feature #2602: Graphs in search_features_profiles
             if search_success and fps:
                 show_graphs = request.args.get(str('show_graphs'), False)
                 if show_graphs == 'true':
@@ -1418,7 +1424,14 @@ def ionosphere():
                         fp_id = fp_elements[0]
                         base_name = fp_elements[2]
                         requested_timestamp = fp_elements[4]
-                        mpaths, images, hdate, m_vars, ts_json, data_to_process, p_id, gimages, gmimages, times_matched, glm_images, l_id_matched, ts_fd, i_ts_json, anomalous_timeseries, f_id_matched, fp_details_list = ionosphere_metric_data(requested_timestamp, base_name, query_context, fp_id)
+
+                        # @modified 20181205 - Bug #2746: webapp time out - Graphs in search_features_profiles
+                        #                      Feature #2602: Graphs in search_features_profiles
+                        # This function was causing the webapp to time out due
+                        # to fetching all the matched Graphite graphs
+                        # mpaths, images, hdate, m_vars, ts_json, data_to_process, p_id, gimages, gmimages, times_matched, glm_images, l_id_matched, ts_fd, i_ts_json, anomalous_timeseries, f_id_matched, fp_details_list = ionosphere_metric_data(requested_timestamp, base_name, query_context, fp_id)
+                        images, gimages = ionosphere_show_graphs(requested_timestamp, base_name, fp_id)
+
                         new_fp = []
                         for fp_element in fp_elements:
                             new_fp.append(fp_element)