feat(dashboard): add resource monitoring tab and information about input status

docs/dashboard.md

@@ -14,11 +14,12 @@ The dashboard page provides some additional information about the add-on
 operations to increase the visibility into what the add-on is actually doing
 under the hood.
 
-As of now, 3 pre-built panels are supported:
+As of now, 4 pre-built panels are supported:
 
 * Overview
 * Data ingestion
 * Errors in the add-on.
+* Resource consumption.
 
 **IMPORTANT**: To fully use the panels available on the monitoring dashboard, use the `solnlib.log`'s [`events_ingested` function](https://github.com/splunk/addonfactory-solutions-library-python/blob/v4.14.0/solnlib/log.py#L253), available from **version 4.14**, to record events.
 

splunk_add_on_ucc_framework/dashboard.py

@@ -46,13 +46,15 @@
     "overview": "overview_definition.json",
     "data_ingestion_tab": "data_ingestion_tab_definition.json",
     "errors_tab": "errors_tab_definition.json",
+    "resources_tab": "resources_tab_definition.json",
 }
 
 data_ingestion = (
     "index=_internal source=*license_usage.log type=Usage "
     "(s IN ({input_names})) | timechart sum(b) as Usage | "
     'rename Usage as \\"Data volume\\"'
 )
+
 data_ingestion_and_events = (
     "index=_internal source=*license_usage.log type=Usage "
     "(s IN ({input_names})) | timechart sum(b) as Usage "
@@ -117,17 +119,30 @@
     '| rename event_account as \\"Account\\", events as \\"Number of events\\", '
     'sparkevent as \\"Event trendline\\"'
 )
+
 table_input_query = (
-    "index = _internal source=*{addon_name}* action=events_ingested "
+    '| rest splunk_server=local /services/data/inputs/all | where $eai:acl.app$ = \\"{addon_name}\\" '
+    '| eval Enabled=if(lower(disabled) IN (\\"1\\", \\"true\\", \\"t\\"), \\"no\\", \\"yes\\") '
+    '| table title, Enabled | rename title as \\"event_input\\" | join type=left event_input [ '
+    "search index = _internal source=*{addon_name_lowercase}* action=events_ingested "
     "| stats latest(_time) as le, sparkline(sum(n_events)) as sparkevent, sum(n_events) as events by event_input "
-    '| eval \\"Last event\\" = strftime(le, \\"%e %b %Y %I:%M%p\\") '
-    '| table event_input, events, sparkevent, \\"Last event\\" '
-    '| rename event_input as \\"Input\\", events as \\"Number of events\\", '
-    'sparkevent as \\"Event trendline\\"'
+    '| eval \\"Last event\\" = strftime(le, \\"%e %b %Y %I:%M%p\\") ] | makemv delim=\\",\\" sparkevent '
+    '| table event_input, Enabled, events, sparkevent, \\"Last event\\" '
+    '| rename event_input as \\"Input\\", events as \\"Number of events\\", sparkevent as \\"Event trendline\\"'
 )
 
 errors_list_query = "index=_internal source=*{addon_name}* ERROR"
 
+resource_cpu_query = (
+    "index = _introspection component=PerProcess data.args=*{addon_name}* "
+    '| timechart avg(data.pct_cpu) as \\"CPU (%)\\"'
+)
+
+resource_memory_query = (
+    "index=_introspection component=PerProcess data.args=*{addon_name}* "
+    '| timechart avg(data.pct_memory) as \\"Memory (%)\\"'
+)
+
 
 def generate_dashboard_content(
     addon_name: str, input_names: List[str], definition_json_name: str
@@ -169,7 +184,9 @@ def generate_dashboard_content(
                     input_names=input_names_str, addon_name=addon_name.lower()
                 ),
                 table_account=table_account_query.format(addon_name=addon_name.lower()),
-                table_input=table_input_query.format(addon_name=addon_name.lower()),
+                table_input=table_input_query.format(
+                    addon_name=addon_name, addon_name_lowercase=addon_name.lower()
+                ),
             )
         )
 
@@ -183,6 +200,18 @@ def generate_dashboard_content(
             )
         )
 
+    if definition_json_name == default_definition_json_filename["resources_tab"]:
+        content = (
+            utils.get_j2_env()
+            .get_template(definition_json_name)
+            .render(
+                resource_cpu=resource_cpu_query.format(addon_name=addon_name.lower()),
+                resource_memory=resource_memory_query.format(
+                    addon_name=addon_name.lower()
+                ),
+            )
+        )
+
     return content
 
 

splunk_add_on_ucc_framework/templates/data_ingestion_tab_definition.json

@@ -163,7 +163,7 @@
   "inputs": {
     "data_ingestion_input": {
       "options": {
-        "defaultValue": "-7d,now",
+        "defaultValue": "-24h,now",
         "token": "data_ingestion_time"
       },
       "title": "Time",

splunk_add_on_ucc_framework/templates/errors_tab_definition.json

@@ -42,9 +42,6 @@
       "options": {
         "xAxisVisibility": "hide",
         "xAxisTitleText": "Time",
-        "seriesColors": [
-          "#A870EF"
-        ],
         "yAxisTitleText": "Errors count"
       },
       "title": "Errors count",
@@ -106,7 +103,7 @@
   "inputs": {
     "errors_tab_input": {
       "options": {
-        "defaultValue": "-7d,now",
+        "defaultValue": "-24h,now",
         "token": "errors_tab_time"
       },
       "title": "Time",

splunk_add_on_ucc_framework/templates/overview_definition.json

@@ -71,9 +71,6 @@
       "options": {
         "xAxisVisibility": "hide",
         "xAxisTitleText": "Time",
-        "seriesColors": [
-          "#A870EF"
-        ],
         "yAxisTitleText": "Error count"
       },
       "title": "Errors",
@@ -128,7 +125,7 @@
   "inputs": {
     "overview_input": {
       "options": {
-        "defaultValue": "-7d,now",
+        "defaultValue": "-24h,now",
         "token": "overview_time"
       },
       "title": "Time",

splunk_add_on_ucc_framework/templates/resources_tab_definition.json

@@ -0,0 +1,186 @@
+{
+  "visualizations": {
+    "resource_tab_label_viz": {
+      "type": "splunk.markdown",
+      "options": {
+        "markdown": "## Resource consumption",
+        "fontSize": "large"
+      }
+    },
+    "resource_tab_description_viz": {
+      "type": "splunk.markdown",
+      "options": {
+        "markdown": "View the resource consumption for your add-on.",
+        "customFontSize": 12,
+        "fontColor": "#6c7685"
+      }
+    },
+    "resource_tab_timerange_label_start_viz": {
+      "type": "splunk.singlevalue",
+      "options": {
+        "majorFontSize": 12,
+        "backgroundColor": "transparent",
+        "majorColor": "#9fa4af"
+      },
+      "dataSources": {
+        "primary": "resource_tab_data_time_label_start_ds"
+      }
+    },
+    "resource_tab_timerange_label_end_viz": {
+      "type": "splunk.singlevalue",
+      "options": {
+        "majorFontSize": 12,
+        "backgroundColor": "transparent",
+        "majorColor": "#9fa4af"
+      },
+      "dataSources": {
+        "primary": "resource_tab_data_time_label_end_ds"
+      }
+    },
+    "resource_tab_cpu_consumption_viz": {
+      "type": "splunk.line",
+      "options": {
+        "xAxisVisibility": "hide",
+        "xAxisTitleText": "Time",
+        "seriesColors": [
+          "#A870EF"
+        ],
+        "yAxisTitleText": "CPU (%)"
+      },
+      "title": "CPU consumption",
+      "dataSources": {
+        "primary": "resource_tab_cpu_consumption_ds"
+      }
+    },
+    "resource_tab_memory_consumption_viz": {
+      "type": "splunk.line",
+      "options": {
+        "xAxisVisibility": "hide",
+        "xAxisTitleText": "Time",
+        "seriesColors": [
+          "#A870EF"
+        ],
+        "yAxisTitleText": "Memory (%)"
+      },
+      "title": "Memory consumption",
+      "dataSources": {
+        "primary": "resource_tab_memory_consumption_ds"
+      }
+    }
+  },
+  "dataSources": {
+    "resource_tab_data_time_label_start_ds": {
+      "type": "ds.search",
+      "options": {
+        "query": "| makeresults | addinfo | eval StartDate = strftime(info_min_time, \"%e %b %Y %I:%M%p\") | table StartDate",
+        "queryParameters": {
+          "earliest": "$resource_tab_time.earliest$",
+          "latest": "$resource_tab_time.latest$"
+        }
+      }
+    },
+    "resource_tab_data_time_label_end_ds": {
+      "type": "ds.search",
+      "options": {
+        "query": "| makeresults | addinfo | eval EndDate = strftime(info_max_time, \"%e %b %Y %I:%M%p\") | table EndDate",
+        "queryParameters": {
+          "earliest": "$resource_tab_time.earliest$",
+          "latest": "$resource_tab_time.latest$"
+        }
+      }
+    },
+    "resource_tab_cpu_consumption_ds": {
+      "type": "ds.search",
+      "options": {
+        "query": "{{resource_cpu}}",
+        "queryParameters": {
+          "earliest": "$resource_tab_time.earliest$",
+          "latest": "$resource_tab_time.latest$"
+        }
+      }
+    },
+    "resource_tab_memory_consumption_ds": {
+      "type": "ds.search",
+      "options": {
+        "query": "{{resource_memory}}",
+        "queryParameters": {
+          "earliest": "$resource_tab_time.earliest$",
+          "latest": "$resource_tab_time.latest$"
+        }
+      }
+    }
+  },
+  "defaults": {},
+  "inputs": {
+    "resource_tab_input": {
+      "options": {
+        "defaultValue": "-24h,now",
+        "token": "resource_tab_time"
+      },
+      "title": "Time",
+      "type": "input.timerange"
+    }
+  },
+  "layout": {
+    "type": "grid",
+    "globalInputs": [
+      "resource_tab_input"
+    ],
+    "structure": [
+      {
+        "item": "resource_tab_label_viz",
+        "position": {
+          "x": 20,
+          "y": 500,
+          "w": 300,
+          "h": 20
+        }
+      },
+      {
+        "item": "resource_tab_description_viz",
+        "position": {
+          "x": 20,
+          "y": 520,
+          "w": 500,
+          "h": 50
+        }
+      },
+      {
+        "item": "resource_tab_timerange_label_start_viz",
+        "position": {
+          "x": 20,
+          "y": 530,
+          "w": 100,
+          "h": 20
+        }
+      },
+      {
+        "item": "resource_tab_timerange_label_end_viz",
+        "position": {
+          "x": 120,
+          "y": 530,
+          "w": 100,
+          "h": 20
+        }
+      },
+      {
+        "item": "resource_tab_cpu_consumption_viz",
+        "position": {
+          "x": 20,
+          "y": 550,
+          "w": 620,
+          "h": 150
+        }
+      },
+      {
+        "item": "resource_tab_memory_consumption_viz",
+        "position": {
+          "x": 640,
+          "y": 550,
+          "w": 620,
+          "h": 150
+        }
+      }
+    ]
+  }
+}

tests/testdata/expected_addons/expected_files_conflict_test/expected_log.json

@@ -30,5 +30,6 @@
   "appserver/static/js/build/custom/data_ingestion_tab_definition.json             created\u001b[0m": "INFO",
   "appserver/static/js/build/custom/errors_tab_definition.json                     created\u001b[0m": "INFO",
   "appserver/static/js/build/custom/overview_definition.json                       created\u001b[0m": "INFO",
-  "appserver/static/js/build/custom/panels_to_display.json                         created\u001b[0m": "INFO"
+  "appserver/static/js/build/custom/panels_to_display.json                         created\u001b[0m": "INFO",
+  "appserver/static/js/build/custom/resources_tab_definition.json                  created\u001b[0m": "INFO"
 }

tests/unit/expected_results/data_ingestion_tab_definition.json

@@ -163,7 +163,7 @@
   "inputs": {
     "data_ingestion_input": {
       "options": {
-        "defaultValue": "-7d,now",
+        "defaultValue": "-24h,now",
         "token": "data_ingestion_time"
       },
       "title": "Time",
@@ -195,7 +195,7 @@
           },
           {
             "label": "Input",
-            "value": "index = _internal source=*splunk_ta_uccexample* action=events_ingested | stats latest(_time) as le, sparkline(sum(n_events)) as sparkevent, sum(n_events) as events by event_input | eval \"Last event\" = strftime(le, \"%e %b %Y %I:%M%p\") | table event_input, events, sparkevent, \"Last event\" | rename event_input as \"Input\", events as \"Number of events\", sparkevent as \"Event trendline\""
+            "value": "| rest splunk_server=local /services/data/inputs/all | where $eai:acl.app$ = \"Splunk_TA_UCCExample\" | eval Enabled=if(lower(disabled) IN (\"1\", \"true\", \"t\"), \"no\", \"yes\") | table title, Enabled | rename title as \"event_input\" | join type=left event_input [ search index = _internal source=*splunk_ta_uccexample* action=events_ingested | stats latest(_time) as le, sparkline(sum(n_events)) as sparkevent, sum(n_events) as events by event_input | eval \"Last event\" = strftime(le, \"%e %b %Y %I:%M%p\") ] | makemv delim=\",\" sparkevent | table event_input, Enabled, events, sparkevent, \"Last event\" | rename event_input as \"Input\", events as \"Number of events\", sparkevent as \"Event trendline\""
           }
         ],
         "defaultValue": "index=_internal source=*license_usage.log type=Usage (s IN (example_input_one*,example_input_two*)) | stats sparkline(sum(b)) as sparkvolume, sum(b) as Bytes by st | join type=left st [search index = _internal source=*splunk_ta_uccexample* action=events_ingested | stats latest(_time) AS le, sparkline(sum(n_events)) as sparkevent, sum(n_events) as events by sourcetype_ingested | rename sourcetype_ingested as st ] | makemv delim=\",\" sparkevent | eval \"Last event\" = strftime(le, \"%e %b %Y %I:%M%p\") | table st, Bytes, sparkvolume, events, sparkevent, \"Last event\" | rename st as \"Source type\", Bytes as \"Data volume\", events as \"Number of events\", sparkvolume as \"Volume trendline (Bytes)\", sparkevent as \"Event trendline\"",

tests/unit/expected_results/errors_tab_definition.json

@@ -42,9 +42,6 @@
       "options": {
         "xAxisVisibility": "hide",
         "xAxisTitleText": "Time",
-        "seriesColors": [
-          "#A870EF"
-        ],
         "yAxisTitleText": "Errors count"
       },
       "title": "Errors count",
@@ -106,7 +103,7 @@
   "inputs": {
     "errors_tab_input": {
       "options": {
-        "defaultValue": "-7d,now",
+        "defaultValue": "-24h,now",
         "token": "errors_tab_time"
       },
       "title": "Time",

tests/unit/expected_results/overview_definition.json

@@ -71,9 +71,6 @@
       "options": {
         "xAxisVisibility": "hide",
         "xAxisTitleText": "Time",
-        "seriesColors": [
-          "#A870EF"
-        ],
         "yAxisTitleText": "Error count"
       },
       "title": "Errors",
@@ -128,7 +125,7 @@
   "inputs": {
     "overview_input": {
       "options": {
-        "defaultValue": "-7d,now",
+        "defaultValue": "-24h,now",
         "token": "overview_time"
       },
       "title": "Time",