Skip to content

Commit

Permalink
feat(dashboard): add resource monitoring tab and information about in…
Browse files Browse the repository at this point in the history
…put status (#1209)

**Issue
number:[ADDON-70514](https://splunk.atlassian.net/browse/ADDON-70514)**

## Summary
Extending monitoring dashboard page with resource monitoring and adding
information about disabled inputs to the table.

### Changes

* added new definition.json for resource consumption
* changed query for the table for "input" item in the dropdown

### User experience

* a new resource monitoring page is available
* in the table when filtered by inputs, a column has been added
informing whether the input is active or not

## Checklist

If your change doesn't seem to apply, please leave them unchecked.

* [x] I have performed a self-review of this change
* [x] Changes have been tested
* [x] Changes are documented
* [x] PR title follows [conventional commit
semantics](https://www.conventionalcommits.org/en/v1.0.0/)

---------

Co-authored-by: Szymon Oleksy <soleksy@splunk.com>
  • Loading branch information
sgoral-splunk and soleksy-splunk committed Jun 11, 2024
1 parent 3b7b9e1 commit e8acfd9
Show file tree
Hide file tree
Showing 14 changed files with 623 additions and 191 deletions.
3 changes: 2 additions & 1 deletion docs/dashboard.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,12 @@ The dashboard page provides some additional information about the add-on
operations to increase the visibility into what the add-on is actually doing
under the hood.

As of now, 3 pre-built panels are supported:
As of now, 4 pre-built panels are supported:

* Overview
* Data ingestion
* Errors in the add-on.
* Resource consumption.

**IMPORTANT**: To fully use the panels available on the monitoring dashboard, use the `solnlib.log`'s [`events_ingested` function](https://github.com/splunk/addonfactory-solutions-library-python/blob/v4.14.0/solnlib/log.py#L253), available from **version 4.14**, to record events.

Expand Down
41 changes: 35 additions & 6 deletions splunk_add_on_ucc_framework/dashboard.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,15 @@
"overview": "overview_definition.json",
"data_ingestion_tab": "data_ingestion_tab_definition.json",
"errors_tab": "errors_tab_definition.json",
"resources_tab": "resources_tab_definition.json",
}

data_ingestion = (
"index=_internal source=*license_usage.log type=Usage "
"(s IN ({input_names})) | timechart sum(b) as Usage | "
'rename Usage as \\"Data volume\\"'
)

data_ingestion_and_events = (
"index=_internal source=*license_usage.log type=Usage "
"(s IN ({input_names})) | timechart sum(b) as Usage "
Expand Down Expand Up @@ -117,17 +119,30 @@
'| rename event_account as \\"Account\\", events as \\"Number of events\\", '
'sparkevent as \\"Event trendline\\"'
)

table_input_query = (
"index = _internal source=*{addon_name}* action=events_ingested "
'| rest splunk_server=local /services/data/inputs/all | where $eai:acl.app$ = \\"{addon_name}\\" '
'| eval Enabled=if(lower(disabled) IN (\\"1\\", \\"true\\", \\"t\\"), \\"no\\", \\"yes\\") '
'| table title, Enabled | rename title as \\"event_input\\" | join type=left event_input [ '
"search index = _internal source=*{addon_name_lowercase}* action=events_ingested "
"| stats latest(_time) as le, sparkline(sum(n_events)) as sparkevent, sum(n_events) as events by event_input "
'| eval \\"Last event\\" = strftime(le, \\"%e %b %Y %I:%M%p\\") '
'| table event_input, events, sparkevent, \\"Last event\\" '
'| rename event_input as \\"Input\\", events as \\"Number of events\\", '
'sparkevent as \\"Event trendline\\"'
'| eval \\"Last event\\" = strftime(le, \\"%e %b %Y %I:%M%p\\") ] | makemv delim=\\",\\" sparkevent '
'| table event_input, Enabled, events, sparkevent, \\"Last event\\" '
'| rename event_input as \\"Input\\", events as \\"Number of events\\", sparkevent as \\"Event trendline\\"'
)

errors_list_query = "index=_internal source=*{addon_name}* ERROR"

resource_cpu_query = (
"index = _introspection component=PerProcess data.args=*{addon_name}* "
'| timechart avg(data.pct_cpu) as \\"CPU (%)\\"'
)

resource_memory_query = (
"index=_introspection component=PerProcess data.args=*{addon_name}* "
'| timechart avg(data.pct_memory) as \\"Memory (%)\\"'
)


def generate_dashboard_content(
addon_name: str, input_names: List[str], definition_json_name: str
Expand Down Expand Up @@ -169,7 +184,9 @@ def generate_dashboard_content(
input_names=input_names_str, addon_name=addon_name.lower()
),
table_account=table_account_query.format(addon_name=addon_name.lower()),
table_input=table_input_query.format(addon_name=addon_name.lower()),
table_input=table_input_query.format(
addon_name=addon_name, addon_name_lowercase=addon_name.lower()
),
)
)

Expand All @@ -183,6 +200,18 @@ def generate_dashboard_content(
)
)

if definition_json_name == default_definition_json_filename["resources_tab"]:
content = (
utils.get_j2_env()
.get_template(definition_json_name)
.render(
resource_cpu=resource_cpu_query.format(addon_name=addon_name.lower()),
resource_memory=resource_memory_query.format(
addon_name=addon_name.lower()
),
)
)

return content


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@
"inputs": {
"data_ingestion_input": {
"options": {
"defaultValue": "-7d,now",
"defaultValue": "-24h,now",
"token": "data_ingestion_time"
},
"title": "Time",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,6 @@
"options": {
"xAxisVisibility": "hide",
"xAxisTitleText": "Time",
"seriesColors": [
"#A870EF"
],
"yAxisTitleText": "Errors count"
},
"title": "Errors count",
Expand Down Expand Up @@ -106,7 +103,7 @@
"inputs": {
"errors_tab_input": {
"options": {
"defaultValue": "-7d,now",
"defaultValue": "-24h,now",
"token": "errors_tab_time"
},
"title": "Time",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,6 @@
"options": {
"xAxisVisibility": "hide",
"xAxisTitleText": "Time",
"seriesColors": [
"#A870EF"
],
"yAxisTitleText": "Error count"
},
"title": "Errors",
Expand Down Expand Up @@ -128,7 +125,7 @@
"inputs": {
"overview_input": {
"options": {
"defaultValue": "-7d,now",
"defaultValue": "-24h,now",
"token": "overview_time"
},
"title": "Time",
Expand Down
186 changes: 186 additions & 0 deletions splunk_add_on_ucc_framework/templates/resources_tab_definition.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,186 @@
{
"visualizations": {
"resource_tab_label_viz": {
"type": "splunk.markdown",
"options": {
"markdown": "## Resource consumption",
"fontSize": "large"
}
},
"resource_tab_description_viz": {
"type": "splunk.markdown",
"options": {
"markdown": "View the resource consumption for your add-on.",
"customFontSize": 12,
"fontColor": "#6c7685"
}
},
"resource_tab_timerange_label_start_viz": {
"type": "splunk.singlevalue",
"options": {
"majorFontSize": 12,
"backgroundColor": "transparent",
"majorColor": "#9fa4af"
},
"dataSources": {
"primary": "resource_tab_data_time_label_start_ds"
}
},
"resource_tab_timerange_label_end_viz": {
"type": "splunk.singlevalue",
"options": {
"majorFontSize": 12,
"backgroundColor": "transparent",
"majorColor": "#9fa4af"
},
"dataSources": {
"primary": "resource_tab_data_time_label_end_ds"
}
},
"resource_tab_cpu_consumption_viz": {
"type": "splunk.line",
"options": {
"xAxisVisibility": "hide",
"xAxisTitleText": "Time",
"seriesColors": [
"#A870EF"
],
"yAxisTitleText": "CPU (%)"
},
"title": "CPU consumption",
"dataSources": {
"primary": "resource_tab_cpu_consumption_ds"
}
},
"resource_tab_memory_consumption_viz": {
"type": "splunk.line",
"options": {
"xAxisVisibility": "hide",
"xAxisTitleText": "Time",
"seriesColors": [
"#A870EF"
],
"yAxisTitleText": "Memory (%)"
},
"title": "Memory consumption",
"dataSources": {
"primary": "resource_tab_memory_consumption_ds"
}
}
},
"dataSources": {
"resource_tab_data_time_label_start_ds": {
"type": "ds.search",
"options": {
"query": "| makeresults | addinfo | eval StartDate = strftime(info_min_time, \"%e %b %Y %I:%M%p\") | table StartDate",
"queryParameters": {
"earliest": "$resource_tab_time.earliest$",
"latest": "$resource_tab_time.latest$"
}
}
},
"resource_tab_data_time_label_end_ds": {
"type": "ds.search",
"options": {
"query": "| makeresults | addinfo | eval EndDate = strftime(info_max_time, \"%e %b %Y %I:%M%p\") | table EndDate",
"queryParameters": {
"earliest": "$resource_tab_time.earliest$",
"latest": "$resource_tab_time.latest$"
}
}
},
"resource_tab_cpu_consumption_ds": {
"type": "ds.search",
"options": {
"query": "{{resource_cpu}}",
"queryParameters": {
"earliest": "$resource_tab_time.earliest$",
"latest": "$resource_tab_time.latest$"
}
}
},
"resource_tab_memory_consumption_ds": {
"type": "ds.search",
"options": {
"query": "{{resource_memory}}",
"queryParameters": {
"earliest": "$resource_tab_time.earliest$",
"latest": "$resource_tab_time.latest$"
}
}
}
},
"defaults": {},
"inputs": {
"resource_tab_input": {
"options": {
"defaultValue": "-24h,now",
"token": "resource_tab_time"
},
"title": "Time",
"type": "input.timerange"
}
},
"layout": {
"type": "grid",
"globalInputs": [
"resource_tab_input"
],
"structure": [
{
"item": "resource_tab_label_viz",
"position": {
"x": 20,
"y": 500,
"w": 300,
"h": 20
}
},
{
"item": "resource_tab_description_viz",
"position": {
"x": 20,
"y": 520,
"w": 500,
"h": 50
}
},
{
"item": "resource_tab_timerange_label_start_viz",
"position": {
"x": 20,
"y": 530,
"w": 100,
"h": 20
}
},
{
"item": "resource_tab_timerange_label_end_viz",
"position": {
"x": 120,
"y": 530,
"w": 100,
"h": 20
}
},
{
"item": "resource_tab_cpu_consumption_viz",
"position": {
"x": 20,
"y": 550,
"w": 620,
"h": 150
}
},
{
"item": "resource_tab_memory_consumption_viz",
"position": {
"x": 640,
"y": 550,
"w": 620,
"h": 150
}
}
]
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -30,5 +30,6 @@
"appserver/static/js/build/custom/data_ingestion_tab_definition.json created\u001b[0m": "INFO",
"appserver/static/js/build/custom/errors_tab_definition.json created\u001b[0m": "INFO",
"appserver/static/js/build/custom/overview_definition.json created\u001b[0m": "INFO",
"appserver/static/js/build/custom/panels_to_display.json created\u001b[0m": "INFO"
"appserver/static/js/build/custom/panels_to_display.json created\u001b[0m": "INFO",
"appserver/static/js/build/custom/resources_tab_definition.json created\u001b[0m": "INFO"
}
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@
"inputs": {
"data_ingestion_input": {
"options": {
"defaultValue": "-7d,now",
"defaultValue": "-24h,now",
"token": "data_ingestion_time"
},
"title": "Time",
Expand Down Expand Up @@ -195,7 +195,7 @@
},
{
"label": "Input",
"value": "index = _internal source=*splunk_ta_uccexample* action=events_ingested | stats latest(_time) as le, sparkline(sum(n_events)) as sparkevent, sum(n_events) as events by event_input | eval \"Last event\" = strftime(le, \"%e %b %Y %I:%M%p\") | table event_input, events, sparkevent, \"Last event\" | rename event_input as \"Input\", events as \"Number of events\", sparkevent as \"Event trendline\""
"value": "| rest splunk_server=local /services/data/inputs/all | where $eai:acl.app$ = \"Splunk_TA_UCCExample\" | eval Enabled=if(lower(disabled) IN (\"1\", \"true\", \"t\"), \"no\", \"yes\") | table title, Enabled | rename title as \"event_input\" | join type=left event_input [ search index = _internal source=*splunk_ta_uccexample* action=events_ingested | stats latest(_time) as le, sparkline(sum(n_events)) as sparkevent, sum(n_events) as events by event_input | eval \"Last event\" = strftime(le, \"%e %b %Y %I:%M%p\") ] | makemv delim=\",\" sparkevent | table event_input, Enabled, events, sparkevent, \"Last event\" | rename event_input as \"Input\", events as \"Number of events\", sparkevent as \"Event trendline\""
}
],
"defaultValue": "index=_internal source=*license_usage.log type=Usage (s IN (example_input_one*,example_input_two*)) | stats sparkline(sum(b)) as sparkvolume, sum(b) as Bytes by st | join type=left st [search index = _internal source=*splunk_ta_uccexample* action=events_ingested | stats latest(_time) AS le, sparkline(sum(n_events)) as sparkevent, sum(n_events) as events by sourcetype_ingested | rename sourcetype_ingested as st ] | makemv delim=\",\" sparkevent | eval \"Last event\" = strftime(le, \"%e %b %Y %I:%M%p\") | table st, Bytes, sparkvolume, events, sparkevent, \"Last event\" | rename st as \"Source type\", Bytes as \"Data volume\", events as \"Number of events\", sparkvolume as \"Volume trendline (Bytes)\", sparkevent as \"Event trendline\"",
Expand Down
5 changes: 1 addition & 4 deletions tests/unit/expected_results/errors_tab_definition.json
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,6 @@
"options": {
"xAxisVisibility": "hide",
"xAxisTitleText": "Time",
"seriesColors": [
"#A870EF"
],
"yAxisTitleText": "Errors count"
},
"title": "Errors count",
Expand Down Expand Up @@ -106,7 +103,7 @@
"inputs": {
"errors_tab_input": {
"options": {
"defaultValue": "-7d,now",
"defaultValue": "-24h,now",
"token": "errors_tab_time"
},
"title": "Time",
Expand Down
5 changes: 1 addition & 4 deletions tests/unit/expected_results/overview_definition.json
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,6 @@
"options": {
"xAxisVisibility": "hide",
"xAxisTitleText": "Time",
"seriesColors": [
"#A870EF"
],
"yAxisTitleText": "Error count"
},
"title": "Errors",
Expand Down Expand Up @@ -128,7 +125,7 @@
"inputs": {
"overview_input": {
"options": {
"defaultValue": "-7d,now",
"defaultValue": "-24h,now",
"token": "overview_time"
},
"title": "Time",
Expand Down
Loading

0 comments on commit e8acfd9

Please sign in to comment.