support_template_idxc_non_clustered.xml

<form version="1.1" hideEdit="False" script="common_control.js, drilldown_action_extension.js, resource_usage_deployment_extension.js">
  <!-- Author: Rob Phillips, rphillips@splunk.com -->
  <label>support_template_idxc_non_clustered</label>
  <search>
    <query> | rest splunk_server_group=dmc_group_indexer  /services/server/info | fields + splunk_server  | eval host_list="\"".splunk_server."\""  | stats values(host_list) as host_list  | eval host_list=" (".mvjoin(host_list, ", ").")"</query>
    <done>
      <set token="host_list">$result.host_list$</set>
    </done>
  </search>
  <search id="hostwide_memory">
    <query>| tstats 
    $hostwide_agg_function$(data.mem_used) as mem_used latest(data.mem) as capacity
 where index=_introspection host IN $host_list$  sourcetype=splunk_resource_usage by host _time $span$
</query>
    <earliest>$time.earliest$</earliest>
    <latest>$time.latest$</latest>
  </search>
  <search id="hostwide_load_avg">
    <query>| tstats 
    $hostwide_agg_function$(data.normalized_load_avg_1min) as normalized_load_avg_1min 
 where index=_introspection  host IN $host_list$ sourcetype=splunk_resource_usage by  host _time $span$
</query>
    <earliest>$time.earliest$</earliest>
    <latest>$time.latest$</latest>
  </search>
  <search id="iostats_await">
    <query>| tstats $hostwide_agg_function$(data.avg_total_ms) as avg_total_ms where component::iostats index=_introspection host IN $host_list$ sourcetype=splunk_resource_usage
    by host data.mount_point _time $span$
    </query>
    <earliest>$time.earliest$</earliest>
    <latest>$time.latest$</latest>
  </search>
  <fieldset autoRun="true" submitButton="false">
    <input type="multiselect" searchWhenChanged="true" token="host">
      <label>Instance</label>
      <search>
        <query>| inputlookup dmc_assets 
| mvcombine search_group 

| where (search_group == "dmc_group_indexer") | table host</query>
      </search>
      <delimiter>, </delimiter>
      <fieldForLabel>host</fieldForLabel>
      <fieldForValue>host</fieldForValue>
      <prefix>(</prefix>
      <suffix>)</suffix>
      <valuePrefix>"</valuePrefix>
      <valueSuffix>"</valueSuffix>
    </input>
    <input type="time" searchWhenChanged="true" token="time">
      <label>Time Range:</label>
      <default>
        <earliest>-120m@m</earliest>
        <latest>now</latest>
      </default>
    </input>
    <input type="dropdown" token="span" searchWhenChanged="true">
      <label>span</label>
      <choice value="30s">30s</choice>
      <choice value="60s">1m</choice>
      <choice value="5m">5m</choice>
      <choice value="15m">15m</choice>
      <choice value="1h">1h</choice>
      <prefix>span=</prefix>
      <default>60s</default>
    </input>
    <input type="checkbox" token="checked_config" searchWhenChanged="true">
      <label></label>
      <choice value="yes">show server info</choice>
      <delimiter> </delimiter>
      <change>
        <condition value="yes">
          <eval token="checked_result_value">"A"</eval>
        </condition>
        <condition>
          <eval token="checked_result_value">"B"</eval>
        </condition>
      </change>
    </input>
    <input type="checkbox" token="checked_resource_usage" searchWhenChanged="true">
      <label></label>
      <choice value="yes">show resource usage</choice>
      <delimiter> </delimiter>
      <change>
        <condition value="yes">
          <eval token="checked_result_value">"A"</eval>
        </condition>
        <condition>
          <eval token="checked_result_value">"B"</eval>
        </condition>
      </change>
    </input>
  </fieldset>
  <row>
    <panel depends="$checked_config$">
      <table>
        <title>server info</title>
        <search>
          <query>| rest /services/server/sysinfo  splunk_server_group=dmc_group_indexer  | search splunk_server IN  $host_list$ | search splunk_server IN  $host$| join splunk_server [| rest /services/server/info  splunk_server_group=dmc_group_indexer  | search splunk_server IN $host_list$ |search splunk_server IN  $host$ ]   | table splunk_server version  numberOfCores, numberOfVirtualCores  physicalMemoryMB ulimits.open_files  transparent_hugepages.effective_state | sort +splunk_server</query>
          <earliest>$time.earliest$</earliest>
          <latest>$time.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="count">5</option>
        <option name="dataOverlayMode">none</option>
        <option name="drilldown">none</option>
        <option name="percentagesRow">false</option>
        <option name="refresh.display">progressbar</option>
        <option name="rowNumbers">true</option>
        <option name="totalsRow">false</option>
        <option name="wrap">true</option>
      </table>
    </panel>
  </row>
  <row>
    <panel depends="$checked_resource_usage$">
      <input type="dropdown" token="hostwide_agg_function" searchWhenChanged="true">
        <label>aggregation function</label>
        <choice value="max">Max</choice>
        <choice value="avg">Avg</choice>
        <choice value="median">Median</choice>
        <choice value="p90">90 percentile</choice>
        <default>p90</default>
        <initialValue>avg</initialValue>
      </input>
      <html>
        <h1>
          <b>
            <span style="color: #ED179F">Maximum Physical Memory Usage Deployment-Wide</span>  </b>

        </h1>

        <b>Description:</b> Shows maximum physical memory usage percentage across members of the selected group.
</html>
      <html>
        <p>
          <b>Recommended Action:</b>Review Monitoring Console Resource Usage Instance views to determine the processes consuming the most memory.</p>
          </html>
      <html>
        <p>Click chart for per host detail.</p>

      </html>
      <chart>
        <search base="hostwide_memory">
          <query>| timechart $span$ max(mem_used) as "memory_used" min(capacity) as "capacity_min" max(capacity) as "capacity_max"</query>
        </search>
        <option name="charting.axisTitleY.text">memory used MB</option>
        <option name="charting.axisY.minimumNumber">0</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.chart">area</option>
        <option name="charting.chart.overlayFields">capacity_min,capacity_max</option>
        <option name="charting.drilldown">all</option>
        <option name="charting.legend.placement">right</option>
        <option name="height">389</option>
        <option name="refresh.display">progressbar</option>
        <drilldown>
          <set token="m1">$click.value2$</set>
        </drilldown>
      </chart>
    </panel>
  </row>
  <row>
    <panel depends="$m1$">
      <chart>
        <title>$hostwide_agg_function$ Memory Usage MB - By Host</title>
        <search base="hostwide_memory">
          <query>| search  host IN $host$  | timechart $span$ max(mem_used) as "memory_used" latest(capacity) as "capacity" by host</query>
        </search>
        <option name="charting.axisTitleY.text">memory used MB</option>
        <option name="charting.axisY.minimumNumber">0</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.chart">line</option>
        <option name="charting.drilldown">all</option>
        <option name="charting.legend.placement">right</option>
        <option name="height">389</option>
        <option name="refresh.display">progressbar</option>
      </chart>
      <html>
    <a data-unset-token="m1">Close this panel</a>
  </html>
    </panel>
  </row>
  <row>
    <panel depends="$checked_resource_usage$">
      <html>
        <h1>
          <b>
            <span style="color: #ED179F">Maximum Load Average Deployment-Wide</span>  </b>

        </h1>

        <b>Description:</b> Shows maximum normalized 1m load avg based on physical cpu core count across all members of the selected group.
</html>
      <html>
        <p>
          <b>Recommended Action:</b>Sustained values above 1 are concerning (1=100% cpu system utilization), review Monitoring Console Resource Usage Instance views to determine the processes consuming the most cpu.</p>
          </html>
      <html>
        <p>Click chart for per host detail.</p>

      </html>
      <chart>
        <search base="hostwide_load_avg">
          <query>| search  host IN $host$  | timechart $span$ max(normalized_load_avg_1min) as "normalized_load_avg_1min" | eval ceiling=1</query>
        </search>
        <option name="charting.axisY.minimumNumber">0</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.chart">area</option>
        <option name="charting.chart.overlayFields">ceiling</option>
        <option name="charting.drilldown">all</option>
        <option name="charting.legend.placement">right</option>
        <option name="height">389</option>
        <option name="refresh.display">progressbar</option>
        <drilldown>
          <set token="c1">$click.value2$</set>
        </drilldown>
      </chart>
    </panel>
  </row>
  <row>
    <panel depends="$c1$">
      <chart>
        <title>$hostwide_agg_function$ Load Average - by host</title>
        <search base="hostwide_load_avg">
          <query>| search  host IN $host$  | timechart $span$ max(normalized_load_avg_1min) as "normalized_load_avg_1min" by host | eval ceiling=1</query>
        </search>
        <option name="charting.axisY.minimumNumber">0</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.chart">line</option>
        <option name="charting.chart.overlayFields">ceiling</option>
        <option name="charting.drilldown">all</option>
        <option name="charting.legend.placement">right</option>
        <option name="height">389</option>
        <option name="refresh.display">progressbar</option>
        <drilldown>
          <set token="p">$click.name2$</set>
        </drilldown>
      </chart>
      <html>
    <a data-unset-token="c1">Close this panel</a>
  </html>
    </panel>
  </row>
  <row>
    <panel depends="$p$">
      <title>Maximum CPU  usage by Splunk process class</title>
      <chart>
        <title>$p$</title>
        <search>
          <query>index=_introspection sourcetype=splunk_resource_usage component=PerProcess host=$p$
| eval process = 'data.process' 
| eval args = 'data.args' 
| eval pid = 'data.pid' 
| eval ppid = 'data.ppid' 
| eval elapsed = 'data.elapsed' 
| eval mem_used = 'data.mem_used' 
| eval mem = 'data.mem' 
| eval pct_memory = 'data.pct_memory' 
| eval normalized_pct_cpu = 'data.normalized_pct_cpu' 
| eval pct_cpu = 'data.pct_cpu' 
| eval sid = 'data.search_props.sid' 
| eval app = 'data.search_props.app' 
| eval label = 'data.search_props.label' 
| eval type = 'data.search_props.type' 
| eval mode = 'data.search_props.mode' 
| eval user = 'data.search_props.user' 
| eval role = 'data.search_props.role' 
| eval label = if(isnotnull('data.search_props.label'), 'data.search_props.label', "") 
| eval provenance = if(isnotnull('data.search_props.provenance'), 'data.search_props.provenance', "unknown") 
| eval search_head = case(isnotnull('data.search_props.search_head') AND 'data.search_props.role' == "peer", 'data.search_props.search_head', isnull('data.search_props.search_head') AND 'data.search_props.role' == "head", "_self", isnull('data.search_props.search_head') AND 'data.search_props.role' == "peer", "_unknown") 
| eval workload_pool = if(isnotnull('data.workload_pool'), 'data.workload_pool', "UNDEFINED") 
| eval process_class = case( process=="splunk-optimize","index service", process=="sh" OR process=="ksh" OR process=="bash" OR like(process,"python%") OR process=="powershell","scripted input", process=="mongod", "KVStore") 
| eval process_class = case( process=="splunkd" AND (like(args,"-p %start%") OR like(args,"service") OR like(args,"%_internal_launch_under_systemd%")),"splunkd server", process=="splunkd" AND isnotnull(sid),"search", process=="splunkd" AND (like(args,"fsck%") OR like(args,"recover-metadata%") OR like(args,"cluster_thing")),"index service", process=="splunkd" AND args=="instrument-resource-usage", "scripted input", (like(process,"python%") AND like(args,"%/appserver/mrsparkle/root.py%")) OR like(process,"splunkweb"),"Splunk Web", isnotnull(process_class), process_class) 
| eval process_class = if(isnull(process_class),"other",process_class) 
| bin _time span=10s 
| eval process = 'data.process' 
| eval args = 'data.args' 
| eval pid = 'data.pid' 
| eval ppid = 'data.ppid' 
| eval elapsed = 'data.elapsed' 
| eval mem_used = 'data.mem_used' 
| eval mem = 'data.mem' 
| eval pct_memory = 'data.pct_memory' 
| eval normalized_pct_cpu = 'data.normalized_pct_cpu' 
| eval pct_cpu = 'data.pct_cpu' 
| eval sid = 'data.search_props.sid' 
| eval app = 'data.search_props.app' 
| eval label = 'data.search_props.label' 
| eval type = 'data.search_props.type' 
| eval mode = 'data.search_props.mode' 
| eval user = 'data.search_props.user' 
| eval role = 'data.search_props.role' 
| eval label = if(isnotnull('data.search_props.label'), 'data.search_props.label', "") 
| eval provenance = if(isnotnull('data.search_props.provenance'), 'data.search_props.provenance', "unknown") 
| eval search_head = case(isnotnull('data.search_props.search_head') AND 'data.search_props.role' == "peer", 'data.search_props.search_head', isnull('data.search_props.search_head') AND 'data.search_props.role' == "head", "_self", isnull('data.search_props.search_head') AND 'data.search_props.role' == "peer", "_unknown") 
| eval workload_pool = if(isnotnull('data.workload_pool'), 'data.workload_pool', "UNDEFINED") 
| stats latest(pct_cpu) AS resource_usage_dedup latest(process_class) AS process_class by pid, _time 
| stats sum(resource_usage_dedup) AS resource_usage by _time, process_class 
| timechart span=1m Max(resource_usage) AS "Resource Usage" by process_class</query>
          <earliest>$time.earliest$</earliest>
          <latest>$time.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">column</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">stacked</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
      <html>
    <a data-unset-token="p">Close this panel</a>
  </html>
    </panel>
    <panel depends="$p$">
      <title>Maximum CPU Resource Usage of Searches</title>
      <chart>
        <title>$p$</title>
        <search>
          <query>index=_introspection host=$p$ sourcetype=splunk_resource_usage component=PerProcess data.search_props.sid::*
        | `dmc_rename_introspection_fields`
        | `dmc_set_bin`
        | stats latest(pct_cpu) AS resource_usage_dedup by _time, type, sid, pid
        | stats sum(resource_usage_dedup) AS sum_resource_usage by _time,  type
        | eval sum_resource_usage = if("pct_cpu" == "pct_cpu", round(sum_resource_usage / 100.0, 2), sum_resource_usage)
        | timechart $span$ Max(sum_resource_usage) AS "Max of resource usage" by type</query>
          <earliest>$time.earliest$</earliest>
          <latest>$time.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">column</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">stacked</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
      <html>
    <a data-unset-token="p">Close this panel</a>
  </html>
    </panel>
  </row>
  <row>
    <panel depends="$p$">
      <title>max cpu pct by sid , label</title>
      <table>
        <title>$p$</title>
        <search>
          <query>index=_introspection host=$p$ sourcetype=splunk_resource_usage component=PerProcess data.search_props.sid::* 
| rename data.search_props.label as label| fillnull value="ad-hoc" label | stats max(data.pct_cpu) as data.pct_cpu by data.search_props.sid label  data.search_props.app data.search_props.provenance data.search_props.user  | sort -data.pct_cpu</query>
          <earliest>$time.earliest$</earliest>
          <latest>$time.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="count">5</option>
        <option name="dataOverlayMode">none</option>
        <option name="drilldown">none</option>
        <option name="percentagesRow">false</option>
        <option name="refresh.display">progressbar</option>
        <option name="rowNumbers">false</option>
        <option name="totalsRow">false</option>
        <option name="wrap">true</option>
      </table>
      <html>
    <a data-unset-token="p">Close this panel</a>
  </html>
    </panel>
  </row>
  <row>
    <panel depends="$checked_resource_usage$">
      <html>
        <h1>
          <b>
            <span style="color: #ED179F">Maximum Disk latency</span>  </b>

        </h1>

        <b>Description:</b> The average time (in milliseconds) for I/O requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.
</html>
      <html>
        <p>
          <b>Recommended Action:</b> Use to review Disk latency. Ideal await times for any mount point where Splunk operates is under 10ms</p>
          </html>
      <chart>
        <search base="iostats_await">
          <query>| search  host IN $host$ | timechart $span$ max(avg_total_ms) as "avg_total_ms" | eval ideal_latency_ms=10</query>
        </search>
        <option name="charting.axisTitleY.text">disk await (ms)</option>
        <option name="charting.axisY.minimumNumber">0</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.chart">area</option>
        <option name="charting.chart.nullValueMode">connect</option>
        <option name="charting.chart.overlayFields">ideal_latency_ms</option>
        <option name="charting.drilldown">all</option>
        <option name="charting.legend.placement">right</option>
        <option name="height">389</option>
        <option name="refresh.display">progressbar</option>
        <drilldown>
          <set token="d1">$click.value2$</set>
        </drilldown>
      </chart>
    </panel>
  </row>
  <row>
    <panel depends="$d1$">
      <chart>
        <title>$hostwide_agg_function$ Disk Latency - by host</title>
        <search base="iostats_await">
          <query>| search  host IN $host$| timechart $span$ max(avg_total_ms) as "avg_total_ms" by host  | eval ideal_latency_ms=10</query>
        </search>
        <option name="charting.axisTitleY.text">disk await (ms)</option>
        <option name="charting.axisY.minimumNumber">0</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.chart">line</option>
        <option name="charting.chart.nullValueMode">connect</option>
        <option name="charting.chart.overlayFields">ideal_latency_ms</option>
        <option name="charting.drilldown">all</option>
        <option name="charting.legend.placement">right</option>
        <option name="height">389</option>
        <option name="refresh.display">progressbar</option>
        <drilldown>
          <set token="d2">$click.name2$</set>
        </drilldown>
      </chart>
      <html>
    <a data-unset-token="d1">Close this panel</a>
  </html>
    </panel>
  </row>
  <row>
    <panel depends="$d2$">
      <chart>
        <title>$hostwide_agg_function$ Disk Latency - by mount point</title>
        <search base="iostats_await">
          <query> | search host="$d2$" | timechart $span$ max(avg_total_ms) as "avg_total_ms" by data.mount_point  | eval ideal_latency_ms=10</query>
        </search>
        <option name="charting.axisTitleY.text">disk await (ms)</option>
        <option name="charting.axisY.minimumNumber">0</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.chart">line</option>
        <option name="charting.chart.nullValueMode">connect</option>
        <option name="charting.chart.overlayFields">ideal_latency_ms</option>
        <option name="charting.drilldown">all</option>
        <option name="charting.legend.placement">right</option>
        <option name="height">389</option>
        <option name="refresh.display">progressbar</option>
      </chart>
      <html>
    <a data-unset-token="d2">Close this panel</a>
  </html>
    </panel>
  </row>
  <row>
    <panel>
      <title>Blocked Queues</title>
      <chart>
        <search>
          <query>index=_internal search_group="dmc_group_indexer"  host IN $host$  sourcetype=splunkd TERM(group=queue) 
                       TERM(blocked=true)  | timechart $span$ minspan=30s count by host useother=f limit=0</query>
          <earliest>$time.earliest$</earliest>
          <latest>$time.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">column</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">stacked</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">all</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
        <drilldown>
          <set token="h1">$click.name2$</set>
        </drilldown>
      </chart>
    </panel>
  </row>
  <row>
    <panel depends="$h1$">
      <html>
    <a data-unset-token="h1">Close this panel</a>
  </html>
      <chart>
        <title>blocked queues for $h1$ by queue name</title>
        <search>
          <query>index=_internal search_group="dmc_group_indexer"  host IN $host$  host=$h1$  sourcetype=splunkd  TERM(group=queue) 
                       TERM(blocked=true)  | timechart span=60s minspan=30s count by name useother=f limit=0</query>
          <earliest>$time.earliest$</earliest>
          <latest>$time.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">column</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">stacked</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
  </row>
  <row>
    <panel>
      <title>active searches</title>
      <chart>
        <search>
          <query>| tstats
max(PREFIX(active_searches=)) as active_searches
where
index::_internal
host IN $host_list$ host IN $host$ sourcetype::splunkd_remote_searches
by host _time span=30s
| timechart span=30s max(active_searches) by host limit=0 useother=f</query>
          <earliest>$time.earliest$</earliest>
          <latest>$time.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">line</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.showDataLabels">minmax</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
    <panel>
      <title>RT searches</title>
      <chart>
        <title>distinct count of real time searches</title>
        <search>
          <query>| tstats 
   dc(data.search_props.sid) as distinct_search_count
 where index=_introspection host IN $host_list$ host IN $host$ sourcetype=splunk_resource_usage component=PerProcess "data.search_props.sid"=* "data.search_props.mode"=RT* by host _time span=30s  | timechart  bins=200 max(distinct_search_count) AS "max rt search concurrency"</query>
          <earliest>$time.earliest$</earliest>
          <latest>$time.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">collapsed</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">area</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">zero</option>
        <option name="charting.chart.showDataLabels">minmax</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
  </row>
  <row>
    <panel>
      <title>thread count by host</title>
      <chart>
        <search>
          <query>| tstats
max(data.t_count) as data.t_count
where index::_introspection host IN $host_list$ host IN $host$  sourcetype::splunk_resource_usage  by host _time span=60s
| timechart span=60s max(data.t_count) as "thread count"  by host limit=0 | eval danger=500</query>
          <earliest>$time.earliest$</earliest>
          <latest>$time.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.text">thread count</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">line</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">all</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
        <drilldown>
          <set token="f0">$click.name2$</set>
        </drilldown>
      </chart>
    </panel>
  </row>
  <row>
    <panel depends="$f0$">
      <title>thread count by process type</title>
      <chart>
        <title>host: $f0$</title>
        <search>
          <query>| tstats
max(data.t_count) as data.t_count
where index::_introspection host=$f0$  sourcetype::splunk_resource_usage  by data.process_type _time span=60s
| timechart span=60s max(data.t_count) as "thread count"  by data.process_type limit=0 | eval danger=500</query>
          <earliest>$time.earliest$</earliest>
          <latest>$time.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.text">thread count</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">column</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.overlayFields">danger</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">stacked</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
      <html>
    <a data-unset-token="f0">Close this panel</a>
  </html>
    </panel>
    <panel depends="$f0$">
      <title>watchdog.log busy threads</title>
      <chart>
        <title>host:$f0$</title>
        <search>
          <query>index=_internal host=$f0$ sourcetype=splunkd source=*watchdog.log* finally received after | rex "finally received after (?&lt;ms&gt;\d+)\s+ms" | timechart span=1m max(ms) by  thread_name limit=0</query>
          <earliest>$time.earliest$</earliest>
          <latest>$time.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.text">ms of unresponsive thread</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">line</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
      <html>
    <a data-unset-token="f0">Close this panel</a>
  </html>
    </panel>
  </row>
  <row>
    <panel>
      <title>data ingest balance</title>
      <input type="time" token="time2" searchWhenChanged="true">
        <label></label>
        <default>
          <earliest>-24h@h</earliest>
          <latest>now</latest>
        </default>
      </input>
      <table>
        <search>
          <query>index=_internal search_group="dmc_group_indexer"  host IN $host$ sourcetype=splunkd source=*metrics.log name=index_thruput  | stats sum(kb) as kb by host | eval MB= round(kb/1024,0) | eval GB= round(kb/1024/1024,0) | table host MB GB | sort -GB</query>
          <earliest>$time2.earliest$</earliest>
          <latest>$time2.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="count">10</option>
        <option name="dataOverlayMode">none</option>
        <option name="drilldown">none</option>
        <option name="percentagesRow">false</option>
        <option name="refresh.display">progressbar</option>
        <option name="rowNumbers">false</option>
        <option name="totalsRow">false</option>
        <option name="wrap">true</option>
      </table>
    </panel>
  </row>
  <row>
    <panel>
      <title>Throughput KB/s</title>
      <chart>
        <search>
          <query>index=_internal search_group="dmc_group_indexer"  host IN $host$  sourcetype=splunkd  source=*metrics.log TERM(name=thruput) | timechart span=1m partial=f per_second(kb) AS kbps by host 
| eval kbps=round(kbps,0)</query>
          <earliest>$time.earliest$</earliest>
          <latest>$time.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.text">KB/s</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">line</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
    <panel>
      <title>new channel creation</title>
      <chart>
        <search>
          <query>| tstats
sum(PREFIX(new_channels=)) as new_channels
where
index=_internal  host IN $host_list$ host IN $host$  sourcetype::splunkd
TERM(group=map) TERM(name=pipelineinputchannel)
by host _time span=30s
| timechart span=30s max(new_channels) by host limit=0 useother=f
| eval instance_channel_threshold=10000</query>
          <earliest>$time.earliest$</earliest>
          <latest>$time.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">area</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.overlayFields">instance_channel_threshold</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
  </row>
</form>