diff --git a/ansible/install/group_vars/all.yml b/ansible/install/group_vars/all.yml index 3a97910c9..ec4f12e5d 100644 --- a/ansible/install/group_vars/all.yml +++ b/ansible/install/group_vars/all.yml @@ -83,6 +83,13 @@ collectd_compute: false gnocchi_status_python_plugin: false gnocchi_status_interval: 30 +######################## +# tail plugin +######################## +# Determines if WARN/INFO messages are also counted +regex_warn: false +regex_info: false + ######################################## # Docker related # (use these if deploying graphite/carbon/grafana as containers) diff --git a/ansible/install/roles/collectd-openstack/templates/compute.collectd.conf.j2 b/ansible/install/roles/collectd-openstack/templates/compute.collectd.conf.j2 index 9ea1a8907..91fb5c622 100644 --- a/ansible/install/roles/collectd-openstack/templates/compute.collectd.conf.j2 +++ b/ansible/install/roles/collectd-openstack/templates/compute.collectd.conf.j2 @@ -128,8 +128,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} @@ -138,8 +154,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} Instance "neutron-l3-agent" @@ -147,8 +179,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} Instance "neutron-metadata-agent" @@ -156,17 +204,50 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" +{% endif %} + Instance "ceilometer-compute" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" +{% endif %} diff --git a/ansible/install/roles/collectd-openstack/templates/controller.collectd.conf.j2 b/ansible/install/roles/collectd-openstack/templates/controller.collectd.conf.j2 index e7eed0d6f..275c30ed7 100644 --- a/ansible/install/roles/collectd-openstack/templates/controller.collectd.conf.j2 +++ b/ansible/install/roles/collectd-openstack/templates/controller.collectd.conf.j2 @@ -125,20 +125,13 @@ PreCacheChain "PreCache" ProcessMatch "aodh-notifier" "python.+aodh-notifier" # 9,10 # Ceilometer (OpenStack Installed) - ProcessMatch "ceilometer-agent-central" "python.+ceilometer-agent-central" # -9,-10 - ProcessMatch "ceilometer-agent-compute" "python.+ceilometer-agent-compute" # -9,-10 ProcessMatch "ceilometer-agent-notification" "ceilometer-agent-notification" # 9,10 - ProcessMatch "ceilometer-alarm-evaluator" "python.+ceilometer-alarm-evaluator" # -9,-10 - ProcessMatch "ceilometer-alarm-notifier" "python.+ceilometer-alarm-notifier" # -9,-10 - ProcessMatch "ceilometer-api" "python.+ceilometer-api" # 9,10 + ProcessMatch "ceilometer-api" "python.+ceilometer-api" # 9,-10(httpd) ProcessMatch "ceilometer-collector" "ceilometer-collector" # 9,10 ProcessMatch "ceilometer-polling" "ceilometer-polling" # 9,10 # Ceph (OpenStack Installed) ProcessMatch "ceph-mon" "^/usr/bin/ceph-mon" # -9,-10 - ProcessMatch "ceph-osd" "^/usr/bin/ceph-osd" # -9,-10 - ProcessMatch "diamond" "python.+diamond" # -9,-10 - ProcessMatch "salt-minion" "python.+salt-minion" # -9,-10 # Cinder (OpenStack Installed) ProcessMatch "cinder-api" "python.+cinder-api" # 9,10 @@ -166,7 +159,6 @@ PreCacheChain "PreCache" ProcessMatch "mysqld" "/usr/libexec/mysqld" # 9,10 ProcessMatch "rabbitmq" "/usr/lib64/erlang/erts-.+/bin/beam.smp" # 9,10 Process "redis-server" # 9,10 - ProcessMatch "tuskar-api" "python.+tuskar-api" # -9,-10 # Heat (OpenStack Installed) ProcessMatch "heat-api" "python.+heat-api --config-file" # 9,10 @@ -175,7 +167,6 @@ PreCacheChain "PreCache" ProcessMatch "heat-engine" "python.+heat-engine" # 9,10 # Keystone (OpenStack Installed) - ProcessMatch "keystone-all" "python.+keystone-all" # -9,-10 ProcessMatch "keystone-admin" "keystone-admin.*-DFOREGROUND" # 9,10 ProcessMatch "keystone-main" "keystone-main.*-DFOREGROUND" # 9,10 ProcessMatch "keystone-token-flush" "keystone-manage.*token_flush" # 9,10 @@ -191,7 +182,6 @@ PreCacheChain "PreCache" # Nova (OpenStack Installed) ProcessMatch "nova-api" "python.+nova-api" # 9,10 ProcessMatch "nova-cert" "python.+nova-cert" # -9,-10 - ProcessMatch "nova-compute" "python.+nova-compute" # -9,-10 ProcessMatch "nova-conductor" "python.+nova-conductor" # 9,10 ProcessMatch "nova-consoleauth" "python.+nova-consoleauth" # 9,10 ProcessMatch "nova-novncproxy" "python.+nova-novncproxy" # 9,10 @@ -250,8 +240,24 @@ PreCacheChain "PreCache" Regex "[ERROR]" DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" +{%if regex_warn %} + + Regex "[Warning]" + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex "[Note]" + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} @@ -260,18 +266,236 @@ PreCacheChain "PreCache" Regex "ERROR REPORT" DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex "WARNING REPORT" + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex "INFO REPORT" + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} + + + + Instance "aodh-evaluator" + + Regex " ERROR " + DSType "CounterInc" + Type "counter" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} + + + Instance "aodh-listener" + + Regex " ERROR " + DSType "CounterInc" + Type "counter" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} + + + Instance "aodh-notifier" + + Regex " ERROR " + DSType "CounterInc" + Type "counter" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} + + + Instance "aodh-wsgi-api" + + Regex ":error" + DSType "CounterInc" + Type "counter" + Instance "error" + + Instance "heat-api-cfn" + + Regex " ERROR " + DSType "CounterInc" + Type "counter" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} + + + Instance "heat-api-cloudwatch" + + Regex " ERROR " + DSType "CounterInc" + Type "counter" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} + + + Instance "heat-api" + + Regex " ERROR " + DSType "CounterInc" + Type "counter" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} + + + Instance "heat-engine" + + Regex " ERROR " + DSType "CounterInc" + Type "counter" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} + + Instance "keystone" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} Instance "keystone-wsgi-admin" @@ -279,7 +503,7 @@ PreCacheChain "PreCache" Regex ":error" DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" @@ -288,19 +512,34 @@ PreCacheChain "PreCache" Regex ":error" DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" - Instance "nova-api" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" +{% endif %} Instance "nova-conductor" @@ -308,8 +547,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} Instance "nova-consoleauth" @@ -317,8 +572,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} Instance "nova-novncproxy" @@ -326,8 +597,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" +{% endif %} Instance "nova-scheduler" @@ -335,8 +622,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} @@ -345,8 +648,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} Instance "neutron-l3-agent" @@ -354,8 +673,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" +{% endif %} Instance "neutron-metadata-agent" @@ -363,8 +698,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" +{% endif %} Instance "neutron-openvswitch-agent" @@ -372,8 +723,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" +{% endif %} Instance "neutron-server" @@ -381,8 +748,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} @@ -391,8 +774,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" +{% endif %} Instance "cinder-scheduler" @@ -400,8 +799,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" +{% endif %} Instance "cinder-volume" @@ -409,8 +824,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" +{% endif %} @@ -419,8 +850,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" +{% endif %} Instance "glance-registry" @@ -428,8 +875,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} # Swift logs all into the same file @@ -439,20 +902,60 @@ PreCacheChain "PreCache" Regex "account-server: ERROR " DSType "CounterInc" Type "counter" - Instance "account-server" + Instance "account-server-error" Regex "container-server: ERROR " DSType "CounterInc" Type "counter" - Instance "container-server" + Instance "container-server-error" Regex "object-server: ERROR " DSType "CounterInc" Type "counter" - Instance "object-server" + Instance "object-server-error" + +{%if regex_warn %} + + Regex "account-server: WARNING " + DSType "CounterInc" + Type "counter" + Instance "account-server-warn" + + Regex "container-server: WARNING " + DSType "CounterInc" + Type "counter" + Instance "container-server-warn" + + + Regex "object-server: WARNING " + DSType "CounterInc" + Type "counter" + Instance "object-server-warn" + +{% endif %} +{%if regex_info %} + + Regex "account-server: INFO " + DSType "CounterInc" + Type "counter" + Instance "account-server-info" + + + Regex "container-server: INFO " + DSType "CounterInc" + Type "counter" + Instance "container-server-info" + + + Regex "object-server: INFO " + DSType "CounterInc" + Type "counter" + Instance "object-server-info" + +{% endif %} @@ -461,8 +964,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" +{% endif %} Instance "ceilometer-central" @@ -470,8 +989,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} Instance "ceilometer-collector" @@ -479,16 +1014,32 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" +{% endif %} Instance "ceilometer-wsgi-api" - Regex " ERROR " + Regex ":error" DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" @@ -498,8 +1049,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" +{% endif %} Instance "gnocchi-metricd" @@ -507,8 +1074,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" +{% endif %} Instance "gnocchi-statsd" @@ -516,8 +1099,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} Instance "gnocchi-wsgi-api" @@ -525,7 +1124,7 @@ PreCacheChain "PreCache" Regex ":error" DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" diff --git a/ansible/install/roles/collectd-openstack/templates/objectstorage.collectd.conf.j2 b/ansible/install/roles/collectd-openstack/templates/objectstorage.collectd.conf.j2 index c3b29c6d4..cef3d68ed 100644 --- a/ansible/install/roles/collectd-openstack/templates/objectstorage.collectd.conf.j2 +++ b/ansible/install/roles/collectd-openstack/templates/objectstorage.collectd.conf.j2 @@ -125,20 +125,60 @@ PreCacheChain "PreCache" Regex "account-server: ERROR " DSType "CounterInc" Type "counter" - Instance "account-server" + Instance "account-server-error" Regex "container-server: ERROR " DSType "CounterInc" Type "counter" - Instance "container-server" + Instance "container-server-error" Regex "object-server: ERROR " DSType "CounterInc" Type "counter" - Instance "object-server" + Instance "object-server-error" +{%if regex_warn %} + + Regex "account-server: WARNING " + DSType "CounterInc" + Type "counter" + Instance "account-server-warn" + + + Regex "container-server: WARNING " + DSType "CounterInc" + Type "counter" + Instance "container-server-warn" + + + Regex "object-server: WARNING " + DSType "CounterInc" + Type "counter" + Instance "object-server-warn" + +{% endif %} +{%if regex_info %} + + Regex "account-server: INFO " + DSType "CounterInc" + Type "counter" + Instance "account-server-info" + + + Regex "container-server: INFO " + DSType "CounterInc" + Type "counter" + Instance "container-server-info" + + + Regex "object-server: INFO " + DSType "CounterInc" + Type "counter" + Instance "object-server-info" + +{% endif %} diff --git a/ansible/install/roles/collectd-openstack/templates/undercloud.collectd.conf.j2 b/ansible/install/roles/collectd-openstack/templates/undercloud.collectd.conf.j2 index 3ebd26172..ce66f8bd5 100644 --- a/ansible/install/roles/collectd-openstack/templates/undercloud.collectd.conf.j2 +++ b/ansible/install/roles/collectd-openstack/templates/undercloud.collectd.conf.j2 @@ -103,14 +103,10 @@ PreCacheChain "PreCache" ProcessMatch "aodh-notifier" "python.+aodh-notifier" # 9,-10 # Ceilometer (OpenStack Installed) - ProcessMatch "ceilometer-agent-central" "python.+ceilometer-agent-central" # -9,-10 - ProcessMatch "ceilometer-agent-compute" "python.+ceilometer-agent-compute" # -9,-10 - ProcessMatch "ceilometer-agent-notification" "python.+ceilometer-agent-notification" # 9,-10 - ProcessMatch "ceilometer-alarm-evaluator" "python.+ceilometer-alarm-evaluator" # -9,-10 - ProcessMatch "ceilometer-alarm-notifier" "python.+ceilometer-alarm-notifier" # -9,-10 - ProcessMatch "ceilometer-api" "python.+ceilometer-api" # 9,-10 - ProcessMatch "ceilometer-collector" "python.+ceilometer-collector" # 9,-10 - ProcessMatch "ceilometer-polling" "python.+ceilometer-polling" # 9,-10 + ProcessMatch "ceilometer-agent-notification" "python.+ceilometer-agent-notification" # 9,10 + ProcessMatch "ceilometer-api" "python.+ceilometer-api" # 9,-10(httpd) + ProcessMatch "ceilometer-collector" "python.+ceilometer-collector" # 9,10 + ProcessMatch "ceilometer-polling" "python.+ceilometer-polling" # 9,10 # Collectd (Browbeat Installed) ProcessMatch "collectd" "/usr/sbin/collectd" @@ -132,12 +128,10 @@ PreCacheChain "PreCache" ProcessMatch "qemu-kvm" "/usr/libexec/qemu-kvm" # -9,-10 ProcessMatch "rabbitmq" "/usr/lib64/erlang/erts-.+/bin/beam.smp" # 9,10 Process "redis-server" # -9,-10 - ProcessMatch "tuskar-api" "python.+tuskar-api" # -9,-10 # Heat (OpenStack Installed) ProcessMatch "heat-api" "python.+heat-api" # 9,10 ProcessMatch "heat-api-cfn" "python.+heat-api-cfn" # 9,10 - ProcessMatch "heat-api-cloudwatch" "python.+heat-api-cloudwatch" # -9,-10 ProcessMatch "heat-engine" "python.+heat-engine" # 9,10 # Ironic (OpenStack Installed) @@ -148,7 +142,6 @@ PreCacheChain "PreCache" ProcessMatch "dnsmasq-ironicinspector" "/sbin/dnsmasq.+/etc/ironic-inspector/" # 9,10 # Keystone (OpenStack Installed) - ProcessMatch "keystone-all" "python.+keystone-all" # -9,-10 ProcessMatch "keystone-admin" "keystone-admin.*-DFOREGROUND" # 9,10 ProcessMatch "keystone-main" "keystone-main.*-DFOREGROUND" # 9,10 ProcessMatch "keystone-token-flush" "keystone-manage.*token_flush" # 9,10 @@ -213,8 +206,24 @@ PreCacheChain "PreCache" Regex "[ERROR]" DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" +{%if regex_warn %} + + Regex "[Warning]" + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex "[Note]" + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} @@ -223,8 +232,24 @@ PreCacheChain "PreCache" Regex "ERROR REPORT" DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex "WARNING REPORT" + DSType "CounterInc" + Type "counter" + Instance "warn" +{% endif %} +{%if regex_info %} + + Regex "INFO REPORT" + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} @@ -233,8 +258,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" +{% endif %} Instance "keystone-wsgi-admin" @@ -242,7 +283,7 @@ PreCacheChain "PreCache" Regex ":error" DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" @@ -251,7 +292,7 @@ PreCacheChain "PreCache" Regex ":error" DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" @@ -261,8 +302,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} Instance "nova-compute" @@ -270,8 +327,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} Instance "nova-conductor" @@ -279,8 +352,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" +{% endif %} Instance "nova-scheduler" @@ -288,8 +377,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} @@ -298,8 +403,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} Instance "neutron-openvswitch-agent" @@ -307,8 +428,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" +{% endif %} Instance "neutron-server" @@ -316,8 +453,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} @@ -326,8 +479,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} Instance "ironic-conductor" @@ -335,8 +504,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" +{% endif %} @@ -345,8 +530,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} @@ -355,8 +556,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} Instance "heat-api" @@ -364,8 +581,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" +{% endif %} Instance "heat-engine" @@ -373,8 +606,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} @@ -383,8 +632,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} Instance "glance-registry" @@ -392,8 +657,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" +{% endif %} @@ -402,8 +683,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} Instance "mistral-engine" @@ -411,8 +708,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} Instance "mistral-executor" @@ -420,8 +733,24 @@ PreCacheChain "PreCache" Regex " ERROR " DSType "CounterInc" Type "counter" - Instance "total" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" +{% endif %} # Swift logs all into the same file @@ -431,20 +760,60 @@ PreCacheChain "PreCache" Regex "account-server: ERROR " DSType "CounterInc" Type "counter" - Instance "account-server" + Instance "account-server-error" Regex "container-server: ERROR " DSType "CounterInc" Type "counter" - Instance "container-server" + Instance "container-server-error" Regex "object-server: ERROR " DSType "CounterInc" Type "counter" - Instance "object-server" + Instance "object-server-error" + +{%if regex_warn %} + + Regex "account-server: WARNING " + DSType "CounterInc" + Type "counter" + Instance "account-server-warn" + + + Regex "container-server: WARNING " + DSType "CounterInc" + Type "counter" + Instance "container-server-warn" + + + Regex "object-server: WARNING " + DSType "CounterInc" + Type "counter" + Instance "object-server-warn" + +{% endif %} +{%if regex_info %} + + Regex "account-server: INFO " + DSType "CounterInc" + Type "counter" + Instance "account-server-info" + + + Regex "container-server: INFO " + DSType "CounterInc" + Type "counter" + Instance "container-server-info" + + + Regex "object-server: INFO " + DSType "CounterInc" + Type "counter" + Instance "object-server-info" +{% endif %} diff --git a/ansible/install/roles/grafana-dashboards/templates/cloud_specific_log.json.j2 b/ansible/install/roles/grafana-dashboards/templates/cloud_specific_log.json.j2 index bd2663535..5b59d75e2 100644 --- a/ansible/install/roles/grafana-dashboards/templates/cloud_specific_log.json.j2 +++ b/ansible/install/roles/grafana-dashboards/templates/cloud_specific_log.json.j2 @@ -75,7 +75,7 @@ "targets": [ { "refId": "A", - "target": "aliasByNode(aliasSub(aliasSub(aliasSub({{dashboard_cloud_name}}.{{host}}.tail-*.*, 'counter-', ''), 'total', ''), 'tail-', ''), 2, 3)" + "target": "aliasByNode(consolidateBy(scale(aliasSub(aliasSub({{dashboard_cloud_name}}.{{host}}.tail-*.counter-*error, 'tail-', ''), 'counter-', ''), 10), 'max'), 2, 3)" } ], "timeFrom": null, @@ -100,9 +100,199 @@ {% endfor %} ], {% if 'undercloud' in group %} - "title": "{{group}} Logs", + "title": "{{group}} ERROR Rate in Logs", {% else %} - "title": "{{group}} ({{batched_hosts[0][0]}} - {{batched_hosts[-1][0]}}) Logs", + "title": "{{group}} ({{batched_hosts[0][0]}} - {{batched_hosts[-1][0]}}) ERROR Rate in Logs", + {% endif %} + "showTitle": true + {% endfor %} + {% endif %} +{% endfor %} +{% for group in dashboard_groups %} + {% if group in groups %} + {% for batched_hosts in groups[group]|hosts_to_dictionary|dictsort|batch(dashboards_batch) %} + {% if vars.initial == 0 %} + {% if vars.update({'initial': (vars.initial + 1)}) %} {% endif %} + {% else %} + }, + {% endif %} + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + {% for index, host in batched_hosts %} + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": 0, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + {% if vars.update({'panel_idx': (vars.panel_idx + 1)}) %} {% endif %} + "id": {{vars.panel_idx}}, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "aliasByNode(consolidateBy(scale(aliasSub(aliasSub({{dashboard_cloud_name}}.{{host}}.tail-*.counter-*warn, 'tail-', ''), 'counter-', ''), 10), 'max'), 2, 3)" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "{{host}} Tail", + "tooltip": { + "shared": true, + "value_type": "individual" + }, + "type": "graph", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "short", + "short" + ] + {% if loop.last %} + } + {% else %} + }, + {% endif %} + {% endfor %} + ], + {% if 'undercloud' in group %} + "title": "{{group}} WARN Rate in Logs", + {% else %} + "title": "{{group}} ({{batched_hosts[0][0]}} - {{batched_hosts[-1][0]}}) WARN Rate in Logs", + {% endif %} + "showTitle": true + {% endfor %} + {% endif %} +{% endfor %} +{% for group in dashboard_groups %} + {% if group in groups %} + {% for batched_hosts in groups[group]|hosts_to_dictionary|dictsort|batch(dashboards_batch) %} + {% if vars.initial == 0 %} + {% if vars.update({'initial': (vars.initial + 1)}) %} {% endif %} + {% else %} + }, + {% endif %} + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + {% for index, host in batched_hosts %} + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": 0, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + {% if vars.update({'panel_idx': (vars.panel_idx + 1)}) %} {% endif %} + "id": {{vars.panel_idx}}, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "aliasByNode(consolidateBy(scale(aliasSub(aliasSub({{dashboard_cloud_name}}.{{host}}.tail-*.counter-*info, 'tail-', ''), 'counter-', ''), 10), 'max'), 2, 3)" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "{{host}} Tail", + "tooltip": { + "shared": true, + "value_type": "individual" + }, + "type": "graph", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "short", + "short" + ] + {% if loop.last %} + } + {% else %} + }, + {% endif %} + {% endfor %} + ], + {% if 'undercloud' in group %} + "title": "{{group}} INFO Rate in Logs", + {% else %} + "title": "{{group}} ({{batched_hosts[0][0]}} - {{batched_hosts[-1][0]}}) INFO Rate in Logs", {% endif %} "showTitle": true {% endfor %} diff --git a/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.json.j2 b/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.json.j2 index df26c169e..f44b138c3 100644 --- a/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.json.j2 +++ b/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.json.j2 @@ -3876,12 +3876,144 @@ "targets": [ { "refId": "A", - "target": "aliasByNode(aliasSub(aliasSub(aliasSub($Cloud.$Node.tail-*.*, 'counter-', ''), 'total', ''), 'tail-', ''), 2, 3)" + "target": "aliasByNode(consolidateBy(scale(aliasSub(aliasSub($Cloud.$Node.tail-*.counter-*error, 'tail-', ''), 'counter-', ''), 10), 'max'), 2, 3)" } ], "timeFrom": null, "timeShift": null, - "title": "$Cloud - $Node - Tail", + "title": "$Cloud - $Node - ERROR Rate in logs", + "tooltip": { + "shared": true, + "value_type": "individual" + }, + "type": "graph", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "short", + "short" + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": 0, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + {% if vars.update({'panel_idx': (vars.panel_idx + 1)}) %} {% endif %} + "id": {{vars.panel_idx}}, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "aliasByNode(consolidateBy(scale(aliasSub(aliasSub($Cloud.$Node.tail-*.counter-*warn, 'tail-', ''), 'counter-', ''), 10), 'max'), 2, 3)" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "$Cloud - $Node - WARN Rate in logs", + "tooltip": { + "shared": true, + "value_type": "individual" + }, + "type": "graph", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "short", + "short" + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": 0, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + {% if vars.update({'panel_idx': (vars.panel_idx + 1)}) %} {% endif %} + "id": {{vars.panel_idx}}, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "aliasByNode(consolidateBy(scale(aliasSub(aliasSub($Cloud.$Node.tail-*.counter-*info, 'tail-', ''), 'counter-', ''), 10), 'max'), 2, 3)" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "$Cloud - $Node - INFO Rate in logs", "tooltip": { "shared": true, "value_type": "individual" @@ -3896,7 +4028,7 @@ } ], "showTitle": true, - "title": "Tail" + "title": "Logs" } ], "schemaVersion": 8,