From ebba21c569eb3024dbf27a7c9ed3e43450c7845c Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Wed, 5 May 2021 10:04:14 +0200 Subject: [PATCH 001/292] setup.cfg: Replace dashes with underscores Setuptools v54.1.0 introduces a warning that the use of dash-separated options in 'setup.cfg' will not be supported in a future version [1]. Get ahead of the issue by replacing the dashes with underscores. Without this, we see 'UserWarning' messages like the following on new enough versions of setuptools: UserWarning: Usage of dash-separated 'description-file' will not be supported in future versions. Please use the underscore name 'description_file' instead [1] https://github.com/pypa/setuptools/commit/a2e9ae4cb Change-Id: I2920e8581adb4505c0d03c43d7f94eef9735d7ed --- setup.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index a7d4cb780..fda5c64d2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,11 +1,11 @@ [metadata] name = kayobe-config summary = Configuration for Kayobe -description-file = +description_file = README.rst author = OpenStack -author-email = openstack-discuss@lists.openstack.org -home-page = https://docs.openstack.org/kayobe/latest/ +author_email = openstack-discuss@lists.openstack.org +home_page = https://docs.openstack.org/kayobe/latest/ classifier = Environment :: OpenStack Intended Audience :: Information Technology From 9eb6a0cb2dfeadaf8de68da9cf03ab1955ed6ed8 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Thu, 13 May 2021 16:21:12 +0100 Subject: [PATCH 002/292] Synchronize kayobe-config A few late additions to the wallaby release. See: - https://docs.openstack.org/kayobe/latest/contributor/releases.html#synchronise-kayobe-config Change-Id: If74ad440a11d654829ac1d628ab3c0ef7aed18e1 --- etc/kayobe/bifrost.yml | 3 +-- etc/kayobe/docker-registry.yml | 3 +++ etc/kayobe/globals.yml | 4 ++++ etc/kayobe/inventory/groups | 8 ++++++- etc/kayobe/kolla.yml | 44 ++++++++++++++++++++++++++++++++++ etc/kayobe/time.yml | 23 ++++++++++++++++++ 6 files changed, 82 insertions(+), 3 deletions(-) diff --git a/etc/kayobe/bifrost.yml b/etc/kayobe/bifrost.yml index 8d20ebebe..91c8bcc56 100644 --- a/etc/kayobe/bifrost.yml +++ b/etc/kayobe/bifrost.yml @@ -21,8 +21,7 @@ # DIB base OS element. Default is {{ os_distribution }}. #kolla_bifrost_dib_os_element: -# DIB image OS release. Default is "focal" when os_distribution is "ubuntu", or -# "8" otherwise. +# DIB image OS release. Default is {{ os_release }}. #kolla_bifrost_dib_os_release: # List of default DIB elements. Default is ["disable-selinux", diff --git a/etc/kayobe/docker-registry.yml b/etc/kayobe/docker-registry.yml index 7ba79ed90..ea6ca9c84 100644 --- a/etc/kayobe/docker-registry.yml +++ b/etc/kayobe/docker-registry.yml @@ -14,6 +14,9 @@ # pull through cache. Default is an empty dict. #docker_registry_env: +# The network mode of the docker registry container. Default is 'host'. +#docker_registry_network_mode: + # The port on which the docker registry server should listen. Default is 4000. #docker_registry_port: diff --git a/etc/kayobe/globals.yml b/etc/kayobe/globals.yml index 64290d926..9efc114f6 100644 --- a/etc/kayobe/globals.yml +++ b/etc/kayobe/globals.yml @@ -49,6 +49,10 @@ # "centos". #os_distribution: +# OS release. Valid options are "8-stream" when os_distribution is "centos", or +# "focal" when os_distribution is "ubuntu". +#os_release: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/inventory/groups b/etc/kayobe/inventory/groups index a009693e3..fa1ced47c 100644 --- a/etc/kayobe/inventory/groups +++ b/etc/kayobe/inventory/groups @@ -42,7 +42,7 @@ storage compute ############################################################################### -# Docker groups. +# Service groups. [docker:children] # Hosts in this group will have Docker installed. @@ -59,6 +59,12 @@ compute # registries which may become unsynchronized. seed +[ntp:children] +# Kayobe will configure Chrony on members of this group. +seed +seed-hypervisor +overcloud + ############################################################################### # Baremetal compute node groups. diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index a08be4214..b43a3b907 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -121,11 +121,33 @@ # Full custom seed inventory contents. #kolla_seed_inventory_custom: +# List of names of default host variables to pass through from kayobe hosts to +# the kolla-ansible seed host, if set. See also +# kolla_seed_inventory_pass_through_host_vars_map. +#kolla_seed_inventory_pass_through_host_vars_default: + +# List of names of additional host variables to pass through from kayobe hosts +# to the kolla-ansible seed host, if set. See also +# kolla_seed_inventory_pass_through_host_vars_map. +#kolla_seed_inventory_pass_through_host_vars_extra: + # List of names of host variables to pass through from kayobe hosts to # the kolla-ansible seed host, if set. See also # kolla_seed_inventory_pass_through_host_vars_map. #kolla_seed_inventory_pass_through_host_vars: +# Dict mapping names of default variables in +# kolla_seed_inventory_pass_through_host_vars to the variable to use in +# kolla-ansible. If a variable name is not in this mapping the kayobe name is +# used. +#kolla_seed_inventory_pass_through_host_vars_map_default: + +# Dict mapping names of extra variables in +# kolla_seed_inventory_pass_through_host_vars to the variable to use in +# kolla-ansible. If a variable name is not in this mapping the kayobe name is +# used. +#kolla_seed_inventory_pass_through_host_vars_map_extra: + # Dict mapping names of variables in # kolla_seed_inventory_pass_through_host_vars to the variable to use in # kolla-ansible. If a variable name is not in this mapping the kayobe name is @@ -158,11 +180,33 @@ # have no hosts mapped to them will be provided with an empty group definition. #kolla_overcloud_inventory_kolla_top_level_groups: +# List of names of default host variables to pass through from kayobe hosts to +# kolla-ansible hosts, if set. See also +# kolla_overcloud_inventory_pass_through_host_vars_map. +#kolla_overcloud_inventory_pass_through_host_vars_default: + +# List of names of additional host variables to pass through from kayobe hosts +# to kolla-ansible hosts, if set. See also +# kolla_overcloud_inventory_pass_through_host_vars_map. +#kolla_overcloud_inventory_pass_through_host_vars_extra: + # List of names of host variables to pass through from kayobe hosts to # kolla-ansible hosts, if set. See also # kolla_overcloud_inventory_pass_through_host_vars_map. #kolla_overcloud_inventory_pass_through_host_vars: +# Dict mapping names of default variables in +# kolla_overcloud_inventory_pass_through_host_vars to the variable to use in +# kolla-ansible. If a variable name is not in this mapping the kayobe name is +# used. +#kolla_overcloud_inventory_pass_through_host_vars_map_default: + +# Dict mapping names of additional variables in +# kolla_overcloud_inventory_pass_through_host_vars to the variable to use in +# kolla-ansible. If a variable name is not in this mapping the kayobe name is +# used. +#kolla_overcloud_inventory_pass_through_host_vars_map_extra: + # Dict mapping names of variables in # kolla_overcloud_inventory_pass_through_host_vars to the variable to use in # kolla-ansible. If a variable name is not in this mapping the kayobe name is diff --git a/etc/kayobe/time.yml b/etc/kayobe/time.yml index c0a86d7c7..23c1d0dc4 100644 --- a/etc/kayobe/time.yml +++ b/etc/kayobe/time.yml @@ -7,6 +7,29 @@ # Name of the local timezone. #timezone: +############################################################################### +# Network Time Protocol (NTP). + +# List of NTP time sources to configure. Format is a list of dictionaries with +# the following keys: +# server: host or pool +# type: (Optional) Defaults to server. Maps to a time source in the +# configuration file. Can be one of server, peer, pool. +# options: (Optional) List of options that depends on type, see Chrony +# documentation for details. +# See: https://chrony.tuxfamily.org/doc/4.0/chrony.conf.html +# +# Example of configuring a pool and customising the pool specific maxsources +# option: +# chrony_ntp_servers: +# - server: pool.ntp.org +# type: pool +# options: +# - option: maxsources +# val: 3 +# +#chrony_ntp_servers: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes From 9024800a25fcdad68351f86a09a34ecbd796d377 Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Wed, 20 Oct 2021 08:42:20 +0200 Subject: [PATCH 003/292] Sync kayobe-config with kayobe changes Change-Id: I016af4a3d214c3f5e2a59c806a40281de024e713 --- etc/kayobe/apt.yml | 6 ++ etc/kayobe/bifrost.yml | 16 ++- etc/kayobe/compute.yml | 28 ++++++ etc/kayobe/controllers.yml | 28 ++++++ etc/kayobe/globals.yml | 11 +++ etc/kayobe/infra-vms.yml | 174 +++++++++++++++++++++++++++++++++ etc/kayobe/inspector.yml | 2 +- etc/kayobe/inventory/groups | 10 ++ etc/kayobe/ipa.yml | 2 +- etc/kayobe/kolla.yml | 17 ++-- etc/kayobe/monitoring.yml | 28 ++++++ etc/kayobe/openstack.yml | 4 +- etc/kayobe/seed-hypervisor.yml | 28 ++++++ etc/kayobe/seed.yml | 31 ++++++ etc/kayobe/storage.yml | 28 ++++++ 15 files changed, 401 insertions(+), 12 deletions(-) create mode 100644 etc/kayobe/infra-vms.yml diff --git a/etc/kayobe/apt.yml b/etc/kayobe/apt.yml index 552a116cf..5f278e322 100644 --- a/etc/kayobe/apt.yml +++ b/etc/kayobe/apt.yml @@ -5,6 +5,12 @@ # Apt cache TTL in seconds. Default is 3600. #apt_cache_valid_time: +# Apt proxy URL for HTTP. Default is empty (no proxy). +#apt_proxy_http: + +# Apt proxy URL for HTTPS. Default is {{ apt_proxy_http }}. +#apt_proxy_https: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/bifrost.yml b/etc/kayobe/bifrost.yml index 91c8bcc56..0bfcec726 100644 --- a/etc/kayobe/bifrost.yml +++ b/etc/kayobe/bifrost.yml @@ -11,6 +11,10 @@ # {{ openstack_branch }}. #kolla_bifrost_source_version: +# Whether Bifrost uses firewalld. Default value is false to avoid conflicting +# with iptables rules configured on the seed host by Kayobe. +#kolla_bifrost_use_firewalld: + # Firewalld zone used by Bifrost. Default is "trusted", to avoid blocking other # services running on the seed host. #kolla_bifrost_firewalld_internal_zone: @@ -39,8 +43,10 @@ # DIB init element. Default is "cloud-init-datasources". #kolla_bifrost_dib_init_element: -# DIB default environment variables. Default is {"DIB_CLOUD_INIT_DATASOURCES": -# "ConfigDrive", "DIB_DISABLE_KERNEL_CLEANUP": 1}. +# DIB default environment variables. Default is +# {DIB_BOOTLOADER_DEFAULT_CMDLINE: +# "nofb nomodeset gfxpayload=text net.ifnames=1", +# "DIB_CLOUD_INIT_DATASOURCES": "ConfigDrive"}. #kolla_bifrost_dib_env_vars_default: # DIB additional environment variables. Default is none. @@ -53,6 +59,12 @@ # List of DIB packages to install. Default is to install no extra packages. #kolla_bifrost_dib_packages: +############################################################################### +# Disk image deployment configuration. + +# Name of disk image file to deploy. Default is "deployment_image.qcow2". +#kolla_bifrost_deploy_image_filename: + ############################################################################### # Ironic configuration. diff --git a/etc/kayobe/compute.yml b/etc/kayobe/compute.yml index 59a68fa78..cd0ceb247 100644 --- a/etc/kayobe/compute.yml +++ b/etc/kayobe/compute.yml @@ -108,6 +108,13 @@ # Dict of sysctl parameters to set. #compute_sysctl_parameters: +############################################################################### +# Compute node tuned configuration. + +# Builtin tuned profile to use. Format is same as that used by giovtorres.tuned +# role. Default is virtual-host. +#compute_tuned_active_builtin_profile: + ############################################################################### # Compute node user configuration. @@ -115,6 +122,27 @@ # singleplatform-eng.users role. #compute_users: +############################################################################### +# Compute node firewalld configuration. + +# Whether to install and enable firewalld. +#compute_firewalld_enabled: + +# A list of zones to create. Each item is a dict containing a 'zone' item. +#compute_firewalld_zones: + +# A firewalld zone to set as the default. Default is unset, in which case the +# default zone will not be changed. +#compute_firewalld_default_zone: + +# A list of firewall rules to apply. Each item is a dict containing arguments +# to pass to the firewalld module. Arguments are omitted if not provided, with +# the following exceptions: +# - offline: true +# - permanent: true +# - state: enabled +#compute_firewalld_rules: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/controllers.yml b/etc/kayobe/controllers.yml index 6a4e45eeb..983251c6c 100644 --- a/etc/kayobe/controllers.yml +++ b/etc/kayobe/controllers.yml @@ -117,6 +117,13 @@ # Dict of sysctl parameters to set. #controller_sysctl_parameters: +############################################################################### +# Controller node tuned configuration. + +# Builtin tuned profile to use. Format is same as that used by giovtorres.tuned +# role. Default is throughput-performance. +#controller_tuned_active_builtin_profile: + ############################################################################### # Controller node user configuration. @@ -124,6 +131,27 @@ # singleplatform-eng.users role. #controller_users: +############################################################################### +# Controller node firewalld configuration. + +# Whether to install and enable firewalld. +#controller_firewalld_enabled: + +# A list of zones to create. Each item is a dict containing a 'zone' item. +#controller_firewalld_zones: + +# A firewalld zone to set as the default. Default is unset, in which case the +# default zone will not be changed. +#controller_firewalld_default_zone: + +# A list of firewall rules to apply. Each item is a dict containing arguments +# to pass to the firewalld module. Arguments are omitted if not provided, with +# the following exceptions: +# - offline: true +# - permanent: true +# - state: enabled +#controller_firewalld_rules: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/globals.yml b/etc/kayobe/globals.yml index 9efc114f6..a4150d8ec 100644 --- a/etc/kayobe/globals.yml +++ b/etc/kayobe/globals.yml @@ -53,6 +53,17 @@ # "focal" when os_distribution is "ubuntu". #os_release: +############################################################################### +# Ansible configuration. + +# Filter to apply to the setup module when gathering facts. Default is to not +# specify a filter. +#kayobe_ansible_setup_filter: + +# Gather subset to apply to the setup module when gathering facts. Default is +# to not specify a gather subset. +#kayobe_ansible_setup_gather_subset: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/infra-vms.yml b/etc/kayobe/infra-vms.yml new file mode 100644 index 000000000..4fb9cc162 --- /dev/null +++ b/etc/kayobe/infra-vms.yml @@ -0,0 +1,174 @@ +--- +############################################################################### +# Infrastructure VM configuration. + +# Name of the infra VM. +#infra_vm_name: + +# Memory in MB. +#infra_vm_memory_mb: + +# Number of vCPUs. +#infra_vm_vcpus: + +# List of volumes. +#infra_vm_volumes: + +# Root volume. +#infra_vm_root_volume: + +# Data volume. +#infra_vm_data_volume: + +# Name of the storage pool for the infra VM volumes. +#infra_vm_pool: + +# Capacity of the infra VM root volume. +#infra_vm_root_capacity: + +# Format of the infra VM root volume. +#infra_vm_root_format: + +# Base image for the infra VM root volume. Default is +# "https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img" +# when os_distribution is "ubuntu", or +# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20210210.0.x86_64.qcow2" +# otherwise. +#infra_vm_root_image: + +# Capacity of the infra VM data volume. +#infra_vm_data_capacity: + +# Format of the infra VM data volume. +#infra_vm_data_format: + +# List of network interfaces to attach to the infra VM. +#infra_vm_interfaces: + +# Hypervisor that the VM runs on. +#infra_vm_hypervisor: + +# Customise ansible_ssh_extra_args for the test that checks SSH connectivity +# after provisioning. Defaults to disabling ssh host key checking. +#infra_vm_wait_connection_ssh_extra_args: + +# OS family. Needed for config drive generation. +# infra_vm_os_family: + +############################################################################### +# Infrastructure VM node configuration. + +# User with which to access the infrastructure vm via SSH during bootstrap, in +# order to setup the Kayobe user account. +#infra_vm_bootstrap_user: + +############################################################################### +# Infrastructure VM network interface configuration. + +# List of networks to which infrastructure vm nodes are attached. +#infra_vm_network_interfaces: + +# List of default networks to which infrastructure vm nodes are attached. +#infra_vm_default_network_interfaces: + +# List of extra networks to which infrastructure vm nodes are attached. +#infra_vm_extra_network_interfaces: + +############################################################################### +# Infrastructure VM node software RAID configuration. + +# List of software RAID arrays. See mrlesmithjr.mdadm role for format. +#infra_vm_mdadm_arrays: + +############################################################################### +# Infrastructure VM node encryption configuration. + +# List of block devices to encrypt. See stackhpc.luks role for format. +#infra_vm_luks_devices: + +############################################################################### +# Infrastructure VM node LVM configuration. + +# List of infrastructure vm volume groups. See mrlesmithjr.manage-lvm role for +# format. +#infra_vm_lvm_groups: + +# Default list of infrastructure vm volume groups. See mrlesmithjr.manage-lvm +# role for format. +#infra_vm_lvm_groups_default: + +# Additional list of infrastructure vm volume groups. See mrlesmithjr.manage-lvm +# role for format. +#infra_vm_lvm_groups_extra: + +# Whether a 'data' LVM volume group should exist on the infrastructure vm. By +# default this contains a 'docker-volumes' logical volume for Docker volume +# storage. It will also be used for Docker container and image storage if +# 'docker_storage_driver' is set to 'devicemapper'. Default is true if +# 'docker_storage_driver' is set to 'devicemapper', or false otherwise. +#infra_vm_lvm_group_data_enabled: + +# Infrastructure VM LVM volume group for data. See mrlesmithjr.manage-lvm role +# for format. +#infra_vm_lvm_group_data: + +# List of disks for use by infrastructure vm LVM data volume group. Default to +# an invalid value to require configuration. +#infra_vm_lvm_group_data_disks: + +# List of LVM logical volumes for the data volume group. +#infra_vm_lvm_group_data_lvs: + +# Docker volumes LVM backing volume. +#infra_vm_lvm_group_data_lv_docker_volumes: + +# Size of docker volumes LVM backing volume. +#infra_vm_lvm_group_data_lv_docker_volumes_size: + +# Filesystem for docker volumes LVM backing volume. ext4 allows for shrinking. +#infra_vm_lvm_group_data_lv_docker_volumes_fs: + +############################################################################### +# Infrastructure VM node sysctl configuration. + +# Dict of sysctl parameters to set. +#infra_vm_sysctl_parameters: + +############################################################################### +# Infrastructure VM node tuned configuration. + +# Builtin tuned profile to use. Format is same as that used by giovtorres.tuned +# role. Default is virtual-guest. +#infra_vm_tuned_active_builtin_profile: + +############################################################################### +# Infrastructure VM node user configuration. + +# List of users to create. This should be in a format accepted by the +# singleplatform-eng.users role. +#infra_vm_users: + +############################################################################### +# Infrastructure VM node firewalld configuration. + +# Whether to install and enable firewalld. +#infra_vm_firewalld_enabled: + +# A list of zones to create. Each item is a dict containing a 'zone' item. +#infra_vm_firewalld_zones: + +# A firewalld zone to set as the default. Default is unset, in which case the +# default zone will not be changed. +#infra_vm_firewalld_default_zone: + +# A list of firewall rules to apply. Each item is a dict containing arguments +# to pass to the firewalld module. Arguments are omitted if not provided, with +# the following exceptions: +# - offline: true +# - permanent: true +# - state: enabled +#infra_vm_firewalld_rules: + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/inspector.yml b/etc/kayobe/inspector.yml index 9e5f05414..a35537392 100644 --- a/etc/kayobe/inspector.yml +++ b/etc/kayobe/inspector.yml @@ -103,7 +103,7 @@ # Ironic inspector rule to save introspection data to the node. #inspector_rule_save_data: -# List of default ironic insepctor rules. +# List of default ironic inspector rules. #inspector_rules_default: # List of additional ironic inspector rules. diff --git a/etc/kayobe/inventory/groups b/etc/kayobe/inventory/groups index fa1ced47c..16619fed8 100644 --- a/etc/kayobe/inventory/groups +++ b/etc/kayobe/inventory/groups @@ -14,6 +14,16 @@ # Build container images on the seed by default. seed +############################################################################### +# Infra VM groups. + +[hypervisors:children] +# Group that contains all hypervisors used for infra VMs +seed-hypervisor + +[infra-vms] +# Empty group to provide declaration of infra-vms group. + ############################################################################### # Overcloud groups. diff --git a/etc/kayobe/ipa.yml b/etc/kayobe/ipa.yml index 49236d13a..e0f2094f8 100644 --- a/etc/kayobe/ipa.yml +++ b/etc/kayobe/ipa.yml @@ -17,7 +17,7 @@ #ipa_builder_source_url: # Version of IPA builder source repository. Default is master. -#ipa_build_source_version: +#ipa_builder_source_version: # List of default Diskimage Builder (DIB) elements to use when building IPA # images. Default is ["centos", "enable-serial-console", diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index b43a3b907..a07406f01 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -65,7 +65,7 @@ # "ubuntu". Default is {{ os_distribution }}. #kolla_base_distro: -# Kolla container image type: binary or source. Default is 'binary'. +# Kolla container image type: binary or source. Default is 'source'. #kolla_install_type: # URL of docker registry to use for Kolla images. Default is not set, in which @@ -75,6 +75,11 @@ # Docker namespace to use for Kolla images. Default is 'kolla'. #kolla_docker_namespace: +# Whether docker should be configured to use an insecure registry for Kolla +# images. Default is false, unless docker_registry_enabled is true and +# docker_registry_enable_tls is false. +#kolla_docker_registry_insecure: + # Username to use to access a docker registry. Default is not set, in which # case the registry will be used without authentication. #kolla_docker_registry_username: @@ -258,14 +263,14 @@ #kolla_enable_ceilometer_ipmi: #kolla_enable_cells: #kolla_enable_central_logging: -#kolla_enable_chrony: +#kolla_enable_ceph_rgw: +#kolla_enable_ceph_rgw_loadbalancer: #kolla_enable_cinder: #kolla_enable_cinder_backend_hnas_nfs: #kolla_enable_cinder_backend_iscsi: #kolla_enable_cinder_backend_lvm: #kolla_enable_cinder_backend_nfs: #kolla_enable_cinder_backend_quobyte: -#kolla_enable_cinder_backend_zfssa_iscsi: #kolla_enable_cinder_backup: #kolla_enable_cinder_horizon_policy_file: #kolla_enable_cloudkitty: @@ -327,6 +332,7 @@ #kolla_enable_keystone_horizon_policy_file: #kolla_enable_kibana: #kolla_enable_kuryr: +#kolla_enable_loadbalancer: #kolla_enable_magnum: #kolla_enable_manila: #kolla_enable_manila_backend_cephfs_native: @@ -350,6 +356,7 @@ #kolla_enable_neutron_infoblox_ipam_agent: #kolla_enable_neutron_metering: #kolla_enable_neutron_mlnx: +#kolla_enable_neutron_packet_logging: #kolla_enable_neutron_port_forwarding: #kolla_enable_neutron_provider_networks: #kolla_enable_neutron_qos: @@ -371,7 +378,6 @@ #kolla_enable_outward_rabbitmq: #kolla_enable_ovn: #kolla_enable_ovs_dpdk: -#kolla_enable_panko: #kolla_enable_placement: #kolla_enable_prometheus: #kolla_enable_prometheus_alertmanager: @@ -379,6 +385,7 @@ #kolla_enable_prometheus_cadvisor: #kolla_enable_prometheus_ceph_mgr_exporter: #kolla_enable_prometheus_elasticsearch_exporter: +#kolla_enable_prometheus_fluentd_integration: #kolla_enable_prometheus_haproxy_exporter: #kolla_enable_prometheus_memcached_exporter: #kolla_enable_prometheus_mysqld_exporter: @@ -388,7 +395,6 @@ #kolla_enable_prometheus_server: #kolla_enable_qdrouterd: #kolla_enable_rabbitmq: -#kolla_enable_rally: #kolla_enable_redis: #kolla_enable_sahara: #kolla_enable_senlin: @@ -400,7 +406,6 @@ #kolla_enable_swift_s3api: #kolla_enable_tacker: #kolla_enable_telegraf: -#kolla_enable_tempest: #kolla_enable_trove: #kolla_enable_trove_singletenant: #kolla_enable_vitrage: diff --git a/etc/kayobe/monitoring.yml b/etc/kayobe/monitoring.yml index e28e5ccf1..f332ab938 100644 --- a/etc/kayobe/monitoring.yml +++ b/etc/kayobe/monitoring.yml @@ -81,6 +81,13 @@ # Dict of sysctl parameters to set. #monitoring_sysctl_parameters: +############################################################################### +# Monitoring node tuned configuration. + +# Builtin tuned profile to use. Format is same as that used by giovtorres.tuned +# role. Default is throughput-performance. +#monitoring_tuned_active_builtin_profile: + ############################################################################### # Monitoring node user configuration. @@ -88,6 +95,27 @@ # singleplatform-eng.users role. #monitoring_users: +############################################################################### +# Monitoring node firewalld configuration. + +# Whether to install and enable firewalld. +#monitoring_firewalld_enabled: + +# A list of zones to create. Each item is a dict containing a 'zone' item. +#monitoring_firewalld_zones: + +# A firewalld zone to set as the default. Default is unset, in which case the +# default zone will not be changed. +#monitoring_firewalld_default_zone: + +# A list of firewall rules to apply. Each item is a dict containing arguments +# to pass to the firewalld module. Arguments are omitted if not provided, with +# the following exceptions: +# - offline: true +# - permanent: true +# - state: enabled +#monitoring_firewalld_rules: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/openstack.yml b/etc/kayobe/openstack.yml index 52c43ca50..f2fe55aa1 100644 --- a/etc/kayobe/openstack.yml +++ b/etc/kayobe/openstack.yml @@ -2,10 +2,10 @@ ############################################################################### # OpenStack release configuration. -# Name of the current OpenStack release. Default is "wallaby". +# Name of the current OpenStack release. Default is "xena". #openstack_release: -# Name of the current OpenStack branch. Default is "stable/wallaby". +# Name of the current OpenStack branch. Default is "stable/xena". #openstack_branch: ############################################################################### diff --git a/etc/kayobe/seed-hypervisor.yml b/etc/kayobe/seed-hypervisor.yml index b14c82344..26cfbe594 100644 --- a/etc/kayobe/seed-hypervisor.yml +++ b/etc/kayobe/seed-hypervisor.yml @@ -97,6 +97,13 @@ # Dict of sysctl parameters to set. #seed_hypervisor_sysctl_parameters: +############################################################################### +# Seed hypervisor tuned configuration. + +# Builtin tuned profile to use. Format is same as that used by giovtorres.tuned +# role. Default is virtual-host. +#seed_hypervisor_tuned_active_builtin_profile: + ############################################################################### # Seed hypervisor user configuration. @@ -104,6 +111,27 @@ # singleplatform-eng.users role. #seed_hypervisor_users: +############################################################################### +# Seed hypervisor node firewalld configuration. + +# Whether to install and enable firewalld. +#seed_hypervisor_firewalld_enabled: + +# A list of zones to create. Each item is a dict containing a 'zone' item. +#seed_hypervisor_firewalld_zones: + +# A firewalld zone to set as the default. Default is unset, in which case the +# default zone will not be changed. +#seed_hypervisor_firewalld_default_zone: + +# A list of firewall rules to apply. Each item is a dict containing arguments +# to pass to the firewalld module. Arguments are omitted if not provided, with +# the following exceptions: +# - offline: true +# - permanent: true +# - state: enabled +#seed_hypervisor_firewalld_rules: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index 35f2aadaa..3a5186ea6 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -18,6 +18,9 @@ # List of extra networks to which seed nodes are attached. #seed_extra_network_interfaces: +# Whether to enable SNAT on seed nodes. Default is false. +#seed_enable_snat: + ############################################################################### # Seed node software RAID configuration. @@ -76,6 +79,13 @@ # Dict of sysctl parameters to set. #seed_sysctl_parameters: +############################################################################### +# Seed node tuned configuration. + +# Builtin tuned profile to use. Format is same as that used by giovtorres.tuned +# role. Default is virtual-guest. +#seed_tuned_active_builtin_profile: + ############################################################################### # Seed node user configuration. @@ -97,6 +107,27 @@ # #seed_containers: +############################################################################### +# Seed node firewalld configuration. + +# Whether to install and enable firewalld. +#seed_firewalld_enabled: + +# A list of zones to create. Each item is a dict containing a 'zone' item. +#seed_firewalld_zones: + +# A firewalld zone to set as the default. Default is unset, in which case the +# default zone will not be changed. +#seed_firewalld_default_zone: + +# A list of firewall rules to apply. Each item is a dict containing arguments +# to pass to the firewalld module. Arguments are omitted if not provided, with +# the following exceptions: +# - offline: true +# - permanent: true +# - state: enabled +#seed_firewalld_rules: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/storage.yml b/etc/kayobe/storage.yml index 47f63dbaa..535666c95 100644 --- a/etc/kayobe/storage.yml +++ b/etc/kayobe/storage.yml @@ -113,6 +113,13 @@ # Dict of sysctl parameters to set. #storage_sysctl_parameters: +############################################################################### +# Storage node tuned configuration. + +# Builtin tuned profile to use. Format is same as that used by giovtorres.tuned +# role. Default is throughput-performance. +#storage_tuned_active_builtin_profile: + ############################################################################### # Storage node user configuration. @@ -120,6 +127,27 @@ # singleplatform-eng.users role. #storage_users: +############################################################################### +# Storage node firewalld configuration. + +# Whether to install and enable firewalld. +#storage_firewalld_enabled: + +# A list of zones to create. Each item is a dict containing a 'zone' item. +#storage_firewalld_zones: + +# A firewalld zone to set as the default. Default is unset, in which case the +# default zone will not be changed. +#storage_firewalld_default_zone: + +# A list of firewall rules to apply. Each item is a dict containing arguments +# to pass to the firewalld module. Arguments are omitted if not provided, with +# the following exceptions: +# - offline: true +# - permanent: true +# - state: enabled +#storage_firewalld_rules: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes From f5b1ed4ea4dd8c220b2c79974a751cd753523c9e Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Mon, 25 Oct 2021 10:02:31 +0000 Subject: [PATCH 004/292] Update .gitreview for stable/xena Change-Id: I5db0ae4f963ee8e7a1e9eb873effc9a8434794d8 --- .gitreview | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitreview b/.gitreview index b9adc8dd9..9a46b9497 100644 --- a/.gitreview +++ b/.gitreview @@ -2,3 +2,4 @@ host=review.opendev.org port=29418 project=openstack/kayobe-config.git +defaultbranch=stable/xena From 574d748a088b52c72f2bccb3884b9b44575f14a1 Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Mon, 25 Oct 2021 10:02:34 +0000 Subject: [PATCH 005/292] Update TOX_CONSTRAINTS_FILE for stable/xena Update the URL to the upper-constraints file to point to the redirect rule on releases.openstack.org so that anyone working on this branch will switch to the correct upper-constraints list automatically when the requirements repository branches. Until the requirements repository has as stable/xena branch, tests will continue to use the upper-constraints list on master. Change-Id: I034c1fdb91313aa7d39ba792eeba8a7c196860db --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 47adec2e2..63e1b3fb2 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ skipsdist = True [testenv] install_command = pip install {opts} {packages} deps = - -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/xena} -r{toxinidir}/test-requirements.txt [testenv:pep8] From 9fe4b1add5dd8fcd3045648f1a2096f69d690de0 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 5 Nov 2021 14:59:16 +0100 Subject: [PATCH 006/292] Fix configuration sync for infra-vms Change-Id: I60c6a1839a54afbfa9c2c92bd5bbf5b247877e8f (cherry picked from commit ace9a4dbe1a7963b041f3edb3f12aee2319f5983) --- etc/kayobe/infra-vms.yml | 2 +- .../group_vars/infra-vms/ansible-python-interpreter | 3 +++ .../group_vars/infra-vms/network-interfaces | 12 ++++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 etc/kayobe/inventory/group_vars/infra-vms/ansible-python-interpreter create mode 100644 etc/kayobe/inventory/group_vars/infra-vms/network-interfaces diff --git a/etc/kayobe/infra-vms.yml b/etc/kayobe/infra-vms.yml index 4fb9cc162..c8aec1cd6 100644 --- a/etc/kayobe/infra-vms.yml +++ b/etc/kayobe/infra-vms.yml @@ -53,7 +53,7 @@ #infra_vm_wait_connection_ssh_extra_args: # OS family. Needed for config drive generation. -# infra_vm_os_family: +#infra_vm_os_family: ############################################################################### # Infrastructure VM node configuration. diff --git a/etc/kayobe/inventory/group_vars/infra-vms/ansible-python-interpreter b/etc/kayobe/inventory/group_vars/infra-vms/ansible-python-interpreter new file mode 100644 index 000000000..54abbf23c --- /dev/null +++ b/etc/kayobe/inventory/group_vars/infra-vms/ansible-python-interpreter @@ -0,0 +1,3 @@ +--- +# Use a virtual environment for remote operations. +ansible_python_interpreter: "{{ virtualenv_path }}/kayobe/bin/python" diff --git a/etc/kayobe/inventory/group_vars/infra-vms/network-interfaces b/etc/kayobe/inventory/group_vars/infra-vms/network-interfaces new file mode 100644 index 000000000..015413ded --- /dev/null +++ b/etc/kayobe/inventory/group_vars/infra-vms/network-interfaces @@ -0,0 +1,12 @@ +--- +############################################################################### +# Network interface definitions for the infra-vms group. + +# Overcloud provisioning network IP information. +# provision_oc_net_interface: +# provision_oc_net_bridge_ports: +# provision_oc_net_bond_slaves: + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes From 661f7c9fcef554deb7765fb4300ef045ef3d50a5 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Mon, 31 Jan 2022 12:49:14 +0100 Subject: [PATCH 007/292] Sync kayobe-config with kayobe changes Change-Id: Iedda041811602f42b51c080bb84c03a4b64a89fa --- etc/kayobe/bifrost.yml | 3 +++ etc/kayobe/ipa.yml | 2 +- etc/kayobe/seed-hypervisor.yml | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/bifrost.yml b/etc/kayobe/bifrost.yml index 0bfcec726..9cabe8121 100644 --- a/etc/kayobe/bifrost.yml +++ b/etc/kayobe/bifrost.yml @@ -86,6 +86,9 @@ #kolla_bifrost_inspector_port_addition: # List of extra kernel parameters for the inspector default PXE configuration. +# Default is {{ inspector_extra_kernel_options }}, defined in inspector.yml. +# When customising this variable, the default extra kernel parameters should be +# kept to retain full node inspection capabilities. #kolla_bifrost_inspector_extra_kernel_options: # List of introspection rules for Bifrost's Ironic Inspector service. diff --git a/etc/kayobe/ipa.yml b/etc/kayobe/ipa.yml index e0f2094f8..519ca2a59 100644 --- a/etc/kayobe/ipa.yml +++ b/etc/kayobe/ipa.yml @@ -16,7 +16,7 @@ # URL of IPA builder source repository. #ipa_builder_source_url: -# Version of IPA builder source repository. Default is master. +# Version of IPA builder source repository. Default is {{ openstack_branch }}. #ipa_builder_source_version: # List of default Diskimage Builder (DIB) elements to use when building IPA diff --git a/etc/kayobe/seed-hypervisor.yml b/etc/kayobe/seed-hypervisor.yml index 26cfbe594..ac72fcd3d 100644 --- a/etc/kayobe/seed-hypervisor.yml +++ b/etc/kayobe/seed-hypervisor.yml @@ -18,6 +18,9 @@ # List of extra networks to which seed hypervisor nodes are attached. #seed_hypervisor_extra_network_interfaces: +# Whether to enable SNAT on seed hypervisor node. Default is false. +#seed_hypervisor_enable_snat: + ############################################################################### # Seed hypervisor node software RAID configuration. From 27d534036971644667a21d22fbdc571b14f9ee75 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 6 Jun 2022 17:01:27 +0100 Subject: [PATCH 008/292] CI: Use ansible_facts for ci-aio and ci-builder environments --- .../ci-aio/inventory/group_vars/controllers/network-interfaces | 2 +- .../ci-builder/inventory/group_vars/seed/network-interfaces | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/environments/ci-aio/inventory/group_vars/controllers/network-interfaces b/etc/kayobe/environments/ci-aio/inventory/group_vars/controllers/network-interfaces index cae8e6a7d..24f050906 100644 --- a/etc/kayobe/environments/ci-aio/inventory/group_vars/controllers/network-interfaces +++ b/etc/kayobe/environments/ci-aio/inventory/group_vars/controllers/network-interfaces @@ -5,7 +5,7 @@ # Controller interface on all-in-one network. aio_interface: breth1 # Use dummy1 if it exists, otherwise the bridge will have no ports. -aio_bridge_ports: "{{ ['dummy1'] if 'ansible_dummy1' in hostvars[inventory_hostname] else [] }}" +aio_bridge_ports: "{{ ['dummy1'] if 'dummy1' in hostvars[inventory_hostname].ansible_facts else [] }}" ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/network-interfaces b/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/network-interfaces index 9abceec09..a583b7214 100644 --- a/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/network-interfaces +++ b/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/network-interfaces @@ -5,7 +5,7 @@ # Controller interface on all-in-one network. aio_interface: breth1 # Use dummy1 if it exists, otherwise the bridge will have no ports. -aio_bridge_ports: "{{ ['dummy1'] if 'ansible_dummy1' in hostvars[inventory_hostname] else [] }}" +aio_bridge_ports: "{{ ['dummy1'] if 'dummy1' in hostvars[inventory_hostname].ansible_facts else [] }}" ############################################################################### # Dummy variable to allow Ansible to accept this file. From 162bd42046d3037b0e2d330423745cbe95c669d5 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 6 Jun 2022 17:03:22 +0100 Subject: [PATCH 009/292] Xena: source is now default for kolla_install_type --- etc/kayobe/kolla.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 119062f22..d843b9b07 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -66,7 +66,7 @@ kolla_ansible_source_version: "{{ stackhpc_kolla_ansible_source_version }}" #kolla_base_distro: # Kolla container image type: binary or source. Default is 'source'. -kolla_install_type: source +#kolla_install_type: # URL of docker registry to use for Kolla images. Default is not set, in which # case Dockerhub will be used. From ee5c9968a275cbddfa31451ed20b87c5e0133201 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 7 Jun 2022 12:46:44 +0100 Subject: [PATCH 010/292] Xena: drop prometheus-server image It was dropped in xena, replaced by prometheus-v2-server. --- etc/kayobe/pulp.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index 38239a519..8e3fdd752 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -275,7 +275,6 @@ stackhpc_pulp_images: - prometheus-mysqld-exporter - prometheus-node-exporter - prometheus-openstack-exporter - - prometheus-server - prometheus-v2-server - rabbitmq - redis From b1bfd9843db98968265c91292c441a9dc5161cb9 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 7 Jun 2022 14:51:56 +0100 Subject: [PATCH 011/292] Xena: update README --- README.rst | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/README.rst b/README.rst index b61f56b4f..3e2f8c87b 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ StackHPC Kayobe Configuration ============================= -This repository provides a base Kayobe configuration for the Wallaby release +This repository provides a base Kayobe configuration for the Xena release of StackHPC OpenStack. StackHPC release train @@ -45,7 +45,7 @@ when used with Kayobe's `multiple environments feature. This configuration should be consumed using the `StackHPC Kayobe fork -`__, which includes +`__, which includes backported support for Ansible collections. New deployments @@ -56,7 +56,7 @@ your configuration. .. code-block:: console - git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/wallaby + git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/xena Existing deployments -------------------- @@ -68,7 +68,7 @@ need to merge the changes in this repository into your repository. git remote add stackhpc https://github.com/stackhpc/stackhpc-kayobe-config git fetch stackhpc - git merge stackhpc/stackhpc/wallaby + git merge stackhpc/stackhpc/xena Updating -------- @@ -80,7 +80,7 @@ consume these updates by merging in the changes with their local configuration. .. code-block:: console git fetch stackhpc - git merge stackhpc/stackhpc/wallaby + git merge stackhpc/stackhpc/xena The intention is to avoid merge conflicts where possible, but there may be cases where this is difficult. We are open to discussion on how best to @@ -125,13 +125,13 @@ Usage ===== The local Pulp service will be deployed as a `Seed custom container -`__ +`__ on next ``kayobe seed service deploy`` or ``kayobe seed service upgrade``. The following custom playbooks are provided in ``etc/kayobe/ansible/``: See the Kayobe `custom playbook documentation -`__ +`__ for information on how to run them. * ``pulp-repo-sync.yml``: Pull packages from Ark to the local Pulp. This will @@ -245,8 +245,8 @@ Clone the Kayobe and Kayobe configuration repositories (this one): cd mkdir -p src pushd src - git clone https://github.com/stackhpc/kayobe.git -b stackhpc/wallaby - git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/wallaby + git clone https://github.com/stackhpc/kayobe.git -b stackhpc/xena + git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/xena popd Create a virtual environment and install Kayobe: @@ -344,8 +344,8 @@ Clone the Kayobe and Kayobe configuration repositories (this one): cd mkdir -p src pushd src - git clone https://github.com/stackhpc/kayobe.git -b stackhpc/wallaby - git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/wallaby + git clone https://github.com/stackhpc/kayobe.git -b stackhpc/xena + git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/xena popd Create a virtual environment and install Kayobe: @@ -410,7 +410,7 @@ At this point you are ready to build and push some container images. kayobe seed container image build --push kayobe overcloud container image build --push -The container images are tagged as ``wallaby-``. This Kayobe +The container images are tagged as ``xena-``. This Kayobe configuration includes a hook that writes the tag to ``~/kolla_tag``, since it is not always simple to determine which tag was last applied to built images. @@ -422,7 +422,7 @@ tag as the value of the ``kolla_openstack_release`` variable. Resources ========= -* Kayobe documentation: https://docs.openstack.org/kayobe/wallaby/ +* Kayobe documentation: https://docs.openstack.org/kayobe/xena/ * Kayobe source: https://opendev.org/openstack/kayobe * Kayobe bugs: https://storyboard.openstack.org/#!/project/openstack/kayobe-config * IRC: #openstack-kolla From f2d41f2b5aa2cd503344d7e26d7b0ca88cfff97d Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 7 Jun 2022 14:55:35 +0100 Subject: [PATCH 012/292] Xena: update package repositories and Kolla image build config --- .../environments/ci-aio/stackhpc-ci.yml | 6 +++-- .../environments/ci-builder/stackhpc-ci.yml | 6 +++-- etc/kayobe/kolla.yml | 24 ++++++++++++------- etc/kayobe/pulp-repo-versions.yml | 7 +++--- etc/kayobe/stackhpc.yml | 22 +++++++++++------ 5 files changed, 42 insertions(+), 23 deletions(-) diff --git a/etc/kayobe/environments/ci-aio/stackhpc-ci.yml b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml index 4fa4bb20b..fc8819ae5 100644 --- a/etc/kayobe/environments/ci-aio/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml @@ -27,13 +27,15 @@ stackhpc_repo_epel_version: "{{ stackhpc_pulp_repo_epel_version }}" stackhpc_repo_epel_modular_version: "{{ stackhpc_pulp_repo_epel_modular_version }}" stackhpc_repo_docker_version: "{{ stackhpc_pulp_repo_docker_version }}" stackhpc_repo_centos_stream_advanced_virtualization_version: "{{ stackhpc_pulp_repo_centos_stream_8_advanced_virtualization_version }}" +stackhpc_repo_centos_stream_nfv_extras_version: "{{ stackhpc_pulp_repo_centos_stream_8_nfv_extras_version }}" stackhpc_repo_centos_stream_nfv_openvswitch_version: "{{ stackhpc_pulp_repo_centos_stream_8_nfv_openvswitch_version }}" -stackhpc_repo_centos_stream_openstack_wallaby_version: "{{ stackhpc_pulp_repo_centos_stream_8_openstack_wallaby_version }}" +stackhpc_repo_centos_stream_openstack_xena_version: "{{ stackhpc_pulp_repo_centos_stream_8_openstack_xena_version }}" +stackhpc_repo_centos_stream_opstools_version: "{{ stackhpc_pulp_repo_centos_stream_8_opstools_version }}" stackhpc_repo_centos_stream_powertools_version: "{{ stackhpc_pulp_repo_centos_stream_8_powertools_version }}" -stackhpc_repo_centos_opstools_version: "{{ stackhpc_pulp_repo_centos_8_opstools_version }}" stackhpc_repo_centos_stream_storage_ceph_pacific_version: "{{ stackhpc_pulp_repo_centos_stream_8_storage_ceph_pacific_version }}" stackhpc_repo_elasticsearch_logstash_kibana_7_x_version: "{{ stackhpc_pulp_repo_elasticsearch_logstash_kibana_7_x_version }}" stackhpc_repo_grafana_version: "{{ stackhpc_pulp_repo_grafana_version }}" +stackhpc_repo_mariadb_10_5_centos8_version: "{{ stackhpc_pulp_repo_mariadb_10_5_centos8_version }}" stackhpc_repo_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_rabbitmq_erlang_version }}" stackhpc_repo_rabbitmq_server_version: "{{ stackhpc_pulp_repo_rabbitmq_server_version }}" stackhpc_repo_treasuredata_4_version: "{{ stackhpc_pulp_repo_treasuredata_4_version }}" diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml index b7224e73b..b7aa7d80d 100644 --- a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml @@ -46,13 +46,15 @@ stackhpc_repo_epel_version: "{{ stackhpc_pulp_repo_epel_version }}" stackhpc_repo_epel_modular_version: "{{ stackhpc_pulp_repo_epel_modular_version }}" stackhpc_repo_docker_version: "{{ stackhpc_pulp_repo_docker_version }}" stackhpc_repo_centos_stream_advanced_virtualization_version: "{{ stackhpc_pulp_repo_centos_stream_8_advanced_virtualization_version }}" +stackhpc_repo_centos_stream_nfv_extras_version: "{{ stackhpc_pulp_repo_centos_stream_8_nfv_extras_version }}" stackhpc_repo_centos_stream_nfv_openvswitch_version: "{{ stackhpc_pulp_repo_centos_stream_8_nfv_openvswitch_version }}" -stackhpc_repo_centos_stream_openstack_wallaby_version: "{{ stackhpc_pulp_repo_centos_stream_8_openstack_wallaby_version }}" +stackhpc_repo_centos_stream_openstack_xena_version: "{{ stackhpc_pulp_repo_centos_stream_8_openstack_xena_version }}" +stackhpc_repo_centos_stream_opstools_version: "{{ stackhpc_pulp_repo_centos_stream_8_opstools_version }}" stackhpc_repo_centos_stream_powertools_version: "{{ stackhpc_pulp_repo_centos_stream_8_powertools_version }}" -stackhpc_repo_centos_opstools_version: "{{ stackhpc_pulp_repo_centos_8_opstools_version }}" stackhpc_repo_centos_stream_storage_ceph_pacific_version: "{{ stackhpc_pulp_repo_centos_stream_8_storage_ceph_pacific_version }}" stackhpc_repo_elasticsearch_logstash_kibana_7_x_version: "{{ stackhpc_pulp_repo_elasticsearch_logstash_kibana_7_x_version }}" stackhpc_repo_grafana_version: "{{ stackhpc_pulp_repo_grafana_version }}" +stackhpc_repo_mariadb_10_5_centos8_version: "{{ stackhpc_pulp_repo_mariadb_10_5_centos8_version }}" stackhpc_repo_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_rabbitmq_erlang_version }}" stackhpc_repo_rabbitmq_server_version: "{{ stackhpc_pulp_repo_rabbitmq_server_version }}" stackhpc_repo_treasuredata_4_version: "{{ stackhpc_pulp_repo_treasuredata_4_version }}" diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index d843b9b07..5dd859822 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -108,7 +108,7 @@ kolla_sources: barbican-base: type: git location: "https://github.com/stackhpc/barbican.git" - reference: stackhpc/wallaby + reference: stackhpc/xena bifrost-base: type: git location: "{{ kolla_bifrost_source_url }}" @@ -116,11 +116,11 @@ kolla_sources: cloudkitty-base: type: git location: https://github.com/stackhpc/cloudkitty.git - reference: stackhpc/wallaby + reference: stackhpc/xena horizon-plugin-cloudkitty-dashboard: type: git location: https://github.com/stackhpc/cloudkitty-dashboard.git - reference: stackhpc/wallaby + reference: stackhpc/xena ironic-inspector-additions-stackhpc-inspector-plugins: # Install our custom inspector plugins. type: git @@ -129,11 +129,11 @@ kolla_sources: magnum-base: type: git location: https://github.com/stackhpc/magnum.git - reference: stackhpc/wallaby + reference: stackhpc/xena neutron-base-plugin-networking-generic-switch: type: git location: https://github.com/stackhpc/networking-generic-switch.git - reference: stackhpc/wallaby + reference: stackhpc/xena ############################################################################### # Kolla image build configuration. @@ -158,14 +158,16 @@ stackhpc_epel_repos: stackhpc_centos_additional_repos: - url: "{{ stackhpc_repo_centos_stream_advanced_virtualization_url }}" file: "CentOS-Advanced-Virtualization.repo" + - url: "{{ stackhpc_repo_centos_stream_nfv_extras_url }}" + file: "CentOS-NFV-extras.repo" - url: "{{ stackhpc_repo_centos_stream_nfv_openvswitch_url }}" file: "CentOS-NFV-OpenvSwitch.repo" - - url: "{{ stackhpc_repo_centos_stream_openstack_wallaby_url }}" - file: "CentOS-OpenStack-wallaby.repo" + - url: "{{ stackhpc_repo_centos_stream_openstack_xena_url }}" + file: "CentOS-OpenStack-xena.repo" + - url: "{{ stackhpc_repo_centos_stream_opstools_url }}" + file: "CentOS-OpsTools.repo" - url: "{{ stackhpc_repo_centos_stream_powertools_url }}" file: "CentOS-Stream-PowerTools.repo" - - url: "{{ stackhpc_repo_centos_opstools_url }}" - file: "CentOS-OpsTools.repo" - url: "{{ stackhpc_repo_centos_stream_storage_ceph_pacific_url }}" file: "CentOS-Ceph-Pacific.repo" @@ -175,6 +177,8 @@ stackhpc_third_party_repos: file: "elasticsearch.repo" - url: "{{ stackhpc_repo_grafana_url }}" file: "grafana.repo" + - url: "{{ stackhpc_repo_mariadb_10_5_centos8_url }}" + file: "mariadb.repo" - url: "{{ stackhpc_repo_rabbitmq_erlang_url }}" file: "rabbitmq_rabbitmq-erlang.repo" - url: "{{ stackhpc_repo_rabbitmq_server_url }}" @@ -231,6 +235,8 @@ kolla_build_blocks: # customization is most commonly packages. The operation should be one of # override, append or remove. The value should be a list. kolla_build_customizations: + base_yum_repo_files_remove: + - proxysql.repo ironic_inspector_pip_packages_append: - /additions/* ovn_base_packages_override: diff --git a/etc/kayobe/pulp-repo-versions.yml b/etc/kayobe/pulp-repo-versions.yml index bbd51790a..97140035f 100644 --- a/etc/kayobe/pulp-repo-versions.yml +++ b/etc/kayobe/pulp-repo-versions.yml @@ -1,13 +1,13 @@ --- # Do not edit! This file is autogenerated by Ansible. -stackhpc_pulp_repo_centos_8_opstools_version: 20211122T102435 -stackhpc_pulp_repo_centos_8_storage_ceph_nautilus_version: 20211122T102435 stackhpc_pulp_repo_centos_stream_8_advanced_virtualization_version: 20211122T102435 stackhpc_pulp_repo_centos_stream_8_appstream_version: 20211122T102435 stackhpc_pulp_repo_centos_stream_8_baseos_version: 20211122T102435 stackhpc_pulp_repo_centos_stream_8_extras_version: 20211122T102435 +stackhpc_pulp_repo_centos_stream_8_nfv_extras_version: '' stackhpc_pulp_repo_centos_stream_8_nfv_openvswitch_version: 20220223T025353 -stackhpc_pulp_repo_centos_stream_8_openstack_wallaby_version: 20211122T102435 +stackhpc_pulp_repo_centos_stream_8_openstack_xena_version: '' +stackhpc_pulp_repo_centos_stream_8_opstools_version: '' stackhpc_pulp_repo_centos_stream_8_powertools_version: 20211122T102435 stackhpc_pulp_repo_centos_stream_8_storage_ceph_pacific_version: 20211210T124731 stackhpc_pulp_repo_docker_version: 20211122T102435 @@ -15,6 +15,7 @@ stackhpc_pulp_repo_elasticsearch_logstash_kibana_7_x_version: 20211122T102435 stackhpc_pulp_repo_epel_modular_version: 20211122T102435 stackhpc_pulp_repo_epel_version: 20211122T102435 stackhpc_pulp_repo_grafana_version: 20220119T030911 +stackhpc_pulp_repo_mariadb_10_5_centos8_version: '' stackhpc_pulp_repo_rabbitmq_erlang_version: 20211122T102435 stackhpc_pulp_repo_rabbitmq_server_version: 20211122T102435 stackhpc_pulp_repo_treasuredata_4_version: 20211122T102435 diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml index b02a23caf..6e5dc42a2 100644 --- a/etc/kayobe/stackhpc.yml +++ b/etc/kayobe/stackhpc.yml @@ -33,22 +33,26 @@ stackhpc_repo_docker_version: "{{ stackhpc_repo_distribution }}" stackhpc_repo_centos_stream_advanced_virtualization_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/centos/8-stream/virt/x86_64/advancedvirt-common/{{ stackhpc_repo_centos_stream_advanced_virtualization_version }}" stackhpc_repo_centos_stream_advanced_virtualization_version: "{{ stackhpc_repo_distribution }}" +# CentOS Stream 8 - NFV Extras +stackhpc_repo_centos_stream_nfv_extras_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/centos/8-stream/nfv/x86_64/network-extras/{{ stackhpc_repo_centos_stream_nfv_extras_version }}" +stackhpc_repo_centos_stream_nfv_extras_version: "{{ stackhpc_repo_distribution }}" + # CentOS Stream 8 - NFV OpenvSwitch stackhpc_repo_centos_stream_nfv_openvswitch_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/centos/8-stream/nfv/x86_64/openvswitch-2/{{ stackhpc_repo_centos_stream_nfv_openvswitch_version }}" stackhpc_repo_centos_stream_nfv_openvswitch_version: "{{ stackhpc_repo_distribution }}" -# CentOS Stream 8 - OpenStack Wallaby -stackhpc_repo_centos_stream_openstack_wallaby_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/centos/8-stream/cloud/x86_64/openstack-wallaby/{{ stackhpc_repo_centos_stream_openstack_wallaby_version }}" -stackhpc_repo_centos_stream_openstack_wallaby_version: "{{ stackhpc_repo_distribution }}" +# CentOS Stream 8 - OpenStack Xena +stackhpc_repo_centos_stream_openstack_xena_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/centos/8-stream/cloud/x86_64/openstack-xena/{{ stackhpc_repo_centos_stream_openstack_xena_version }}" +stackhpc_repo_centos_stream_openstack_xena_version: "{{ stackhpc_repo_distribution }}" + +# CentOS Stream 8 - OpsTools - collectd +stackhpc_repo_centos_stream_opstools_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/centos/8-stream/opstools/x86_64/collectd-5/{{ stackhpc_repo_centos_stream_opstools_version }}" +stackhpc_repo_centos_stream_opstools_version: "{{ stackhpc_repo_distribution }}" # CentOS Stream 8 - PowerTools stackhpc_repo_centos_stream_powertools_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/centos/8-stream/PowerTools/x86_64/os/{{ stackhpc_repo_centos_stream_powertools_version }}" stackhpc_repo_centos_stream_powertools_version: "{{ stackhpc_repo_distribution }}" -# CentOS 8 - OpsTools - collectd -stackhpc_repo_centos_opstools_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/centos/8/opstools/x86_64/collectd-5/{{ stackhpc_repo_centos_opstools_version }}" -stackhpc_repo_centos_opstools_version: "{{ stackhpc_repo_distribution }}" - # CentOS Stream 8 - Ceph Pacific stackhpc_repo_centos_stream_storage_ceph_pacific_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/centos/8-stream/storage/x86_64/ceph-pacific/{{ stackhpc_repo_centos_stream_storage_ceph_pacific_version }}" stackhpc_repo_centos_stream_storage_ceph_pacific_version: "{{ stackhpc_repo_distribution }}" @@ -61,6 +65,10 @@ stackhpc_repo_elasticsearch_logstash_kibana_7_x_version: "{{ stackhpc_repo_distr stackhpc_repo_grafana_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/grafana/oss/rpm/{{ stackhpc_repo_grafana_version }}" stackhpc_repo_grafana_version: "{{ stackhpc_repo_distribution }}" +# MariaDB 10.5 CentOS 8 +stackhpc_repo_mariadb_10_5_centos8_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/mariadb-10.5/yum/centos8-amd64/{{ stackhpc_repo_mariadb_10_5_centos8_version }}" +stackhpc_repo_mariadb_10_5_centos8_version: "{{ stackhpc_repo_distribution }}" + # RabbitMQ - Erlang stackhpc_repo_rabbitmq_erlang_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rabbitmq/erlang/el/8/x86_64/{{ stackhpc_repo_rabbitmq_erlang_version }}" stackhpc_repo_rabbitmq_erlang_version: "{{ stackhpc_repo_distribution }}" From d62390fd7899dede2dacfb30978af41f17e3195c Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 7 Jun 2022 14:56:09 +0100 Subject: [PATCH 013/292] Xena: update image tags --- etc/kayobe/kolla.yml | 2 +- etc/kayobe/kolla/globals.yml | 19 ------------------- 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 5dd859822..bb8e88d5f 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -90,7 +90,7 @@ kolla_docker_registry_password: "{{ stackhpc_docker_registry_password }}" # Kolla OpenStack release version. This should be a Docker image tag. # Default is {{ openstack_release }}. -kolla_openstack_release: wallaby-20211207T102718 +kolla_openstack_release: TODO # Docker tag applied to built container images. Default is # {{ kolla_openstack_release }}. diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index f36f2eebf..a6cbb0520 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -1,22 +1,3 @@ --- docker_yum_baseurl: "{{ stackhpc_repo_docker_url }}" docker_yum_gpgkey: "https://download.docker.com/linux/{% raw %}{{ ansible_facts.distribution | lower }}{% endraw %}/gpg" - -barbican_tag: wallaby-20220311T133847 -cinder_tag: wallaby-20220309T083940 -cloudkitty_tag: wallaby-20220119T122428 -grafana_tag: wallaby-20220210T160332 -horizon_tag: wallaby-20220525T174700 -ironic_pxe_tag: wallaby-20220216T152518 -magnum_tag: wallaby-20220603T085526 -manila_tag: wallaby-20211210T140839 -neutron_tag: wallaby-20220412T124526 -nova_tag: wallaby-20220328T103755 -octavia_tag: wallaby-20220524T085900 -ovn_tag: wallaby-20220223T143249 -openvswitch_tag: wallaby-20220223T143249 -prometheus_tag: wallaby-20220525T174700 -prometheus_jiralert_tag: wallaby-20220119T122428 -prometheus_libvirt_exporter_tag: wallaby-20220325T122042 -prometheus_msteams_tag: wallaby-20220119T122428 -prometheus_openstack_exporter_tag: wallaby-20220119T122428 From 21d4e8a7136e71ac7309f643cd13fb29d89afbe2 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 7 Jun 2022 15:21:07 +0100 Subject: [PATCH 014/292] Xena: update Github Actions workflows --- .github/workflows/container-image-build.yml | 2 +- .github/workflows/promote.yml | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/container-image-build.yml b/.github/workflows/container-image-build.yml index d04e1e001..7cc08eeb4 100644 --- a/.github/workflows/container-image-build.yml +++ b/.github/workflows/container-image-build.yml @@ -35,7 +35,7 @@ jobs: uses: actions/checkout@v3 with: repository: stackhpc/kayobe - ref: refs/heads/stackhpc/wallaby + ref: refs/heads/stackhpc/xena path: src/kayobe # FIXME: Failed in kolla-ansible : Ensure the latest version of pip is installed diff --git a/.github/workflows/promote.yml b/.github/workflows/promote.yml index 81b05b774..69a245baf 100644 --- a/.github/workflows/promote.yml +++ b/.github/workflows/promote.yml @@ -3,9 +3,8 @@ name: Promote package repositories on: push: branches: - # NOTE(mgoddard): Keep this list tightly controlled, since we only want - # to promote from release branches. - - stackhpc/wallaby + # NOTE(mgoddard): Reference only the current release branch here. + - stackhpc/xena jobs: promote: name: Trigger package repository promotion From 8938c5029bfaccfdebdb4c7fd1657926c69298eb Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 13 Jun 2022 15:47:49 +0100 Subject: [PATCH 015/292] Xena: configure test pulp server as insecure in CI environments --- etc/kayobe/environments/ci-aio/stackhpc-ci.yml | 5 +++++ etc/kayobe/environments/ci-builder/stackhpc-ci.yml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/etc/kayobe/environments/ci-aio/stackhpc-ci.yml b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml index fc8819ae5..67ff004bd 100644 --- a/etc/kayobe/environments/ci-aio/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml @@ -5,6 +5,11 @@ # Docker namespace to use for Kolla images. Default is 'kolla'. kolla_docker_namespace: stackhpc-dev +# Whether docker should be configured to use an insecure registry for Kolla +# images. Default is false, unless docker_registry_enabled is true and +# docker_registry_enable_tls is false. +kolla_docker_registry_insecure: "{{ 'https' not in stackhpc_repo_mirror_url }}" + ############################################################################### # Network configuration. diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml index b7aa7d80d..99dff8e54 100644 --- a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml @@ -5,6 +5,11 @@ # Docker namespace to use for Kolla images. Default is 'kolla'. kolla_docker_namespace: stackhpc-dev +# Whether docker should be configured to use an insecure registry for Kolla +# images. Default is false, unless docker_registry_enabled is true and +# docker_registry_enable_tls is false. +kolla_docker_registry_insecure: "{{ 'https' not in stackhpc_repo_mirror_url }}" + # Kolla feature flag configuration. kolla_enable_barbican: true kolla_enable_central_logging: true From fc052d937e3ff550409577274cef95da3134ef4d Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 17 Jun 2022 15:31:09 +0100 Subject: [PATCH 016/292] Xena: update Pulp repository versions --- etc/kayobe/pulp-repo-versions.yml | 36 +++++++++++++++---------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/etc/kayobe/pulp-repo-versions.yml b/etc/kayobe/pulp-repo-versions.yml index 97140035f..54dc3d098 100644 --- a/etc/kayobe/pulp-repo-versions.yml +++ b/etc/kayobe/pulp-repo-versions.yml @@ -1,21 +1,21 @@ --- # Do not edit! This file is autogenerated by Ansible. stackhpc_pulp_repo_centos_stream_8_advanced_virtualization_version: 20211122T102435 -stackhpc_pulp_repo_centos_stream_8_appstream_version: 20211122T102435 -stackhpc_pulp_repo_centos_stream_8_baseos_version: 20211122T102435 -stackhpc_pulp_repo_centos_stream_8_extras_version: 20211122T102435 -stackhpc_pulp_repo_centos_stream_8_nfv_extras_version: '' -stackhpc_pulp_repo_centos_stream_8_nfv_openvswitch_version: 20220223T025353 -stackhpc_pulp_repo_centos_stream_8_openstack_xena_version: '' -stackhpc_pulp_repo_centos_stream_8_opstools_version: '' -stackhpc_pulp_repo_centos_stream_8_powertools_version: 20211122T102435 -stackhpc_pulp_repo_centos_stream_8_storage_ceph_pacific_version: 20211210T124731 -stackhpc_pulp_repo_docker_version: 20211122T102435 -stackhpc_pulp_repo_elasticsearch_logstash_kibana_7_x_version: 20211122T102435 -stackhpc_pulp_repo_epel_modular_version: 20211122T102435 -stackhpc_pulp_repo_epel_version: 20211122T102435 -stackhpc_pulp_repo_grafana_version: 20220119T030911 -stackhpc_pulp_repo_mariadb_10_5_centos8_version: '' -stackhpc_pulp_repo_rabbitmq_erlang_version: 20211122T102435 -stackhpc_pulp_repo_rabbitmq_server_version: 20211122T102435 -stackhpc_pulp_repo_treasuredata_4_version: 20211122T102435 +stackhpc_pulp_repo_centos_stream_8_appstream_version: 20220607T061247 +stackhpc_pulp_repo_centos_stream_8_baseos_version: 20220607T061247 +stackhpc_pulp_repo_centos_stream_8_extras_version: 20220401T032901 +stackhpc_pulp_repo_centos_stream_8_nfv_extras_version: 20220609T110556 +stackhpc_pulp_repo_centos_stream_8_nfv_openvswitch_version: 20220616T202817 +stackhpc_pulp_repo_centos_stream_8_openstack_xena_version: 20220612T035123 +stackhpc_pulp_repo_centos_stream_8_opstools_version: 20220617T100837 +stackhpc_pulp_repo_centos_stream_8_powertools_version: 20220607T061247 +stackhpc_pulp_repo_centos_stream_8_storage_ceph_pacific_version: 20220525T030654 +stackhpc_pulp_repo_docker_version: 20220607T061247 +stackhpc_pulp_repo_elasticsearch_logstash_kibana_7_x_version: 20220525T030654 +stackhpc_pulp_repo_epel_modular_version: 20220604T032730 +stackhpc_pulp_repo_epel_version: 20220617T050552 +stackhpc_pulp_repo_grafana_version: 20220616T113902 +stackhpc_pulp_repo_mariadb_10_5_centos8_version: 20220609T110556 +stackhpc_pulp_repo_rabbitmq_erlang_version: 20220616T113902 +stackhpc_pulp_repo_rabbitmq_server_version: 20220602T033149 +stackhpc_pulp_repo_treasuredata_4_version: 20220429T160649 From d1f8c7e612e763412144f0a72e0b82115fa5c579 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 8 Jul 2022 11:15:03 +0100 Subject: [PATCH 017/292] Sync with stackhpc/kayobe @ ae2e0ed2fc78c1a61d299b4f6c2aba5d190345bb This includes config from backported changes. --- etc/kayobe/apt.yml | 35 +++++++++++++++++++++++ etc/kayobe/bifrost.yml | 6 ++++ etc/kayobe/compute.yml | 47 ++++++++++++++++++++++++++++++ etc/kayobe/dnf.yml | 8 +++++- etc/kayobe/globals.yml | 7 +++-- etc/kayobe/infra-vms.yml | 3 ++ etc/kayobe/kolla.yml | 23 ++++++++++++++- etc/kayobe/overcloud-dib.yml | 55 ++++++++++++++++++++++++++++++++++++ etc/kayobe/proxy.yml | 22 +++++++++++++++ etc/kayobe/time.yml | 3 ++ 10 files changed, 204 insertions(+), 5 deletions(-) create mode 100644 etc/kayobe/overcloud-dib.yml create mode 100644 etc/kayobe/proxy.yml diff --git a/etc/kayobe/apt.yml b/etc/kayobe/apt.yml index 5f278e322..34bfdd2ef 100644 --- a/etc/kayobe/apt.yml +++ b/etc/kayobe/apt.yml @@ -11,6 +11,41 @@ # Apt proxy URL for HTTPS. Default is {{ apt_proxy_http }}. #apt_proxy_https: +# List of Apt configuration options. Each item is a dict with the following +# keys: +# * content: free-form configuration file content +# * filename: name of a file in /etc/apt/apt.conf.d/ in which to write the +# configuration +# Default is an empty list. +#apt_config: + +# List of apt keys. Each item is a dict containing the following keys: +# * url: URL of key +# * filename: Name of a file in which to store the downloaded key. The +# extension should be '.asc' for ASCII-armoured keys, or '.gpg' otherwise. +# Default is an empty list. +#apt_keys: + +# A list of Apt repositories. Each item is a dict with the following keys: +# * types: whitespace-separated list of repository types, e.g. deb or deb-src +# (optional, default is 'deb') +# * url: URL of the repository +# * suites: whitespace-separated list of suites, e.g. focal (optional, default +# is ansible_facts.distribution_release) +# * components: whitespace-separated list of components, e.g. main (optional, +# default is 'main') +# * signed_by: whitespace-separated list of names of GPG keyring files in +# apt_keys_path (optional, default is unset) +# * architecture: whitespace-separated list of architectures that will be used +# (optional, default is unset) +# Default is an empty list. +#apt_repositories: + +# Whether to disable repositories in /etc/apt/sources.list. This may be used +# when replacing the distribution repositories via apt_repositories. +# Default is false. +#apt_disable_sources_list: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/bifrost.yml b/etc/kayobe/bifrost.yml index cb8705ce3..0da08f1c5 100644 --- a/etc/kayobe/bifrost.yml +++ b/etc/kayobe/bifrost.yml @@ -65,6 +65,12 @@ kolla_bifrost_source_version: "{{ stackhpc_bifrost_source_version }}" # Name of disk image file to deploy. Default is "deployment_image.qcow2". #kolla_bifrost_deploy_image_filename: +# UUID of the root filesystem contained within the deployment image. +# See below URL for instructions on how to extract it: +# https://docs.openstack.org/ironic/latest/admin/raid.html#image-requirements +# Default is none. +#kolla_bifrost_deploy_image_rootfs: + ############################################################################### # Ironic configuration. diff --git a/etc/kayobe/compute.yml b/etc/kayobe/compute.yml index cd0ceb247..b1d8d6562 100644 --- a/etc/kayobe/compute.yml +++ b/etc/kayobe/compute.yml @@ -143,6 +143,53 @@ # - state: enabled #compute_firewalld_rules: +############################################################################### +# Compute node host libvirt configuration. + +# Whether to enable a host libvirt daemon. Default is true if kolla_enable_nova +# is true and kolla_enable_nova_libvirt_container is false. +#compute_libvirt_enabled: + +# A dict of default configuration options to write to +# /etc/libvirt/libvirtd.conf. +#compute_libvirt_conf_default: + +# A dict of additional configuration options to write to +# /etc/libvirt/libvirtd.conf. +#compute_libvirt_conf_extra: + +# A dict of configuration options to write to /etc/libvirt/libvirtd.conf. +# Default is a combination of compute_libvirt_conf_default and +# compute_libvirt_conf_extra. +#compute_libvirt_conf: + +# Numerical log level for libvirtd. Default is 3. +#compute_libvirtd_log_level: + +# A dict of default configuration options to write to +# /etc/libvirt/qemu.conf. +#compute_qemu_conf_default: + +# A dict of additional configuration options to write to +# /etc/libvirt/qemu.conf. +#compute_qemu_conf_extra: + +# A dict of configuration options to write to /etc/libvirt/qemu.conf. +# Default is a combination of compute_qemu_conf_default and +# compute_qemu_conf_extra. +#compute_qemu_conf: + +# Whether to enable a libvirt TLS listener. Default is false. +#compute_libvirt_enable_tls: + +# Whether to install a Ceph package repository on CentOS and Rocky hosts. +# Default is true. +#compute_libvirt_ceph_repo_install: + +# Ceph package repository release to install on CentOS and Rocky hosts when +# compute_libvirt_ceph_repo_install is true. Default is 'pacific'. +#compute_libvirt_ceph_repo_release: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/dnf.yml b/etc/kayobe/dnf.yml index 5e7ad2894..359c9cefc 100644 --- a/etc/kayobe/dnf.yml +++ b/etc/kayobe/dnf.yml @@ -12,12 +12,18 @@ # Whether or not to use a local Yum mirror. Default value is 'false'. #dnf_use_local_mirror: -# Mirror FQDN for Yum repos. Default value is 'mirror.centos.org'. +# Mirror FQDN for Yum CentOS repos. Default value is 'mirror.centos.org'. #dnf_centos_mirror_host: # Mirror directory for Yum CentOS repos. Default value is 'centos'. #dnf_centos_mirror_directory: +# Mirror FQDN for Yum Rocky repos. Default value is 'dl.rockylinux.org'. +#dnf_rocky_mirror_host: + +# Mirror directory for Yum Rocky repos. Default value is 'pub/rocky'. +#dnf_rocky_mirror_directory: + # Mirror FQDN for Yum EPEL repos. Default value is # 'download.fedoraproject.org'. #dnf_epel_mirror_host: diff --git a/etc/kayobe/globals.yml b/etc/kayobe/globals.yml index a4150d8ec..b926fc9be 100644 --- a/etc/kayobe/globals.yml +++ b/etc/kayobe/globals.yml @@ -45,12 +45,13 @@ ############################################################################### # OS distribution. -# OS distribution name. Valid options are "centos", "ubuntu". Default is -# "centos". +# OS distribution name. Valid options are "centos", "rocky", "ubuntu". Default +# is "centos". #os_distribution: # OS release. Valid options are "8-stream" when os_distribution is "centos", or -# "focal" when os_distribution is "ubuntu". +# "8" when os_distribution is "rocky", or "focal" when os_distribution is +# "ubuntu". #os_release: ############################################################################### diff --git a/etc/kayobe/infra-vms.yml b/etc/kayobe/infra-vms.yml index c8aec1cd6..e16adea1d 100644 --- a/etc/kayobe/infra-vms.yml +++ b/etc/kayobe/infra-vms.yml @@ -32,6 +32,9 @@ # Base image for the infra VM root volume. Default is # "https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img" # when os_distribution is "ubuntu", or +# http://dl.rockylinux.org/pub/rocky/8.5/images/Rocky-8-GenericCloud-8.5-20211114.2.x86_64.qcow2 +# when os_distribution is "rocky", +# or # "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20210210.0.x86_64.qcow2" # otherwise. #infra_vm_root_image: diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index bb8e88d5f..4dfeda081 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -50,6 +50,11 @@ kolla_ansible_source_version: "{{ stackhpc_kolla_ansible_source_version }}" # Extra requirements to install inside the kolla-ansible virtualenv. #kolla_ansible_venv_extra_requirements: +# Pip requirement specifier for the ansible package. NOTE: This limits the +# version of ansible used by kolla-ansible to avoid new releases from breaking +# tested code. Changes to this limit should be tested. +#kolla_ansible_venv_ansible: + # Path to Kolla-ansible configuration directory. Default is $KOLLA_CONFIG_PATH # or /etc/kolla if $KOLLA_CONFIG_PATH is not set. #kolla_config_path: @@ -62,7 +67,8 @@ kolla_ansible_source_version: "{{ stackhpc_kolla_ansible_source_version }}" # Kolla configuration. # Kolla base container image distribution. Options are "centos", "debian", -# "ubuntu". Default is {{ os_distribution }}. +# "ubuntu". Default is +# {{ 'centos' if os_distribution == 'rocky' else os_distribution }}. #kolla_base_distro: # Kolla container image type: binary or source. Default is 'source'. @@ -617,6 +623,21 @@ kolla_build_customizations: # default. #kolla_internal_fqdn_cacert: +############################################################################### +# Proxy configuration + +# HTTP proxy URL (format: http(s)://[user:password@]proxy_name:port) used by +# Kolla. Default value is "{{ http_proxy }}". +#kolla_http_proxy: + +# HTTPS proxy URL (format: http(s)://[user:password@]proxy_name:port) used by +# Kolla. Default value is "{{ https_proxy }}". +#kolla_https_proxy: + +# List of domains, hostnames, IP addresses and networks for which no proxy is +# used. Default value is "{{ no_proxy }}". +#kolla_no_proxy: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/overcloud-dib.yml b/etc/kayobe/overcloud-dib.yml new file mode 100644 index 000000000..0a95c97e2 --- /dev/null +++ b/etc/kayobe/overcloud-dib.yml @@ -0,0 +1,55 @@ +--- +# Overcloud host disk image configuration. + +############################################################################### +# Diskimage-builder configuration for overcloud host disk images. + +# Whether to build host disk images with DIB directly instead of through +# Bifrost. Setting it to true disables Bifrost image build and allows images to +# be built with the `kayobe overcloud host image build` command. Default value +# is False. This will change in a future release. +#overcloud_dib_build_host_images: + +# DIB base OS element. Default is {{ os_distribution }}. +#overcloud_dib_os_element: + +# DIB image OS release. Default is {{ os_release }}. +#overcloud_dib_os_release: + +# List of default DIB elements. Default is ["centos", "cloud-init-datasources", +# "disable-selinux", "enable-serial-console", "vm"] when +# overcloud_dib_os_element is "centos", or ["ubuntu", "cloud-init-datasources", +# "enable-serial-console", "vm"] when overcloud_dib_os_element is "ubuntu". +#overcloud_dib_elements_default: + +# List of additional DIB elements. Default is none. +#overcloud_dib_elements_extra: + +# List of DIB elements. Default is a combination of +# overcloud_dib_elements_default and overcloud_dib_elements_extra. +#overcloud_dib_elements: + +# DIB default environment variables. Default is +# {"DIB_BOOTLOADER_DEFAULT_CMDLINE": "nofb nomodeset gfxpayload=text +# net.ifnames=1", "DIB_CLOUD_INIT_DATASOURCES": "ConfigDrive", "DIB_RELEASE": +# "{{ overcloud_dib_os_release }}"}. +#overcloud_dib_env_vars_default: + +# DIB additional environment variables. Default is none. +#overcloud_dib_env_vars_extra: + +# DIB environment variables. Default is combination of +# overcloud_dib_env_vars_default and overcloud_dib_env_vars_extra. +#overcloud_dib_env_vars: + +# List of DIB packages to install. Default is to install no extra packages. +#overcloud_dib_packages: + +# Upper constraints file for installing packages in the virtual environment +# used for building overcloud host disk images. Default is {{ +# pip_upper_constraints_file }}. +#overcloud_dib_upper_constraints_file: + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/proxy.yml b/etc/kayobe/proxy.yml new file mode 100644 index 000000000..714b9dae5 --- /dev/null +++ b/etc/kayobe/proxy.yml @@ -0,0 +1,22 @@ +--- +############################################################################### +# Configuration of HTTP(S) proxies. + +# HTTP proxy URL (format: http(s)://[user:password@]proxy_name:port). By +# default no proxy is used. +#http_proxy: + +# HTTPS proxy URL (format: http(s)://[user:password@]proxy_name:port). By +# default no proxy is used. +#https_proxy: + +# List of domains, hostnames, IP addresses and networks for which no proxy is +# used. Defaults to ["127.0.0.1", "localhost", "{{ ('http://' ~ +# docker_registry) | urlsplit('hostname') }}"] if docker_registry is set, or +# ["127.0.0.1", "localhost"] otherwise. This is configured only if either +# http_proxy or https_proxy is set. +#no_proxy: + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/time.yml b/etc/kayobe/time.yml index 23c1d0dc4..8bfbd3391 100644 --- a/etc/kayobe/time.yml +++ b/etc/kayobe/time.yml @@ -30,6 +30,9 @@ # #chrony_ntp_servers: +# Synchronise hardware clock with system time. Default is true. +#chrony_rtcsync_enabled: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes From 244ab8587f949d3491e8777256b84673dcc315eb Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 19 Jul 2022 11:35:27 +0100 Subject: [PATCH 018/292] Xena: drop comment in tox.ini --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8c048cff2..787fe1044 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,6 @@ commands = [testenv:releasenotes] allowlist_externals = rm skip_install = true -# NOTE: Using Xena upper constraints for reno 3.4.0, with support for default_branch. deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/xena} -r{toxinidir}/releasenotes/requirements.txt From 84e6e9a3c60c87ef636d8a68573f56719557c8a5 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 19 Jul 2022 11:37:38 +0100 Subject: [PATCH 019/292] Xena: update release notes --- releasenotes/config.yaml | 2 +- releasenotes/source/index.rst | 1 + releasenotes/source/xena.rst | 6 ++++++ 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 releasenotes/source/xena.rst diff --git a/releasenotes/config.yaml b/releasenotes/config.yaml index 88d4c778b..55f94b723 100644 --- a/releasenotes/config.yaml +++ b/releasenotes/config.yaml @@ -1,4 +1,4 @@ --- # This needs to be updated to the latest release. -default_branch: stackhpc/wallaby +default_branch: stackhpc/xena release_tag_re: stackhpc/\d+\.\d+\.\d+\.\d diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst index 97a4a4dd2..e1ea00229 100644 --- a/releasenotes/source/index.rst +++ b/releasenotes/source/index.rst @@ -7,6 +7,7 @@ Contents .. toctree:: :maxdepth: 2 + xena wallaby diff --git a/releasenotes/source/xena.rst b/releasenotes/source/xena.rst new file mode 100644 index 000000000..e5cdefbdb --- /dev/null +++ b/releasenotes/source/xena.rst @@ -0,0 +1,6 @@ +========================= +Xena Series Release Notes +========================= + +.. release-notes:: + :branch: stackhpc/xena From 9a660456e00544d860754102119a34019b044546 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 22 Jul 2022 11:37:29 +0100 Subject: [PATCH 020/292] Revert multi-image overcloud-dib config This was accidentally included in the Rocky Linux Wallaby backport (without the accompanying code). --- etc/kayobe/overcloud-dib.yml | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/etc/kayobe/overcloud-dib.yml b/etc/kayobe/overcloud-dib.yml index f53804e42..0a95c97e2 100644 --- a/etc/kayobe/overcloud-dib.yml +++ b/etc/kayobe/overcloud-dib.yml @@ -10,14 +10,6 @@ # is False. This will change in a future release. #overcloud_dib_build_host_images: -# List of overcloud host disk images to build. Each element is a dict defining -# an image in a format accepted by the stackhpc.os-images role. Default is to -# build an image named "deployment_image" configured with the overcloud_dib_* -# variables defined below: {"name": "deployment_image", "elements": "{{ -# overcloud_dib_elements }}", "env": "{{ overcloud_dib_env_vars }}", -# "packages": "{{ overcloud_dib_packages }}"}. -#overcloud_dib_host_images: - # DIB base OS element. Default is {{ os_distribution }}. #overcloud_dib_os_element: @@ -53,19 +45,6 @@ # List of DIB packages to install. Default is to install no extra packages. #overcloud_dib_packages: -# List of default git repositories containing Diskimage Builder (DIB) elements. -# See stackhpc.os-images role for usage. Default is empty. -#overcloud_dib_git_elements_default: - -# List of additional git repositories containing Diskimage Builder (DIB) -# elements. See stackhpc.os-images role for usage. Default is empty. -#overcloud_dib_git_elements_extra: - -# List of git repositories containing Diskimage Builder (DIB) elements. See -# stackhpc.os-images role for usage. Default is a combination of -# overcloud_dib_git_elements_default and overcloud_dib_git_elements_extra. -#overcloud_dib_git_elements: - # Upper constraints file for installing packages in the virtual environment # used for building overcloud host disk images. Default is {{ # pip_upper_constraints_file }}. From 03d26242097aa1888693e9cad97244c2f5fb9d1f Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 22 Jul 2022 11:59:58 +0100 Subject: [PATCH 021/292] Fix up seed config after Wallaby merge --- etc/kayobe/seed-vm.yml | 8 ++------ etc/kayobe/seed.yml | 21 --------------------- 2 files changed, 2 insertions(+), 27 deletions(-) diff --git a/etc/kayobe/seed-vm.yml b/etc/kayobe/seed-vm.yml index f8c2ab8a9..59dc766ae 100644 --- a/etc/kayobe/seed-vm.yml +++ b/etc/kayobe/seed-vm.yml @@ -25,12 +25,8 @@ # Base image for the seed VM root volume. Default is # "https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img" -# when os_distribution is "ubuntu", -# http://dl.rockylinux.org/pub/rocky/8.5/images/Rocky-8-GenericCloud-8.5-20211114.2.x86_64.qcow2 -# when os_distribution is "rocky", -# or -# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20210603.0.x86_64.qcow2" -# otherwise. +# when os_distribution is "ubuntu", or +# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20210210.0.x86_64.qcow2" #seed_vm_root_image: # Capacity of the seed VM data volume. diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index e3956e958..bcbec5556 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -164,27 +164,6 @@ seed_containers: >- # - state: enabled #seed_firewalld_rules: -############################################################################### -# Seed node firewalld configuration. - -# Whether to install and enable firewalld. -#seed_firewalld_enabled: - -# A list of zones to create. Each item is a dict containing a 'zone' item. -#seed_firewalld_zones: - -# A firewalld zone to set as the default. Default is unset, in which case the -# default zone will not be changed. -#seed_firewalld_default_zone: - -# A list of firewall rules to apply. Each item is a dict containing arguments -# to pass to the firewalld module. Arguments are omitted if not provided, with -# the following exceptions: -# - offline: true -# - permanent: true -# - state: enabled -#seed_firewalld_rules: - ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes From 2df3e71a829fa9e86a680693a97b5f9241e09e79 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 20 Jul 2022 16:14:37 +0100 Subject: [PATCH 022/292] Add custom playbooks for Cephadm Adds playbooks that use the stackhpc.cephadm collection to deploy and configure Ceph. The cephadm-gather-keys.yml playbook gathers configuration files and keys for a deployed Ceph cluster, and generates the necessary files for Kolla Ansible, adding them to kayobe-config. --- etc/kayobe/ansible/cephadm-crush-rules.yml | 11 +++++++++++ etc/kayobe/ansible/cephadm-deploy.yml | 11 +++++++++++ etc/kayobe/ansible/cephadm-ec-profiles.yml | 11 +++++++++++ etc/kayobe/ansible/cephadm-keys.yml | 11 +++++++++++ etc/kayobe/ansible/cephadm-pools.yml | 11 +++++++++++ etc/kayobe/ansible/cephadm.yml | 7 +++++++ etc/kayobe/ansible/requirements.yml | 2 ++ 7 files changed, 64 insertions(+) create mode 100644 etc/kayobe/ansible/cephadm-crush-rules.yml create mode 100644 etc/kayobe/ansible/cephadm-deploy.yml create mode 100644 etc/kayobe/ansible/cephadm-ec-profiles.yml create mode 100644 etc/kayobe/ansible/cephadm-keys.yml create mode 100644 etc/kayobe/ansible/cephadm-pools.yml create mode 100644 etc/kayobe/ansible/cephadm.yml diff --git a/etc/kayobe/ansible/cephadm-crush-rules.yml b/etc/kayobe/ansible/cephadm-crush-rules.yml new file mode 100644 index 000000000..b95b9f960 --- /dev/null +++ b/etc/kayobe/ansible/cephadm-crush-rules.yml @@ -0,0 +1,11 @@ +--- +- name: Define Ceph CRUSH rules + gather_facts: false + hosts: mons + become: true + tags: + - cephadm + - cephadm-crush-rules + tasks: + - import_role: + name: stackhpc.cephadm.crush_rules diff --git a/etc/kayobe/ansible/cephadm-deploy.yml b/etc/kayobe/ansible/cephadm-deploy.yml new file mode 100644 index 000000000..9cb0f712b --- /dev/null +++ b/etc/kayobe/ansible/cephadm-deploy.yml @@ -0,0 +1,11 @@ +--- +- name: Deploy Ceph using Cephadm + any_errors_fatal: true + gather_facts: true + hosts: ceph + tags: + - cephadm + - cephadm-deploy + tasks: + - import_role: + name: stackhpc.cephadm.cephadm diff --git a/etc/kayobe/ansible/cephadm-ec-profiles.yml b/etc/kayobe/ansible/cephadm-ec-profiles.yml new file mode 100644 index 000000000..5fb0f3bdc --- /dev/null +++ b/etc/kayobe/ansible/cephadm-ec-profiles.yml @@ -0,0 +1,11 @@ +--- +- name: Define Ceph EC profiles + gather_facts: false + hosts: mons + become: true + tags: + - cephadm + - cephadm-ec-profiles + tasks: + - import_role: + name: stackhpc.cephadm.ec_profiles diff --git a/etc/kayobe/ansible/cephadm-keys.yml b/etc/kayobe/ansible/cephadm-keys.yml new file mode 100644 index 000000000..6f632f4ca --- /dev/null +++ b/etc/kayobe/ansible/cephadm-keys.yml @@ -0,0 +1,11 @@ +--- +- name: Generate Ceph keys + gather_facts: false + hosts: mons + become: true + tags: + - cephadm + - cephadm-keys + tasks: + - import_role: + name: stackhpc.cephadm.keys diff --git a/etc/kayobe/ansible/cephadm-pools.yml b/etc/kayobe/ansible/cephadm-pools.yml new file mode 100644 index 000000000..938e6de19 --- /dev/null +++ b/etc/kayobe/ansible/cephadm-pools.yml @@ -0,0 +1,11 @@ +--- +- name: Define Ceph pools + gather_facts: false + hosts: mons + become: true + tags: + - cephadm + - cephadm-keys + tasks: + - import_role: + name: stackhpc.cephadm.pools diff --git a/etc/kayobe/ansible/cephadm.yml b/etc/kayobe/ansible/cephadm.yml new file mode 100644 index 000000000..96c667f62 --- /dev/null +++ b/etc/kayobe/ansible/cephadm.yml @@ -0,0 +1,7 @@ +--- +# Deploy Ceph via Cephadm. Create EC profiles, CRUSH rules, pools and keys. +- import_playbook: cephadm-deploy.yml +- import_playbook: cephadm-ec-profiles.yml +- import_playbook: cephadm-crush-rules.yml +- import_playbook: cephadm-pools.yml +- import_playbook: cephadm-keys.yml diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index d8c1ad3f2..52b18f408 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -1,5 +1,7 @@ --- collections: + - name: stackhpc.cephadm + version: 1.7.0 - name: stackhpc.pulp version: 0.3.0 - name: pulp.squeezer From ddb1d86ec6982945769966edcdbd4d5326104a9a Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 22 Jul 2022 16:24:36 +0100 Subject: [PATCH 023/292] Add Ceph image to Pulp registry Sync from quay.io/ceph/ceph. Images are synced when stackhpc_sync_ceph_images is true, and it defaults to false. --- etc/kayobe/pulp.yml | 55 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 12 deletions(-) diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index 8e3fdd752..f100af098 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -161,7 +161,7 @@ stackhpc_pulp_distribution_rpm_production: stackhpc_release_pulp_registry_url: "{{ stackhpc_release_pulp_url }}" # List of all image names. -stackhpc_pulp_images: +stackhpc_pulp_images_kolla: - barbican-api - barbican-base - barbican-keystone-listener @@ -282,33 +282,64 @@ stackhpc_pulp_images: - redis-sentinel # Common parameters for container image repositories. -stackhpc_pulp_repository_container_repos_common: +stackhpc_pulp_repository_container_repos_kolla_common: url: "{{ stackhpc_release_pulp_registry_url }}" policy: on_demand remote_username: "{{ stackhpc_release_pulp_username }}" remote_password: "{{ stackhpc_release_pulp_password }}" state: present + required: true -# List of container image repositories. -stackhpc_pulp_repository_container_repos: >- +# List of Kolla container image repositories. +stackhpc_pulp_repository_container_repos_kolla: >- {%- set repos = [] -%} - {%- for image in stackhpc_pulp_images -%} + {%- for image in stackhpc_pulp_images_kolla -%} {%- set image_repo = kolla_docker_namespace ~ "/" ~ kolla_base_distro ~ "-" ~ kolla_install_type ~ "-" ~ image -%} {%- set repo = {"name": image_repo} -%} - {%- set _ = repos.append(stackhpc_pulp_repository_container_repos_common | combine(repo)) -%} + {%- set _ = repos.append(stackhpc_pulp_repository_container_repos_kolla_common | combine(repo)) -%} {%- endfor -%} {{ repos }} -# Common parameters for container image distributions. -stackhpc_pulp_distribution_container_common: +# Common parameters for Kolla container image distributions. +stackhpc_pulp_distribution_container_kolla_common: state: present + required: true -# List of container image distributions. -stackhpc_pulp_distribution_container: >- +# List of Kolla container image distributions. +stackhpc_pulp_distribution_container_kolla: >- {%- set distributions = [] -%} - {%- for image in stackhpc_pulp_images -%} + {%- for image in stackhpc_pulp_images_kolla -%} {%- set image_repo = kolla_docker_namespace ~ "/" ~ kolla_base_distro ~ "-" ~ kolla_install_type ~ "-" ~ image -%} {%- set distribution = {"name": image_repo, "repository": image_repo, "base_path": image_repo} -%} - {%- set _ = distributions.append(stackhpc_pulp_distribution_container_common | combine(distribution)) -%} + {%- set _ = distributions.append(stackhpc_pulp_distribution_container_kolla_common | combine(distribution)) -%} {%- endfor -%} {{ distributions }} + +# Whether to sync Ceph container images. +stackhpc_sync_ceph_images: false + +# List of Ceph container image repositories. +stackhpc_pulp_repository_container_repos_ceph: + - name: "ceph/ceph" + url: "https://quay.io" + policy: on_demand + state: present + required: "{{ stackhpc_sync_ceph_images | bool }}" + +# List of Ceph container image distributions. +stackhpc_pulp_distribution_container_ceph: + - name: ceph + repository: ceph/ceph + base_path: ceph/ceph + state: present + required: "{{ stackhpc_sync_ceph_images | bool }}" + +# List of container image repositories. +stackhpc_pulp_repository_container_repos: >- + {{ (stackhpc_pulp_repository_container_repos_kolla + + stackhpc_pulp_repository_container_repos_ceph) | selectattr('required') }} + +# List of container image distributions. +stackhpc_pulp_distribution_container: >- + {{ (stackhpc_pulp_distribution_container_kolla + + stackhpc_pulp_distribution_container_ceph) | selectattr('required') }} From 2249ca06fba9eeeb6cf14f70f18106dd966c6eab Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 28 Jul 2022 13:30:38 +0100 Subject: [PATCH 024/292] Add cephadm.yml configuration defaults * Ceph Pacific release * Kayobe storage network -> Ceph public network * Kayobe storage management network -> Ceph cluster network * Use local Pulp for container images --- etc/kayobe/cephadm.yml | 58 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 etc/kayobe/cephadm.yml diff --git a/etc/kayobe/cephadm.yml b/etc/kayobe/cephadm.yml new file mode 100644 index 000000000..0998f1181 --- /dev/null +++ b/etc/kayobe/cephadm.yml @@ -0,0 +1,58 @@ +--- +############################################################################### +# Cephadm deployment configuration. + +# Ceph release name. +cephadm_ceph_release: "pacific" + +# Ceph FSID. +#cephadm_fsid: + +# Ceph container image. +cephadm_image: "{{ stackhpc_docker_registry }}/ceph/ceph:{{ cephadm_image_tag }}" + +# Ceph container image tag. +cephadm_image_tag: "v16.2.5" + +# Ceph container image registry URL. +cephadm_registry_url: "{{ stackhpc_docker_registry }}" + +# Ceph container image registry username. +cephadm_registry_username: "{{ stackhpc_docker_registry_username }}" + +# Ceph container image registry password. +cephadm_registry_password: "{{ stackhpc_docker_registry_password }}" + +# Ceph public network interface. +cephadm_public_interface: "{{ storage_net_name | net_interface }}" + +# Ceph public network subnet CIDR. +cephadm_public_network: "{{ storage_net_name | net_cidr }}" + +# Ceph cluster network interface. +cephadm_cluster_interface: "{{ storage_mgmt_net_name | net_interface }}" + +# Ceph cluster network subnet CIDR. +cephadm_cluster_network: "{{ storage_mgmt_net_name | net_cidr }}" + +# Whether to enable firewalld for Ceph storage hosts. +#cephadm_enable_firewalld: + +# Ceph OSD specification. +#cephadm_osd_spec: + +############################################################################### +# Ceph post-deployment configuration. + +# List of Ceph erasure coding profiles. See stackhpc.cephadm.ec_profiles role +# for format. +#cephadm_ec_profiles: + +# List of Ceph CRUSH rules. See stackhpc.cephadm.crush_rules role for format. +#cephadm_crush_rules: + +# List of Ceph pools. See stackhpc.cephadm.pools role for format. +#cephadm_pools: + +# List of Cephx keys. See stackhpc.cephadm.keys role for format. +#cephadm_keys: From 70b0c823014b95ff2b7b4444bba5d178ddfc52fb Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 28 Jul 2022 20:31:05 +0100 Subject: [PATCH 025/292] Add Cephadm groups to inventory --- etc/kayobe/inventory/groups | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/etc/kayobe/inventory/groups b/etc/kayobe/inventory/groups index 16619fed8..e51195309 100644 --- a/etc/kayobe/inventory/groups +++ b/etc/kayobe/inventory/groups @@ -97,3 +97,18 @@ overcloud mgmt-switches ctl-switches hs-switches + +############################################################################### +# Ceph groups. + +[ceph:children] +mons +mgrs +osds +rgws + +# Empty groups to provide declaration of groups required for Cephadm. +[mons] +[mgrs] +[osds] +[rgws] From 6db34c494e7e9a99448a823f10ccf0f1f683e1e6 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 28 Jul 2022 14:17:14 +0100 Subject: [PATCH 026/292] Add a playbook to generate Kayobe configuration for Ceph via Cephadm --- etc/kayobe/ansible/cephadm-gather-keys.yml | 82 ++++++++++++++++++++++ etc/kayobe/cephadm.yml | 42 +++++++++++ 2 files changed, 124 insertions(+) create mode 100644 etc/kayobe/ansible/cephadm-gather-keys.yml diff --git a/etc/kayobe/ansible/cephadm-gather-keys.yml b/etc/kayobe/ansible/cephadm-gather-keys.yml new file mode 100644 index 000000000..d73f15a9e --- /dev/null +++ b/etc/kayobe/ansible/cephadm-gather-keys.yml @@ -0,0 +1,82 @@ +--- +- name: Gather Ceph configuration and keys and populate kayobe-config + gather_facts: false + hosts: mons + run_once: true + tags: + - cephadm + - cephadm-gather-keys + vars: + # Map from an OpenStack service to the directory in which to store Ceph keys for it. + kolla_service_to_key_dir: + cinder-backup: cinder/cinder-backup + cinder-volume: cinder/cinder-volume + glance: glance + manila: manila + nova: nova + # Map from an OpenStack service to the directory in which to store Ceph configuration for it. + kolla_service_to_conf_dir: + cinder-backup: cinder + cinder-volume: cinder + glance: glance + manila: manila + nova: nova + tasks: + - name: Get Ceph keys + stackhpc.cephadm.cephadm_key: + name: "{{ item }}" + state: info + register: cephadm_key_info + become: true + loop: "{{ kolla_ceph_services | selectattr('required') | map(attribute='keys') | flatten | unique }}" + + - name: Generate ceph.conf + command: "cephadm shell -- ceph config generate-minimal-conf" + become: true + register: cephadm_ceph_conf + changed_when: false + + - name: Ensure Kolla config directories are present + file: + state: directory + path: "{{ kayobe_env_config_path }}/kolla/config/{{ kolla_service_to_key_dir[item.name] }}" + loop: "{{ kolla_ceph_services | selectattr('required') }}" + loop_control: + label: + service: "{{ item.name }}" + delegate_to: localhost + + - name: Save Ceph keys to Kayobe configuration + vars: + key_info: "{{ cephadm_key_info.results | selectattr('item', 'equalto', item.1) | first }}" + cephadm_key: "{{ (key_info.stdout | from_json | first)['key'] }}" + cephadm_user: "{{ (key_info.stdout | from_json | first)['entity'] }}" + copy: + content: "{{ cephadm_key }}" + dest: "{{ kayobe_env_config_path }}/kolla/config/{{ kolla_service_to_key_dir[item.0.name] }}/ceph.{{ cephadm_user }}.keyring" + loop: "{{ query('subelements', kolla_ceph_services | selectattr('required'), 'keys') }}" + loop_control: + label: + service: "{{ item.0.name }}" + key: "{{ item.1 }}" + delegate_to: localhost + notify: Please add and commit the Kayobe configuration + + - name: Save ceph.conf to Kayobe configuration + copy: + content: "{{ cephadm_ceph_conf.stdout }}" + dest: "{{ kayobe_env_config_path }}/kolla/config/{{ kolla_service_to_conf_dir[item.0.name] }}/ceph.conf" + loop: "{{ query('subelements', kolla_ceph_services | selectattr('required'), 'keys') }}" + loop_control: + label: + service: "{{ item.0.name }}" + key: "{{ item.1 }}" + delegate_to: localhost + notify: Please add and commit the Kayobe configuration + + handlers: + - name: Please add and commit the Kayobe configuration + debug: + msg: >- + Please add and commit the Ceph configuration files and keys in Kayobe + configuration. Remember to encrypt the keys using Ansible Vault. diff --git a/etc/kayobe/cephadm.yml b/etc/kayobe/cephadm.yml index 0998f1181..eff49c565 100644 --- a/etc/kayobe/cephadm.yml +++ b/etc/kayobe/cephadm.yml @@ -56,3 +56,45 @@ cephadm_cluster_network: "{{ storage_mgmt_net_name | net_cidr }}" # List of Cephx keys. See stackhpc.cephadm.keys role for format. #cephadm_keys: + +############################################################################### +# Kolla Ceph auto-configuration. + +# List of Kolla Ansible services that require Ceph configuration files. +kolla_ceph_services: + - name: cinder-backup + keys: + - client.cinder + - client.cinder-backup + required: "{{ kolla_ceph_cinder_backup_required | bool }}" + - name: cinder-volume + keys: + - client.cinder + required: "{{ kolla_ceph_cinder_volume_required | bool }}" + - name: glance + keys: + - client.glance + required: "{{ kolla_ceph_glance_required | bool }}" + - name: manila + keys: + - client.manila + required: "{{ kolla_ceph_manila_required | bool }}" + - name: nova + keys: + - client.cinder + required: "{{ kolla_ceph_nova_required | bool }}" + +# Whether to generate Ceph configuration for Cinder volume. +kolla_ceph_cinder_volume_required: "{{ kolla_enable_cinder | bool }}" + +# Whether to generate Ceph configuration for Cinder backup. +kolla_ceph_cinder_backup_required: "{{ kolla_enable_cinder_backup | default(false) | bool }}" + +# Whether to generate Ceph configuration for Glance. +kolla_ceph_glance_required: "{{ kolla_enable_glance | bool }}" + +# Whether to generate Ceph configuration for Manila. +kolla_ceph_manila_required: "{{ kolla_enable_manila | bool }}" + +# Whether to generate Ceph configuration for Nova. +kolla_ceph_nova_required: "{{ kolla_enable_nova | bool }}" From 1631ea9c0deb1ee5db9c883403ae375ea3c5ff39 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 5 Aug 2022 16:56:32 +0100 Subject: [PATCH 027/292] Add multiple overcloud DIB image support config This has now been backported to Kayobe in stackhpc/xena. --- etc/kayobe/overcloud-dib.yml | 38 ++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/etc/kayobe/overcloud-dib.yml b/etc/kayobe/overcloud-dib.yml index 0a95c97e2..3d69f8eb1 100644 --- a/etc/kayobe/overcloud-dib.yml +++ b/etc/kayobe/overcloud-dib.yml @@ -7,10 +7,19 @@ # Whether to build host disk images with DIB directly instead of through # Bifrost. Setting it to true disables Bifrost image build and allows images to # be built with the `kayobe overcloud host image build` command. Default value -# is False. This will change in a future release. +# is {{ os_distribution == 'rocky' }}. This will change in a future release. #overcloud_dib_build_host_images: -# DIB base OS element. Default is {{ os_distribution }}. +# List of overcloud host disk images to build. Each element is a dict defining +# an image in a format accepted by the stackhpc.os-images role. Default is to +# build an image named "deployment_image" configured with the overcloud_dib_* +# variables defined below: {"name": "deployment_image", "elements": "{{ +# overcloud_dib_elements }}", "env": "{{ overcloud_dib_env_vars }}", +# "packages": "{{ overcloud_dib_packages }}"}. +#overcloud_dib_host_images: + +# DIB base OS element. Default is {{ 'rocky-container' if os_distribution == +# 'rocky' else os_distribution }}. #overcloud_dib_os_element: # DIB image OS release. Default is {{ os_release }}. @@ -18,8 +27,11 @@ # List of default DIB elements. Default is ["centos", "cloud-init-datasources", # "disable-selinux", "enable-serial-console", "vm"] when -# overcloud_dib_os_element is "centos", or ["ubuntu", "cloud-init-datasources", -# "enable-serial-console", "vm"] when overcloud_dib_os_element is "ubuntu". +# overcloud_dib_os_element is "centos", or ["rocky-container", +# "cloud-init-datasources", "disable-selinux", "enable-serial-console", "vm"] +# when overcloud_dib_os_element is "rocky" or +# ["ubuntu", "cloud-init-datasources", "enable-serial-console", "vm"] +# when overcloud_dib_os_element is "ubuntu". #overcloud_dib_elements_default: # List of additional DIB elements. Default is none. @@ -31,8 +43,9 @@ # DIB default environment variables. Default is # {"DIB_BOOTLOADER_DEFAULT_CMDLINE": "nofb nomodeset gfxpayload=text -# net.ifnames=1", "DIB_CLOUD_INIT_DATASOURCES": "ConfigDrive", "DIB_RELEASE": -# "{{ overcloud_dib_os_release }}"}. +# net.ifnames=1", "DIB_CLOUD_INIT_DATASOURCES": "ConfigDrive", +# "DIB_CONTAINERFILE_RUNTIME": "docker", "DIB_CONTAINERFILE_NETWORK_DRIVER": +# "host", "DIB_RELEASE": "{{ overcloud_dib_os_release }}"}. #overcloud_dib_env_vars_default: # DIB additional environment variables. Default is none. @@ -45,6 +58,19 @@ # List of DIB packages to install. Default is to install no extra packages. #overcloud_dib_packages: +# List of default git repositories containing Diskimage Builder (DIB) elements. +# See stackhpc.os-images role for usage. Default is empty. +#overcloud_dib_git_elements_default: + +# List of additional git repositories containing Diskimage Builder (DIB) +# elements. See stackhpc.os-images role for usage. Default is empty. +#overcloud_dib_git_elements_extra: + +# List of git repositories containing Diskimage Builder (DIB) elements. See +# stackhpc.os-images role for usage. Default is a combination of +# overcloud_dib_git_elements_default and overcloud_dib_git_elements_extra. +#overcloud_dib_git_elements: + # Upper constraints file for installing packages in the virtual environment # used for building overcloud host disk images. Default is {{ # pip_upper_constraints_file }}. From 0381e16bf2591dc431dd6b69967a9b1bf77564d3 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 28 Jul 2022 15:45:13 +0000 Subject: [PATCH 028/292] Add StackHPC overcloud DIB configuration --- etc/kayobe/overcloud-dib.yml | 10 +- etc/kayobe/stackhpc-overcloud-dib.yml | 212 ++++++++++++++++++ .../notes/overcloud-dib-938eac005662a2cb.yaml | 4 + 3 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 etc/kayobe/stackhpc-overcloud-dib.yml create mode 100644 releasenotes/notes/overcloud-dib-938eac005662a2cb.yaml diff --git a/etc/kayobe/overcloud-dib.yml b/etc/kayobe/overcloud-dib.yml index 3d69f8eb1..78fff9471 100644 --- a/etc/kayobe/overcloud-dib.yml +++ b/etc/kayobe/overcloud-dib.yml @@ -10,6 +10,10 @@ # is {{ os_distribution == 'rocky' }}. This will change in a future release. #overcloud_dib_build_host_images: +# List of additional host packages to install. +overcloud_dib_host_packages_extra: + - "{% if 'ubuntu-minimal' in overcloud_dib_host_images | map(attribute='elements') | flatten | list %}debootstrap{% endif %}" + # List of overcloud host disk images to build. Each element is a dict defining # an image in a format accepted by the stackhpc.os-images role. Default is to # build an image named "deployment_image" configured with the overcloud_dib_* @@ -64,7 +68,11 @@ # List of additional git repositories containing Diskimage Builder (DIB) # elements. See stackhpc.os-images role for usage. Default is empty. -#overcloud_dib_git_elements_extra: +overcloud_dib_git_elements_extra: + - repo: "https://github.com/stackhpc/stackhpc-image-elements" + local: "{{ source_checkout_path }}/stackhpc-image-elememts" + version: "v1.4.0" + elements_path: "elements" # List of git repositories containing Diskimage Builder (DIB) elements. See # stackhpc.os-images role for usage. Default is a combination of diff --git a/etc/kayobe/stackhpc-overcloud-dib.yml b/etc/kayobe/stackhpc-overcloud-dib.yml new file mode 100644 index 000000000..0482cae0e --- /dev/null +++ b/etc/kayobe/stackhpc-overcloud-dib.yml @@ -0,0 +1,212 @@ +--- +# StackHPC overcloud host disk image configuration. + +############################################################################### +# Diskimage-builder configuration for overcloud host disk images. + +# StackHPC overcloud host disk image Diskimage Builder recipe. This may be used +# as an item in the overcloud_dib_host_images list when +# overcloud_dib_build_host_images is true. +# Example: +# overcloud_dib_host_images: +# - "{{ stackhpc_overcloud_dib_host_image }}" +stackhpc_overcloud_dib_host_image: + name: "{{ stackhpc_overcloud_dib_name }}" + elements: "{{ stackhpc_overcloud_dib_elements }}" + env: "{{ stackhpc_overcloud_dib_env_vars }}" + packages: "{{ stackhpc_overcloud_dib_packages }}" + +# StackHPC overcloud DIB image name. +stackhpc_overcloud_dib_name: "deployment_image" + +# StackHPC overcloud DIB image elements. +stackhpc_overcloud_dib_elements: + - "{{ os_distribution }}-{% if os_distribution == 'rocky' %}container{% else %}minimal{% endif %}" + - "cloud-init-datasources" + - "{% if os_distribution in ['centos', 'rocky'] %}disable-selinux{% endif %}" + - "enable-serial-console" + - "vm" + - "block-device-efi" + - "cloud-init" + - "{% if os_distribution in ['centos', 'rocky'] %}dracut-regenerate{% endif %}" + - "{% if os_distribution == 'ubuntu' %}lvm{% endif %}" + - "openssh-server" + - "{% if os_distribution == 'ubuntu' %}sudoers{% endif %}" + +# StackHPC overcloud DIB image environment variables. +stackhpc_overcloud_dib_env_vars: + DIB_BLOCK_DEVICE_CONFIG: "{{ stackhpc_overcloud_dib_block_device_config_uefi_lvm }}" + DIB_BOOTLOADER_DEFAULT_CMDLINE: "nofb nomodeset gfxpayload=text net.ifnames=1 rd.auto" + DIB_CLOUD_INIT_DATASOURCES: "ConfigDrive" + DIB_CONTAINERFILE_RUNTIME: "docker" + DIB_CONTAINERFILE_NETWORK_DRIVER: "host" + DIB_DISTRIBUTION_MIRROR: "{{ stackhpc_repo_ubuntu_focal_url if os_distribution == 'ubuntu' else '' }}" + DIB_DRACUT_ENABLED_MODULES_DEFAULT_CONFIG: "{{ stackhpc_overcloud_dib_dracut_enabled_modules_default_config }}" + DIB_RELEASE: "{{ overcloud_dib_os_release }}" + DIB_SUDOERS_FILENAME: "no-fqdn" + # Avoid DNS queries during sudo commands, since we might not always have working DNS. + DIB_SUDOERS_CONFIG: | + Defaults !fqdn + # FIXME: Support templating repo files. + # DIB_YUM_MINIMAL_BOOTSTRAP_REPOS: /path/to/dir/containing/dib-mirror-*.repo + YUM: dnf + +# StackHPC overcloud DIB image packages. +stackhpc_overcloud_dib_packages: + - "logrotate" + - "net-tools" + +# StackHPC overcloud DIB image block device configuration. +# This image layout conforms to the CIS partition benchmarks. +# This configuration builds a UEFI-compatible image with 3 partitions. +# * p0: EFI ESP bootloader +# * p1: EFI BSP +# * p2: LVM PV (rootpv) +# The rootpv PV is in the rootvg VG, and has the following LVs: +# * lv_root -> / +# * lv_tmp -> /tmp +# * lv_var -> /var +# * lv_var_tmp -> /var/tmp +# * lv_log -> /var/log +# * lv_audit -> /var/log/audit +# * lv_home -> /home +stackhpc_overcloud_dib_block_device_config_uefi_lvm: | + - local_loop: + name: image0 + size: 20GiB + - partitioning: + base: image0 + label: gpt + partitions: + - name: ESP + type: 'EF00' + size: 550MiB + mkfs: + type: vfat + mount: + mount_point: /boot/efi + fstab: + options: "defaults" + fsck-passno: 2 + - name: BSP + type: 'EF02' + size: 8MiB + - name: root + type: '8E00' + flags: [ boot ] + size: 100% + - lvm: + name: lvm + base: [ root ] + pvs: + - name: rootpv + base: root + options: [ "--force" ] + vgs: + - name: rootvg + base: [ "rootpv" ] + options: [ "--force" ] + lvs: + - name: lv_root + base: rootvg + size: 5G + - name: lv_tmp + base: rootvg + size: 1G + - name: lv_var + base: rootvg + size: 1G + - name: lv_var_tmp + base: rootvg + size: 1G + - name: lv_log + base: rootvg + size: 1G + - name: lv_audit + base: rootvg + size: 128M + - name: lv_home + base: rootvg + size: 128M + - mkfs: + name: fs_root + base: lv_root + type: ext4 + label: "rootfs" + mount: + mount_point: / + fstab: + options: "defaults" + fsck-passno: 1 + - mkfs: + name: fs_tmp + base: lv_tmp + type: ext4 + label: "tmpfs" + mount: + mount_point: /tmp + fstab: + options: "rw,noexec,nosuid,nodev" + fsck-passno: 2 + - mkfs: + name: fs_var + base: lv_var + type: ext4 + label: "varfs" + mount: + mount_point: /var + fstab: + options: "defaults" + fsck-passno: 2 + - mkfs: + name: fs_var_tmp + base: lv_var_tmp + type: ext4 + label: "vartmpfs" + mount: + mount_point: /var/tmp + fstab: + options: "rw,noexec,nosuid,nodev" + fsck-passno: 2 + - mkfs: + name: fs_log + base: lv_log + type: ext4 + label: "logfs" + mount: + mount_point: /var/log + fstab: + options: "defaults" + fsck-passno: 2 + - mkfs: + name: fs_audit + base: lv_audit + type: ext4 + label: "auditfs" + mount: + mount_point: /var/log/audit + fstab: + options: "defaults" + fsck-passno: 2 + - mkfs: + name: fs_home + base: lv_home + type: ext4 + label: "homefs" + mount: + mount_point: /home + fstab: + options: "rw,nodev" + fsck-passno: 2 + +# StackHPC overcloud DIB image Dracut module configuration. +stackhpc_overcloud_dib_dracut_enabled_modules_default_config: | + - name: crypt + packages: + - cryptsetup + - name: lvm + packages: + - lvm2 + - name: mdraid + packages: + - mdraid diff --git a/releasenotes/notes/overcloud-dib-938eac005662a2cb.yaml b/releasenotes/notes/overcloud-dib-938eac005662a2cb.yaml new file mode 100644 index 000000000..c8178302a --- /dev/null +++ b/releasenotes/notes/overcloud-dib-938eac005662a2cb.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Adds a standard overcloud Diskimage Builder (DIB) host image configuration. From 4102cdeac08926cc7392f83884ae3f12f92bbc81 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 28 Jul 2022 15:47:42 +0000 Subject: [PATCH 029/292] Add StackHPC LVM configuration --- .../inventory/group_vars/all/stackhpc-lvm | 150 ++++++++++++++++++ releasenotes/notes/lvm-3c785770722f768a.yaml | 5 + 2 files changed, 155 insertions(+) create mode 100644 etc/kayobe/inventory/group_vars/all/stackhpc-lvm create mode 100644 releasenotes/notes/lvm-3c785770722f768a.yaml diff --git a/etc/kayobe/inventory/group_vars/all/stackhpc-lvm b/etc/kayobe/inventory/group_vars/all/stackhpc-lvm new file mode 100644 index 000000000..8d710ba0d --- /dev/null +++ b/etc/kayobe/inventory/group_vars/all/stackhpc-lvm @@ -0,0 +1,150 @@ +--- +############################################################################### +# StackHPC LVM Volume Group (VG) configuration. + +# StackHPC rootvg VG. +# This VG contains a set of LVs that conform to the CIS partition benchmarks. +# This layout is compatible with the overcloud DIB host image configuration in +# stackhpc-overcloud-dib.yml. This may be used as an item in one of the +# *_lvm_groups variables. +# Example: +# controller_lvm_groups: +# - "{{ stackhpc_lvm_group_rootvg }}" +stackhpc_lvm_group_rootvg: + vgname: rootvg + disks: "{{ stackhpc_lvm_group_rootvg_disks }}" + create: true + lvnames: "{{ stackhpc_lvm_group_rootvg_lvs_default + stackhpc_lvm_group_rootvg_lvs_extra }}" + +# List of disks to include in the rootvg VG. +stackhpc_lvm_group_rootvg_disks: + - /dev/disk/by-partlabel/root + +# List of default LVs to include in the rootvg VG. +stackhpc_lvm_group_rootvg_lvs_default: + - "{{ stackhpc_lvm_lv_swap }}" + - "{{ stackhpc_lvm_lv_root }}" + - "{{ stackhpc_lvm_lv_tmp }}" + - "{{ stackhpc_lvm_lv_var }}" + - "{{ stackhpc_lvm_lv_var_tmp }}" + - "{{ stackhpc_lvm_lv_log }}" + - "{{ stackhpc_lvm_lv_audit }}" + - "{{ stackhpc_lvm_lv_home }}" + +# List of extra LVs to include in the rootvg VG. +stackhpc_lvm_group_rootvg_lvs_extra: [] + +############################################################################### +# StackHPC LVM Logical Volume (LV) configuration. + +# StackHPC LVM lv_swap LV size. +stackhpc_lvm_lv_swap_size: 16g + +# StackHPC LVM lv_root LV size. +stackhpc_lvm_lv_root_size: 50g + +# StackHPC LVM lv_tmp LV size. +stackhpc_lvm_lv_tmp_size: 10g + +# StackHPC LVM lv_var LV size. +stackhpc_lvm_lv_var_size: 20g + +# StackHPC LVM lv_var_tmp LV size. +stackhpc_lvm_lv_var_tmp_size: 2g + +# StackHPC LVM lv_log LV size. +stackhpc_lvm_lv_log_size: 20g + +# StackHPC LVM lv_audit LV size. +stackhpc_lvm_lv_audit_size: 10g + +# StackHPC LVM lv_home LV size. +stackhpc_lvm_lv_home_size: 10g + +# StackHPC LVM lv_docker LV size. +stackhpc_lvm_lv_docker_size: 100%FREE + +# StackHPC LVM lv_swap LV. +stackhpc_lvm_lv_swap: + lvname: lv_swap + size: "{{ stackhpc_lvm_lv_swap_size }}" + create: true + mount: false + +# StackHPC LVM lv_root LV. +stackhpc_lvm_lv_root: + lvname: lv_root + size: "{{ stackhpc_lvm_lv_root_size }}" + create: true + filesystem: ext4 + mount: true + mntp: / + +# StackHPC LVM lv_tmp LV. +stackhpc_lvm_lv_tmp: + lvname: lv_tmp + size: "{{ stackhpc_lvm_lv_tmp_size }}" + create: true + filesystem: ext4 + mount: true + mntp: /tmp + +# StackHPC LVM lv_var LV. +stackhpc_lvm_lv_var: + lvname: lv_var + size: "{{ stackhpc_lvm_lv_var_size }}" + create: true + filesystem: ext4 + mount: true + mntp: /var + +# StackHPC LVM lv_var_tmp LV. +stackhpc_lvm_lv_var_tmp: + lvname: lv_var_tmp + size: "{{ stackhpc_lvm_lv_var_tmp_size }}" + create: true + filesystem: ext4 + mount: true + mntp: /var/tmp + +# StackHPC LVM lv_log LV. +stackhpc_lvm_lv_log: + lvname: lv_log + size: "{{ stackhpc_lvm_lv_log_size }}" + create: true + filesystem: ext4 + mount: true + mntp: /var/log + +# StackHPC LVM lv_audit LV. +stackhpc_lvm_lv_audit: + lvname: lv_audit + size: "{{ stackhpc_lvm_lv_audit_size }}" + create: true + filesystem: ext4 + mount: true + mntp: /var/log/audit + +# StackHPC LVM lv_home LV. +stackhpc_lvm_lv_home: + lvname: lv_home + size: "{{ stackhpc_lvm_lv_home_size }}" + create: true + filesystem: ext4 + mount: true + mntp: /home + +# StackHPC LVM lv_docker LV. +# NOTE: This is not included in the rootvg by default, but may be added via +# stackhpc_lvm_group_rootvg_lvs_extra. +# Example: +# stackhpc_lvm_group_rootvg_lvs_extra: +# - "{{ stackhpc_lvm_lv_docker }}" +# Alternatively, this may reside in a separate VG. +stackhpc_lvm_lv_docker: + lvname: lv_docker + size: "{{ stackhpc_lvm_lv_docker_size }}" + create: true + filesystem: ext4 + mount: true + mntp: /var/lib/docker diff --git a/releasenotes/notes/lvm-3c785770722f768a.yaml b/releasenotes/notes/lvm-3c785770722f768a.yaml new file mode 100644 index 000000000..5818c3e23 --- /dev/null +++ b/releasenotes/notes/lvm-3c785770722f768a.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Adds a standard LVM configuration that is compatible with the new overcloud + host image. From f093d1f7a79d3a0afd47eb0819545956a13a9347 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 22 Jul 2022 11:22:36 +0100 Subject: [PATCH 030/292] Add swap.yml custom playbook Creates a swap filesystem on the lv_swap LV in the rootvg VG. May be added as a post hook to overcloud host configure. --- etc/kayobe/ansible/swap.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 etc/kayobe/ansible/swap.yml diff --git a/etc/kayobe/ansible/swap.yml b/etc/kayobe/ansible/swap.yml new file mode 100644 index 000000000..e47397212 --- /dev/null +++ b/etc/kayobe/ansible/swap.yml @@ -0,0 +1,35 @@ +--- +# Custom playbook to configure a swap device. This may be used as a +# post-overcloud host configure hook. +# +# Variables: +# * swap_group: Host pattern against which to target the playbook. Default is +# 'overcloud'. +# * swap_device: Name of the swap device to configure. Default is +# '/dev/rootvg/lv_swap'. + +- name: Configure swap + hosts: "{{ swap_group | default('overcloud') }}" + tags: + - swap + vars: + swap_device: "/dev/rootvg/lv_swap" + become: true + tasks: + - name: Ensure swap filesystem is present + filesystem: + fstype: "swap" + dev: "{{ swap_device }}" + + - name: Ensure swap device present in fstab + mount: + name: "none" + src: "{{ swap_device }}" + fstype: "swap" + state: "present" + + # It does no harm to run this when swap is already active + - name: Enable swap devices + command: "/sbin/swapon -a" + when: + - ansible_facts.swaptotal_mb == 0 From 5ef8de0e558a45583c1691a8322f4dbb0b76e4b8 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 27 Jul 2022 15:05:48 +0000 Subject: [PATCH 031/292] Add growroot.yml custom playbook Grows the root partition & rootpv LVM PV to fill the available space. This playbook may be added as a pre-hook for overcloud host configure. --- etc/kayobe/ansible/growroot.yml | 62 +++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 etc/kayobe/ansible/growroot.yml diff --git a/etc/kayobe/ansible/growroot.yml b/etc/kayobe/ansible/growroot.yml new file mode 100644 index 000000000..62c7eed1f --- /dev/null +++ b/etc/kayobe/ansible/growroot.yml @@ -0,0 +1,62 @@ +--- +# Custom playbook to grow the partition and LVM PV of the root VG. This allows +# for expansion of LVs in that VG. This may be used as a pre-overcloud host +# configure hook. +# +# Variables: +# * growroot_group: Host pattern against which to target the playbook. Default +# is 'overcloud'. +# * growroot_vg: Name of the VG containing the PV to grow. Default is 'rootvg'. + +- name: Grow root PV + hosts: "{{ growroot_group | default('overcloud') }}" + # Avoid using facts because this may be used as a pre overcloud host + # configure hook, and we don't want to populate the fact cache (if one is in + # use) with the bootstrap user's context. + gather_facts: false + tags: + - growroot + vars: + ansible_user: "{{ bootstrap_user }}" + # We can't assume that a virtualenv exists at this point, so use the system + # python interpreter. + ansible_python_interpreter: /usr/bin/python3 + # Work around no known_hosts entry on first boot. + ansible_ssh_common_args: "-o StrictHostKeyChecking=no" + # Name of the LVM VG containing the root PV. + growroot_vg: "rootvg" + + tasks: + - name: Ensure growpart is installed + package: + name: "{% if os_distribution in ['centos', 'rocky'] %}cloud-utils-growpart{% else %}cloud-guest-utils{% endif %}" + state: present + cache_valid_time: "{{ apt_cache_valid_time if ansible_facts.os_family == 'Debian' else omit }}" + update_cache: "{{ True if ansible_facts.os_family == 'Debian' else omit }}" + become: True + + - name: Get root PV device + command: "pvs --select vg_name={{ growroot_vg }} --reportformat json" + register: pvs + become: True + changed_when: False + + - name: Grow partition + command: "growpart {{ disk }} {{ part_num }}" + vars: + pv: "{{ pvs.stdout | from_json }}" + disk_tmp: "{{ pv.report[0].pv[0].pv_name[:-1] }}" + disk: "{{ disk_tmp[:-1] if disk_tmp[-1] == 'p' else disk_tmp }}" + part_num: "{{ pv.report[0].pv[0].pv_name[-1] }}" + become: True + failed_when: "growpart.rc != 0 and 'NOCHANGE' not in growpart.stdout" + changed_when: "'NOCHANGE' not in growpart.stdout" + register: growpart + + - name: Grow LVM PV + command: "pvresize {{ disk }}" + vars: + pv: "{{ pvs.stdout | from_json }}" + disk: "{{ pv.report[0].pv[0].pv_name }}" + become: True + when: "'NOCHANGE' not in growpart.stdout" From a47748d9fcde457fe3fc2e5553a982594933fa38 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 28 Jul 2022 15:18:31 +0100 Subject: [PATCH 032/292] Xena: drop downstream Barbican We are not carrying downstream Barbican patches in Xena. --- etc/kayobe/kolla.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index f13ff205c..7ff7aae3a 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -111,10 +111,6 @@ kolla_openstack_release: TODO # location: https://github.com/openstack/ironic # reference: master kolla_sources: - barbican-base: - type: git - location: "https://github.com/stackhpc/barbican.git" - reference: stackhpc/xena bifrost-base: type: git location: "{{ kolla_bifrost_source_url }}" From 2c9048761f52231ee9549c7e9c6c1c86f4cccc03 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 28 Jul 2022 15:19:21 +0100 Subject: [PATCH 033/292] Xena: Replace kolla build blocks with ARG overrides Some of the build blocks now include a URL, which we do not need to override. --- etc/kayobe/kolla.yml | 29 ++++++++++++----------------- etc/kayobe/kolla/kolla-build.conf | 2 ++ 2 files changed, 14 insertions(+), 17 deletions(-) create mode 100644 etc/kayobe/kolla/kolla-build.conf diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 7ff7aae3a..eee50bdab 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -227,23 +227,6 @@ kolla_build_blocks: RUN grafana-cli plugins install vonage-status-panel ironic_inspector_header: | ADD additions-archive / - prometheus_v2_server_repository_version: | - ARG prometheus_version='2.35.0' - prometheus_alertmanager_repository_version: | - ARG prometheus_alertmanager_version='0.24.0' - prometheus_cadvisor_repository_version: | - ARG prometheus_cadvisor_version='0.40.0' - ARG prometheus_cadvisor_sha256sum='8e3df91eee70c72ac3f3184b9fab1229b037c2e850ff1593103b2bd9028c57c0' - node_exporter_repository_version: | - ARG node_exporter_version='1.3.1' - prometheus_memcached_exporter_repository_version: | - ARG memcached_exporter_version='0.9.0' - haproxy_exporter_repository_version: | - ARG haproxy_exporter_version='0.13.0' - mysqld_exporter_repository_version: | - ARG mysqld_exporter_version='0.13.0' - prometheus_blackbox_exporter_repository_version: | - ARG blackbox_exporter_version='0.19.0' # Dict mapping image customization variable names to their values. # Each variable takes the form: @@ -286,6 +269,18 @@ kolla_build_customizations_ubuntu: {} # override, append or remove. The value should be a list. kolla_build_customizations: "{{ kolla_build_customizations_common | combine(kolla_build_customizations_centos if kolla_base_distro == 'centos' else kolla_build_customizations_ubuntu) }}" +# Dict mapping Kolla Dockerfile ARG names to their values. +kolla_build_args: + blackbox_exporter_version: "0.19.0" + haproxy_exporter_version: "0.13.0" + memcached_exporter_version: "0.9.0" + mysqld_exporter_version: "0.13.0" + node_exporter_version: "1.3.1" + prometheus_version: "2.35.0" + prometheus_alertmanager_version: "0.24.0" + prometheus_cadvisor_sha256sum: "8e3df91eee70c72ac3f3184b9fab1229b037c2e850ff1593103b2bd9028c57c0" + prometheus_cadvisor_version: "0.40.0" + ############################################################################### # Kolla-ansible inventory configuration. diff --git a/etc/kayobe/kolla/kolla-build.conf b/etc/kayobe/kolla/kolla-build.conf new file mode 100644 index 000000000..83d8bb715 --- /dev/null +++ b/etc/kayobe/kolla/kolla-build.conf @@ -0,0 +1,2 @@ +[DEFAULT] +build_args = {{ kolla_build_args.items() | map('join', ':') | join(',') }} From 2f18bcb3fdffbe5112bf21c5e1c35575cbc40361 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 28 Jul 2022 15:41:02 +0100 Subject: [PATCH 034/292] Xena: add initial Kolla tags --- etc/kayobe/kolla.yml | 2 +- etc/kayobe/kolla/globals.yml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index eee50bdab..5fdd860f5 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -96,7 +96,7 @@ kolla_docker_registry_password: "{{ stackhpc_docker_registry_password }}" # Kolla OpenStack release version. This should be a Docker image tag. # Default is {{ openstack_release }}. -kolla_openstack_release: TODO +kolla_openstack_release: xena-20220728T100435 # Docker tag applied to built container images. Default is # {{ kolla_openstack_release }}. diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 5505501fb..88b1559cc 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -4,3 +4,6 @@ enable_docker_repo: "{% raw %}{{ ansible_facts.os_family == 'RedHat' }}{% endraw docker_yum_baseurl: "{{ stackhpc_repo_docker_url }}" docker_yum_gpgkey: "https://download.docker.com/linux/{% raw %}{{ ansible_facts.distribution | lower }}{% endraw %}/gpg" + +barbican_tag: xena-20220728T143658 +prometheus_tag: xena-20220728T143658 From 51148dbcf2fcfef1e1c5008a11175aa1fbf4b4b9 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 7 Jun 2022 10:31:15 +0100 Subject: [PATCH 035/292] docker: enable live restore --- etc/kayobe/docker.yml | 2 +- releasenotes/notes/docker-live-restore-6b6c55455202422c.yaml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/docker-live-restore-6b6c55455202422c.yaml diff --git a/etc/kayobe/docker.yml b/etc/kayobe/docker.yml index fcd7c8eb0..c4ac21c3b 100644 --- a/etc/kayobe/docker.yml +++ b/etc/kayobe/docker.yml @@ -31,7 +31,7 @@ docker_registry: "{{ stackhpc_docker_registry }}" #docker_registry_mirrors: # Enable live-restore on docker daemon -#docker_daemon_live_restore: +docker_daemon_live_restore: true ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/releasenotes/notes/docker-live-restore-6b6c55455202422c.yaml b/releasenotes/notes/docker-live-restore-6b6c55455202422c.yaml new file mode 100644 index 000000000..8c516d3d0 --- /dev/null +++ b/releasenotes/notes/docker-live-restore-6b6c55455202422c.yaml @@ -0,0 +1,5 @@ +--- +upgrade: + - | + Enables Docker live restore by default. This may be disabled by setting + ``docker_daemon_live_restore`` to ``false`` in ``docker.yml``. From e913d76110f4a23c7150a66dffb925e4e2ae9478 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 7 Jun 2022 15:01:57 +0100 Subject: [PATCH 036/292] Add basic Ansible tuning * Set forks to 20 * YAML-formatted output * Disable fact variable injection * Enable SSH pipelining --- etc/kayobe/ansible.cfg | 11 +++++++++++ .../notes/ansible-tuning-142bbcf4f646e6e0.yaml | 6 ++++++ 2 files changed, 17 insertions(+) create mode 100644 etc/kayobe/ansible.cfg create mode 100644 releasenotes/notes/ansible-tuning-142bbcf4f646e6e0.yaml diff --git a/etc/kayobe/ansible.cfg b/etc/kayobe/ansible.cfg new file mode 100644 index 000000000..696ed923a --- /dev/null +++ b/etc/kayobe/ansible.cfg @@ -0,0 +1,11 @@ +[defaults] +forks = 20 +# Use the YAML stdout callback plugin. +stdout_callback = yaml +# Use the stdout_callback when running ad-hoc commands. +bin_ansible_callbacks = True +# Disable fact variable injection to improve performance. +inject_facts_as_vars = False + +[ssh_connection] +pipelining = True diff --git a/releasenotes/notes/ansible-tuning-142bbcf4f646e6e0.yaml b/releasenotes/notes/ansible-tuning-142bbcf4f646e6e0.yaml new file mode 100644 index 000000000..13e4144e0 --- /dev/null +++ b/releasenotes/notes/ansible-tuning-142bbcf4f646e6e0.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Adds some basic tuning of Ansible, including use of 20 forks, enabling + SSH pipelining, YAML-formatted output, and disabling fact variable + injection. From 2410d77f5574408eb614f6c46a5ca412d43aa3ee Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 28 Jul 2022 15:42:17 +0100 Subject: [PATCH 037/292] Xena: use Kayobe stackhpc/xena branch in requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 73ef04b58..ae14ea044 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -git+https://github.com/stackhpc/kayobe@stackhpc/wallaby +git+https://github.com/stackhpc/kayobe@stackhpc/xena From 451bcc5f82f0890dbd29b8bdca8947112ac2e064 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 9 Aug 2022 11:28:05 +0100 Subject: [PATCH 038/292] Xena: use openstack_release for UCA distribution --- etc/kayobe/kolla.yml | 2 +- etc/kayobe/pulp.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 5fdd860f5..df6780e69 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -194,7 +194,7 @@ stackhpc_ubuntu_focal_repos: - "deb {{ stackhpc_repo_ubuntu_focal_url }} focal-updates main universe" - "deb {{ stackhpc_repo_ubuntu_focal_url }} focal-backports main universe" - "deb {{ stackhpc_repo_ubuntu_focal_security_url }} focal-security main universe" - - "deb {{ stackhpc_repo_ubuntu_cloud_archive_url }} focal-updates/wallaby main" + - "deb {{ stackhpc_repo_ubuntu_cloud_archive_url }} focal-updates/{{ openstack_release }} main" # Dict mapping Jinja2 block names in kolla's Docker images to their contents. kolla_build_blocks: diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index adf328cf2..94dcf6df9 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -68,7 +68,7 @@ stackhpc_pulp_repository_deb_repos: policy: on_demand architectures: amd64 components: main - distributions: focal-updates/wallaby + distributions: "focal-updates/{{ openstack_release }}" mirror: true state: present required: "{{ stackhpc_pulp_sync_ubuntu_focal | bool }}" From a949d6e1b7730856f2bd251ff20754b9b13e7863 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 9 Aug 2022 14:18:22 +0100 Subject: [PATCH 039/292] Update kayobe-automation to fix rsync issue --- .automation | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.automation b/.automation index d7cab138e..a5749321e 160000 --- a/.automation +++ b/.automation @@ -1 +1 @@ -Subproject commit d7cab138e827011b2574cd420d29c06fbd8dc56b +Subproject commit a5749321ebcaa9798f1d63d7dc15f1380540f0a3 From ef12f8b34d56484cb3a0c1594241022f6d23878a Mon Sep 17 00:00:00 2001 From: stackhpc-ci <22933334+stackhpc-ci@users.noreply.github.com> Date: Wed, 10 Aug 2022 15:03:08 +0000 Subject: [PATCH 040/292] feat: automatic update of workflows stackhpc/xena --- .github/workflows/tag-and-release.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/workflows/tag-and-release.yml diff --git a/.github/workflows/tag-and-release.yml b/.github/workflows/tag-and-release.yml new file mode 100644 index 000000000..7180b59e7 --- /dev/null +++ b/.github/workflows/tag-and-release.yml @@ -0,0 +1,11 @@ +--- +name: Tag & Release +'on': + push: + branches: + - stackhpc/xena +permissions: + contents: write +jobs: + tag-and-release: + uses: stackhpc/.github/.github/workflows/tag-and-release.yml@main From 014ac7375446880d2cd360f22336a73b436ca460 Mon Sep 17 00:00:00 2001 From: stackhpc-ci <22933334+stackhpc-ci@users.noreply.github.com> Date: Wed, 10 Aug 2022 15:04:10 +0000 Subject: [PATCH 041/292] feat: automatic update of community files stackhpc/xena --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..e9a948a0d --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @stackhpc/kayobe From 912dfae8fef338f993523e0353f0c02092d335d6 Mon Sep 17 00:00:00 2001 From: stackhpc-ci <22933334+stackhpc-ci@users.noreply.github.com> Date: Wed, 10 Aug 2022 15:52:30 +0000 Subject: [PATCH 042/292] feat: automatic update of workflows stackhpc/xena --- .github/workflows/tag-and-release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tag-and-release.yml b/.github/workflows/tag-and-release.yml index 7180b59e7..af47d87b5 100644 --- a/.github/workflows/tag-and-release.yml +++ b/.github/workflows/tag-and-release.yml @@ -9,3 +9,5 @@ permissions: jobs: tag-and-release: uses: stackhpc/.github/.github/workflows/tag-and-release.yml@main + with: + upstream: https://github.com/openstack/kayobe-config From 817bea96fc73097ec141a98ddc7c5da1ae39a1d4 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 24 Aug 2022 09:18:46 +0100 Subject: [PATCH 043/292] growpart.yml: Don't assume facts are present --- etc/kayobe/ansible/growroot.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/etc/kayobe/ansible/growroot.yml b/etc/kayobe/ansible/growroot.yml index 62c7eed1f..dd73e168a 100644 --- a/etc/kayobe/ansible/growroot.yml +++ b/etc/kayobe/ansible/growroot.yml @@ -25,14 +25,16 @@ ansible_ssh_common_args: "-o StrictHostKeyChecking=no" # Name of the LVM VG containing the root PV. growroot_vg: "rootvg" + # Don't assume facts are present. + os_family: "{{ ansible_facts.os_family | default('Debian' if os_distribution == 'ubuntu' else 'RedHat') }}" tasks: - name: Ensure growpart is installed package: - name: "{% if os_distribution in ['centos', 'rocky'] %}cloud-utils-growpart{% else %}cloud-guest-utils{% endif %}" + name: "{% if os_family == 'RedHat' %}cloud-utils-growpart{% else %}cloud-guest-utils{% endif %}" state: present - cache_valid_time: "{{ apt_cache_valid_time if ansible_facts.os_family == 'Debian' else omit }}" - update_cache: "{{ True if ansible_facts.os_family == 'Debian' else omit }}" + cache_valid_time: "{{ apt_cache_valid_time if os_family == 'Debian' else omit }}" + update_cache: "{{ True if os_family == 'Debian' else omit }}" become: True - name: Get root PV device From 04f1582dc16d1f0b96f1e69b2db3572da67222f5 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 24 Aug 2022 09:34:51 +0100 Subject: [PATCH 044/292] Overcloud DIB: stop using StackHPC package repos for Ubuntu image We are not currently syncing Ubuntu packages, since the on_demand mirror in Ark does not work if the upstream mirror pulls packages (which it does sometimes). --- etc/kayobe/stackhpc-overcloud-dib.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/stackhpc-overcloud-dib.yml b/etc/kayobe/stackhpc-overcloud-dib.yml index 0482cae0e..84fffbc0f 100644 --- a/etc/kayobe/stackhpc-overcloud-dib.yml +++ b/etc/kayobe/stackhpc-overcloud-dib.yml @@ -40,7 +40,10 @@ stackhpc_overcloud_dib_env_vars: DIB_CLOUD_INIT_DATASOURCES: "ConfigDrive" DIB_CONTAINERFILE_RUNTIME: "docker" DIB_CONTAINERFILE_NETWORK_DRIVER: "host" - DIB_DISTRIBUTION_MIRROR: "{{ stackhpc_repo_ubuntu_focal_url if os_distribution == 'ubuntu' else '' }}" + # NOTE: Not currently syncing Ubuntu packages, since the on_demand mirror in + # Ark does not work if the upstream mirror pulls packages (which it does + # sometimes). + # DIB_DISTRIBUTION_MIRROR: "{{ stackhpc_repo_ubuntu_focal_url if os_distribution == 'ubuntu' else '' }}" DIB_DRACUT_ENABLED_MODULES_DEFAULT_CONFIG: "{{ stackhpc_overcloud_dib_dracut_enabled_modules_default_config }}" DIB_RELEASE: "{{ overcloud_dib_os_release }}" DIB_SUDOERS_FILENAME: "no-fqdn" From 3931b6eebb58f15737d6735b11c17b39d0a471b7 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 24 Aug 2022 12:17:18 +0200 Subject: [PATCH 045/292] Support filtering Kolla container images to sync/publish It can take a long time to sync and publish all Kolla container images to the local Pulp service. Sometimes we know we just need to update a few images. This change adds support for filtering the image list, using regexes in a similar way to kolla-build. Example: kayobe playbook run ... -e stackhpc_pulp_images_kolla_filter='"^glance nova-compute$"' --- etc/kayobe/pulp.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index e1a8e1526..bdc91053e 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -428,6 +428,15 @@ stackhpc_pulp_images_kolla: - redis-base - redis-sentinel +# Whitespace-separated list of regular expressions matching Kolla image names. +# Usage is similar to kolla-build CLI arguments. +# Example: +# kayobe playbook run ... -e stackhpc_pulp_images_kolla_filter='"^glance nova-compute$"' +stackhpc_pulp_images_kolla_filter: ".*" + +# Filtered list of Kolla container image names. +stackhpc_pulp_images_kolla_filtered: "{{ stackhpc_pulp_images_kolla | select('search', '(' ~ stackhpc_pulp_images_kolla_filter.split() | join('|') ~ ')') | list }}" + # Common parameters for container image repositories. stackhpc_pulp_repository_container_repos_kolla_common: url: "{{ stackhpc_release_pulp_registry_url }}" @@ -440,7 +449,7 @@ stackhpc_pulp_repository_container_repos_kolla_common: # List of Kolla container image repositories. stackhpc_pulp_repository_container_repos_kolla: >- {%- set repos = [] -%} - {%- for image in stackhpc_pulp_images_kolla -%} + {%- for image in stackhpc_pulp_images_kolla_filtered -%} {%- set image_repo = kolla_docker_namespace ~ "/" ~ kolla_base_distro ~ "-" ~ kolla_install_type ~ "-" ~ image -%} {%- set repo = {"name": image_repo} -%} {%- set _ = repos.append(stackhpc_pulp_repository_container_repos_kolla_common | combine(repo)) -%} @@ -455,7 +464,7 @@ stackhpc_pulp_distribution_container_kolla_common: # List of Kolla container image distributions. stackhpc_pulp_distribution_container_kolla: >- {%- set distributions = [] -%} - {%- for image in stackhpc_pulp_images_kolla -%} + {%- for image in stackhpc_pulp_images_kolla_filtered -%} {%- set image_repo = kolla_docker_namespace ~ "/" ~ kolla_base_distro ~ "-" ~ kolla_install_type ~ "-" ~ image -%} {%- set distribution = {"name": image_repo, "repository": image_repo, "base_path": image_repo} -%} {%- set _ = distributions.append(stackhpc_pulp_distribution_container_kolla_common | combine(distribution)) -%} From 0f61e4eaa170573d8144c66308616d15de13c6bb Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 25 Aug 2022 11:54:14 +0100 Subject: [PATCH 046/292] growroot.yml: Fail if the expected volume group doesn't exist --- etc/kayobe/ansible/growroot.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/etc/kayobe/ansible/growroot.yml b/etc/kayobe/ansible/growroot.yml index dd73e168a..40f3476fb 100644 --- a/etc/kayobe/ansible/growroot.yml +++ b/etc/kayobe/ansible/growroot.yml @@ -43,6 +43,12 @@ become: True changed_when: False + - name: Fail if root PV device not found + fail: + msg: > + Expected LVM physical volume devices not found in volume group {{ growroot_vg }} + when: (pvs.stdout | from_json).report[0].pv | length == 0 + - name: Grow partition command: "growpart {{ disk }} {{ part_num }}" vars: From 10b96b423f85fc73609e7e63f89b9d315788e529 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 26 Aug 2022 16:07:04 +0100 Subject: [PATCH 047/292] growroot: Avoid package installation when growpart is installed In a pre-bootstrapped environment which requires a proxy for package installation, we may not be able to perform package updates. --- etc/kayobe/ansible/growroot.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/etc/kayobe/ansible/growroot.yml b/etc/kayobe/ansible/growroot.yml index 40f3476fb..6bfe63e41 100644 --- a/etc/kayobe/ansible/growroot.yml +++ b/etc/kayobe/ansible/growroot.yml @@ -29,6 +29,14 @@ os_family: "{{ ansible_facts.os_family | default('Debian' if os_distribution == 'ubuntu' else 'RedHat') }}" tasks: + - name: Check if growpart is installed + shell: + cmd: type growpart + changed_when: false + failed_when: false + register: growpart_check + become: true + - name: Ensure growpart is installed package: name: "{% if os_family == 'RedHat' %}cloud-utils-growpart{% else %}cloud-guest-utils{% endif %}" @@ -36,6 +44,7 @@ cache_valid_time: "{{ apt_cache_valid_time if os_family == 'Debian' else omit }}" update_cache: "{{ True if os_family == 'Debian' else omit }}" become: True + when: growpart_check.rc != 0 - name: Get root PV device command: "pvs --select vg_name={{ growroot_vg }} --reportformat json" From e9373343c4f2af8f2b72b75cacdd14d9b64cc2e5 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 31 Aug 2022 15:05:05 +0100 Subject: [PATCH 048/292] Add ceph group as a child of the storage group --- etc/kayobe/inventory/groups | 3 +++ 1 file changed, 3 insertions(+) diff --git a/etc/kayobe/inventory/groups b/etc/kayobe/inventory/groups index e51195309..570779a93 100644 --- a/etc/kayobe/inventory/groups +++ b/etc/kayobe/inventory/groups @@ -101,6 +101,9 @@ hs-switches ############################################################################### # Ceph groups. +[storage:children] +ceph + [ceph:children] mons mgrs From b9746b12ee9375b5846d5c6a3c2fc2001bd8db68 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 31 Aug 2022 20:11:52 +0100 Subject: [PATCH 049/292] Adds Rocky 8 repository config for local pulp (#146) * Adds Rocky 8 repository config for local pulp We support Rocky 8 as a host OS with os_distribution == 'rocky'. This adds the configuration necessary to use the RPM snapshots from ark. * Abstract over version number v1 Unsure if it would be better to: 1) Update the remote on the repository 2) Change the publication to point to a new repository and keep repositories for every minor version Advantages of 1: - Single repository in pulp Advantages of 2: - Less chance of conflicts (although I think an old repo is just an old snapshot) * Fixups * Address code review comment Upadting variable name after feedback --- etc/kayobe/pulp-repo-versions.yml | 5 ++ etc/kayobe/pulp.yml | 129 +++++++++++++++++++++++++++--- 2 files changed, 125 insertions(+), 9 deletions(-) diff --git a/etc/kayobe/pulp-repo-versions.yml b/etc/kayobe/pulp-repo-versions.yml index 7f5398601..a4c8cf6f4 100644 --- a/etc/kayobe/pulp-repo-versions.yml +++ b/etc/kayobe/pulp-repo-versions.yml @@ -23,3 +23,8 @@ stackhpc_pulp_repo_treasuredata_4_version: 20220429T160649 stackhpc_pulp_repo_ubuntu_cloud_archive_version: 20220804T040153 stackhpc_pulp_repo_ubuntu_focal_security_version: 20220804T040153 stackhpc_pulp_repo_ubuntu_focal_version: 20220804T040153 +stackhpc_pulp_repo_rocky_8_6_appstream_version: "20220606T111205" +stackhpc_pulp_repo_rocky_8_6_baseos_version: "20220606T111205" +stackhpc_pulp_repo_rocky_8_6_extras_version: "20220606T111205" +stackhpc_pulp_repo_rocky_8_6_nfv_version: "20220606T111205" +stackhpc_pulp_repo_rocky_8_6_powertools_version: "20220606T111205" diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index 7efec2735..2ec7b9762 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -167,6 +167,21 @@ stackhpc_pulp_distribution_deb_production: # Whether to sync CentOS Stream 8 packages. stackhpc_pulp_sync_centos_stream8: "{{ os_distribution == 'centos' }}" +# Whether to sync Rocky Linux 8 packages. +stackhpc_pulp_sync_rocky_8: "{{ os_distribution == 'rocky' }}" +# Rocky 8 minor version number. Supported values: 6. +stackhpc_pulp_repo_rocky_8_minor_version: 6 +# Rocky 8 Snapshot versions. The defaults use the appropriate version from +# pulp-repo-versions.yml for the selected minor release. +stackhpc_pulp_repo_rocky_8_appstream_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_8_%s_appstream_version' % stackhpc_pulp_repo_rocky_8_minor_version) }}" +stackhpc_pulp_repo_rocky_8_baseos_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_8_%s_baseos_version' % stackhpc_pulp_repo_rocky_8_minor_version) }}" +stackhpc_pulp_repo_rocky_8_extras_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_8_%s_extras_version' % stackhpc_pulp_repo_rocky_8_minor_version) }}" +stackhpc_pulp_repo_rocky_8_nfv_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_8_%s_nfv_version' % stackhpc_pulp_repo_rocky_8_minor_version) }}" +stackhpc_pulp_repo_rocky_8_powertools_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_8_%s_powertools_version' % stackhpc_pulp_repo_rocky_8_minor_version) }}" + +# Whether to sync packages common to all RHEL derivatives. +stackhpc_pulp_sync_el_8: "{{ stackhpc_pulp_sync_rocky_8 or stackhpc_pulp_sync_centos_stream8 }}" + stackhpc_pulp_repository_rpm_repos: # Base CentOS 8 Stream repositories - name: CentOS Stream 8 - AppStream @@ -194,6 +209,48 @@ stackhpc_pulp_repository_rpm_repos: state: present required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}" + # Base Rocky 8 repositories + - name: Rocky Linux 8 - AppStream + url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/AppStream/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_appstream_version }}" + client_cert: "{{ stackhpc_release_pulp_client_cert }}" + client_key: "{{ stackhpc_release_pulp_client_key }}" + policy: on_demand + sync_policy: mirror_complete + state: present + required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" + - name: Rocky Linux 8 - BaseOS + url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/BaseOS/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_baseos_version }}" + client_cert: "{{ stackhpc_release_pulp_client_cert }}" + client_key: "{{ stackhpc_release_pulp_client_key }}" + policy: on_demand + sync_policy: mirror_complete + state: present + required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" + - name: Rocky Linux 8 - Extras + url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/extras/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_extras_version }}" + client_cert: "{{ stackhpc_release_pulp_client_cert }}" + client_key: "{{ stackhpc_release_pulp_client_key }}" + policy: on_demand + sync_policy: mirror_complete + state: present + required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" + - name: Rocky Linux 8 - NFV + url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/nfv/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_nfv_version }}" + client_cert: "{{ stackhpc_release_pulp_client_cert }}" + client_key: "{{ stackhpc_release_pulp_client_key }}" + policy: on_demand + sync_policy: mirror_complete + state: present + required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" + - name: Rocky Linux 8 - PowerTools + url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/PowerTools/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_6_powertools_version }}" + client_cert: "{{ stackhpc_release_pulp_client_cert }}" + client_key: "{{ stackhpc_release_pulp_client_key }}" + policy: on_demand + sync_policy: mirror_complete + state: present + required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" + # EPEL repositories - name: Extra Packages for Enterprise Linux 8 - x86_64 url: "{{ stackhpc_release_pulp_content_url }}/epel/8/Everything/x86_64/{{ stackhpc_pulp_repo_epel_version }}" @@ -202,7 +259,7 @@ stackhpc_pulp_repository_rpm_repos: policy: on_demand sync_policy: mirror_content_only state: present - required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}" + required: "{{ stackhpc_pulp_sync_el_8 | bool }}" - name: Extra Packages for Enterprise Linux Modular 8 - x86_64 url: "{{ stackhpc_release_pulp_content_url }}/epel/8/Modular/x86_64/{{ stackhpc_pulp_repo_epel_modular_version }}" client_cert: "{{ stackhpc_release_pulp_client_cert }}" @@ -210,7 +267,7 @@ stackhpc_pulp_repository_rpm_repos: policy: on_demand sync_policy: mirror_complete state: present - required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}" + required: "{{ stackhpc_pulp_sync_el_8 | bool }}" # Third-party repositories - name: Docker CE for CentOS 8 @@ -220,7 +277,7 @@ stackhpc_pulp_repository_rpm_repos: policy: on_demand sync_policy: mirror_complete state: present - required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}" + required: "{{ stackhpc_pulp_sync_el_8 }}" # Publication format is a subset of distribution. stackhpc_pulp_publication_rpm_development: "{{ stackhpc_pulp_distribution_rpm_development }}" @@ -247,24 +304,51 @@ stackhpc_pulp_distribution_rpm_development: state: present required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}" + # Base Rocky 8 repositories + - name: "rocky-8-appstream-development" + repository: Rocky Linux 8 - AppStream + base_path: "rocky/8/AppStream/x86_64/os/development" + state: present + required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" + - name: "rocky-8-baseos-development" + repository: Rocky Linux 8 - BaseOS + base_path: "rocky/8/BaseOS/x86_64/os/development" + state: present + required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" + - name: "rocky-8-extras-development" + repository: Rocky Linux 8 - Extras + base_path: "rocky/8/extras/x86_64/os/development" + state: present + required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" + - name: "rocky-8-nfv-development" + repository: Rocky Linux 8 - NFV + base_path: "rocky/8/nfv/x86_64/os/development" + state: present + required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" + - name: "rocky-powertools-development" + repository: Rocky Linux 8 - PowerTools + base_path: "rocky/8/PowerTools/x86_64/os/development" + state: present + required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" + # EPEL repositories - name: "extra-packages-for-enterprise-linux-8-x86_64-development" repository: Extra Packages for Enterprise Linux 8 - x86_64 base_path: "epel/8/Everything/x86_64/development" state: present - required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}" + required: "{{ stackhpc_pulp_sync_el_8 | bool }}" - name: "extra-packages-for-enterprise-linux-modular-8-x86_64-development" repository: Extra Packages for Enterprise Linux Modular 8 - x86_64 base_path: "epel/8/Modular/x86_64/development" state: present - required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}" + required: "{{ stackhpc_pulp_sync_el_8 | bool }}" # Third-party repositories - name: "docker-ce-for-centos-8-development" repository: Docker CE for CentOS 8 base_path: "docker-ce/centos/8/x86_64/stable/development" state: present - required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}" + required: "{{ stackhpc_pulp_sync_el_8 | bool }}" # Development gets promoted to production. stackhpc_pulp_distribution_rpm_production: @@ -285,24 +369,51 @@ stackhpc_pulp_distribution_rpm_production: state: present required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}" + # Base Rocky 8 repositories + - name: "rocky-8-appstream-production" + distribution: "rocky-8-appstream-development" + base_path: "rocky/8/AppStream/x86_64/os/production" + state: present + required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" + - name: "rocky-8-baseos-production" + distribution: "rocky-8-baseos-development" + base_path: "rocky/8/BaseOS/x86_64/os/production" + state: present + required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" + - name: "rocky-8-extras-production" + distribution: "rocky-8-extras-development" + base_path: "rocky/8/extras/x86_64/os/production" + state: present + required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" + - name: "rocky-8-nfv-production" + distribution: "rocky-8-nfv-development" + base_path: "rocky/8/nfv/x86_64/os/production" + state: present + required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" + - name: "rocky-8-powertools-production" + distribution: "rocky-8-powertools-development" + base_path: "rocky/8/PowerTools/x86_64/os/production" + state: present + required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" + # EPEL repositories - name: "extra-packages-for-enterprise-linux-8-x86_64-production" base_path: "epel/8/Everything/x86_64/production" distribution: "extra-packages-for-enterprise-linux-8-x86_64-development" state: present - required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}" + required: "{{ stackhpc_pulp_sync_el_8 | bool }}" - name: "extra-packages-for-enterprise-linux-modular-8-x86_64-production" base_path: "epel/8/Modular/x86_64/production" distribution: "extra-packages-for-enterprise-linux-modular-8-x86_64-development" state: present - required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}" + required: "{{ stackhpc_pulp_sync_el_8 | bool }}" # Third-party repositories - name: "docker-ce-for-centos-8-production" base_path: "docker-ce/centos/8/x86_64/stable/production" distribution: "docker-ce-for-centos-8-development" state: present - required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}" + required: "{{ stackhpc_pulp_sync_el_8 | bool }}" ############################################################################### # Containers From 422be9f0d581d9caa0d161b6c54c90147feb7e4f Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Thu, 1 Sep 2022 10:56:15 +0100 Subject: [PATCH 050/292] Update docker baseurl to work with Rocky 8 (#153) This is the equivalent change of: https://review.opendev.org/c/openstack/ansible-collection-kolla/+/831642 --- etc/kayobe/kolla/globals.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 3c0906229..79f05f90c 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -1,7 +1,7 @@ # yamllint disable-file --- docker_yum_baseurl: "{{ stackhpc_repo_docker_url }}" -docker_yum_gpgkey: "https://download.docker.com/linux/{% raw %}{{ ansible_facts.distribution | lower }}{% endraw %}/gpg" +docker_yum_gpgkey: "https://download.docker.com/linux/centos/gpg" {% if kolla_base_distro == 'centos' %} barbican_tag: xena-20220728T143658 From 86c9655db27fbafbd257797906897e46d483eae8 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Wed, 18 May 2022 10:13:39 +0200 Subject: [PATCH 051/292] [CI] Move queue setting to project level Per [1]. [1] http://lists.zuul-ci.org/pipermail/zuul-discuss/2022-May/001801.html Change-Id: I034d7dc4ea40654221d8a748f0fbe836a2e729e5 (cherry picked from commit b6e75b5cd6011886f87ba81c29996b1adb1f2fd9) --- zuul.d/project.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 4cee99909..2551dd112 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -1,10 +1,10 @@ --- - project: + queue: kayobe check: jobs: - openstack-tox-pep8 gate: - queue: kayobe jobs: - openstack-tox-pep8 From 4d574f9bda303005a673208eca9b2b876a540de0 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Tue, 20 Sep 2022 15:54:36 +0100 Subject: [PATCH 052/292] Bump rocky snapshot Reportedly this fixed a race condition with legacy network scripts. --- etc/kayobe/pulp-repo-versions.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/etc/kayobe/pulp-repo-versions.yml b/etc/kayobe/pulp-repo-versions.yml index a4c8cf6f4..75c59e28c 100644 --- a/etc/kayobe/pulp-repo-versions.yml +++ b/etc/kayobe/pulp-repo-versions.yml @@ -23,8 +23,8 @@ stackhpc_pulp_repo_treasuredata_4_version: 20220429T160649 stackhpc_pulp_repo_ubuntu_cloud_archive_version: 20220804T040153 stackhpc_pulp_repo_ubuntu_focal_security_version: 20220804T040153 stackhpc_pulp_repo_ubuntu_focal_version: 20220804T040153 -stackhpc_pulp_repo_rocky_8_6_appstream_version: "20220606T111205" -stackhpc_pulp_repo_rocky_8_6_baseos_version: "20220606T111205" -stackhpc_pulp_repo_rocky_8_6_extras_version: "20220606T111205" -stackhpc_pulp_repo_rocky_8_6_nfv_version: "20220606T111205" -stackhpc_pulp_repo_rocky_8_6_powertools_version: "20220606T111205" +stackhpc_pulp_repo_rocky_8_6_appstream_version: 20220918T035853 +stackhpc_pulp_repo_rocky_8_6_baseos_version: 20220918T035853 +stackhpc_pulp_repo_rocky_8_6_extras_version: 20220918T035853 +stackhpc_pulp_repo_rocky_8_6_nfv_version: 20220918T035853 +stackhpc_pulp_repo_rocky_8_6_powertools_version: 20220918T035853 From 29ac53876e2e8684a05e1ed8051640c873ad3511 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Tue, 20 Sep 2022 16:22:23 +0100 Subject: [PATCH 053/292] Adds OFED 5.7.x --- etc/kayobe/pulp-repo-versions.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/kayobe/pulp-repo-versions.yml b/etc/kayobe/pulp-repo-versions.yml index 75c59e28c..b3416bb66 100644 --- a/etc/kayobe/pulp-repo-versions.yml +++ b/etc/kayobe/pulp-repo-versions.yml @@ -28,3 +28,4 @@ stackhpc_pulp_repo_rocky_8_6_baseos_version: 20220918T035853 stackhpc_pulp_repo_rocky_8_6_extras_version: 20220918T035853 stackhpc_pulp_repo_rocky_8_6_nfv_version: 20220918T035853 stackhpc_pulp_repo_rocky_8_6_powertools_version: 20220918T035853 +stackhpc_pulp_repo_mlnx_ofed_5_7_1_0_2_0_rhel8_6_version: 20220920T151419 From 762d80e5c010805d1c61fa24aa935d5855d84689 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Tue, 20 Sep 2022 16:46:30 +0100 Subject: [PATCH 054/292] Use output from release train See: https://github.com/stackhpc/stackhpc-release-train/actions/runs/3091443294 --- etc/kayobe/pulp-repo-versions.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/pulp-repo-versions.yml b/etc/kayobe/pulp-repo-versions.yml index b3416bb66..e4abd8868 100644 --- a/etc/kayobe/pulp-repo-versions.yml +++ b/etc/kayobe/pulp-repo-versions.yml @@ -24,8 +24,8 @@ stackhpc_pulp_repo_ubuntu_cloud_archive_version: 20220804T040153 stackhpc_pulp_repo_ubuntu_focal_security_version: 20220804T040153 stackhpc_pulp_repo_ubuntu_focal_version: 20220804T040153 stackhpc_pulp_repo_rocky_8_6_appstream_version: 20220918T035853 -stackhpc_pulp_repo_rocky_8_6_baseos_version: 20220918T035853 -stackhpc_pulp_repo_rocky_8_6_extras_version: 20220918T035853 +stackhpc_pulp_repo_rocky_8_6_baseos_version: 20220914T080246 +stackhpc_pulp_repo_rocky_8_6_extras_version: 20220904T041706 stackhpc_pulp_repo_rocky_8_6_nfv_version: 20220918T035853 stackhpc_pulp_repo_rocky_8_6_powertools_version: 20220918T035853 stackhpc_pulp_repo_mlnx_ofed_5_7_1_0_2_0_rhel8_6_version: 20220920T151419 From 11bf9fcb21d4e937fa15f96929a08de0f2eef896 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 20 Sep 2022 17:10:54 +0100 Subject: [PATCH 055/292] cephadm: Bump collection to 1.8.0 This fixes several issues. https://github.com/stackhpc/ansible-collection-cephadm/releases/tag/1.8.0 --- etc/kayobe/ansible/requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index 52b18f408..a00fa1efc 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -1,7 +1,7 @@ --- collections: - name: stackhpc.cephadm - version: 1.7.0 + version: 1.8.0 - name: stackhpc.pulp version: 0.3.0 - name: pulp.squeezer From e90ade1e0778e86b0fa63c7cacbe2449272eb8a8 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 20 Sep 2022 17:11:14 +0100 Subject: [PATCH 056/292] cephadm: use admin overcloud network for SSH access --- etc/kayobe/cephadm.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/etc/kayobe/cephadm.yml b/etc/kayobe/cephadm.yml index eff49c565..b82111cb7 100644 --- a/etc/kayobe/cephadm.yml +++ b/etc/kayobe/cephadm.yml @@ -23,6 +23,9 @@ cephadm_registry_username: "{{ stackhpc_docker_registry_username }}" # Ceph container image registry password. cephadm_registry_password: "{{ stackhpc_docker_registry_password }}" +# Cephadm admin network interface. +cephadm_admin_interface: "{{ admin_oc_net_name | net_interface }}" + # Ceph public network interface. cephadm_public_interface: "{{ storage_net_name | net_interface }}" From 33172bebec2ffdc2c2702aa46bda50a0c7b5eb1a Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 20 Sep 2022 17:12:03 +0100 Subject: [PATCH 057/292] cephadm: sync only a specific container image tag Syncing all tags in quay.io/ceph/ceph can take a long time, and may even fail before getting to the tag we're interested in. --- etc/kayobe/pulp.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index 2ec7b9762..5805dc92a 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -597,6 +597,7 @@ stackhpc_pulp_repository_container_repos_ceph: url: "https://quay.io" policy: on_demand state: present + include_tags: "{{ cephadm_image_tag }}" required: "{{ stackhpc_sync_ceph_images | bool }}" # List of Ceph container image distributions. From 8abcbc9b613553b3880ca9d24f7e3ea8ea910ad6 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 21 Sep 2022 10:39:54 +0100 Subject: [PATCH 058/292] Cephadm: Remove leading tabs from generated ceph.conf --- etc/kayobe/ansible/cephadm-gather-keys.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/cephadm-gather-keys.yml b/etc/kayobe/ansible/cephadm-gather-keys.yml index d73f15a9e..4f0330deb 100644 --- a/etc/kayobe/ansible/cephadm-gather-keys.yml +++ b/etc/kayobe/ansible/cephadm-gather-keys.yml @@ -64,7 +64,8 @@ - name: Save ceph.conf to Kayobe configuration copy: - content: "{{ cephadm_ceph_conf.stdout }}" + # Kolla Ansible's merge_configs module does not like the leading tabs in ceph.conf. + content: "{{ cephadm_ceph_conf.stdout | regex_replace('\\t') }}" dest: "{{ kayobe_env_config_path }}/kolla/config/{{ kolla_service_to_conf_dir[item.0.name] }}/ceph.conf" loop: "{{ query('subelements', kolla_ceph_services | selectattr('required'), 'keys') }}" loop_control: From 65ca43407b776290df294a5aba1226d07285de3d Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 21 Sep 2022 12:29:26 +0100 Subject: [PATCH 059/292] Cephadm: Fix keyring generation Previously only the key was included, but we need the full file including caps. --- etc/kayobe/ansible/cephadm-gather-keys.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/etc/kayobe/ansible/cephadm-gather-keys.yml b/etc/kayobe/ansible/cephadm-gather-keys.yml index 4f0330deb..bfaeaaf32 100644 --- a/etc/kayobe/ansible/cephadm-gather-keys.yml +++ b/etc/kayobe/ansible/cephadm-gather-keys.yml @@ -25,6 +25,7 @@ - name: Get Ceph keys stackhpc.cephadm.cephadm_key: name: "{{ item }}" + output_format: plain state: info register: cephadm_key_info become: true @@ -49,10 +50,12 @@ - name: Save Ceph keys to Kayobe configuration vars: key_info: "{{ cephadm_key_info.results | selectattr('item', 'equalto', item.1) | first }}" - cephadm_key: "{{ (key_info.stdout | from_json | first)['key'] }}" - cephadm_user: "{{ (key_info.stdout | from_json | first)['entity'] }}" + cephadm_key: "{{ key_info.stdout }}" + cephadm_user: "{{ item.1 }}" copy: - content: "{{ cephadm_key }}" + # Include a trailing newline. + content: | + {{ cephadm_key }} dest: "{{ kayobe_env_config_path }}/kolla/config/{{ kolla_service_to_key_dir[item.0.name] }}/ceph.{{ cephadm_user }}.keyring" loop: "{{ query('subelements', kolla_ceph_services | selectattr('required'), 'keys') }}" loop_control: From 4b695f53326a555bf9c06dfba7ead9fdacb28734 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 21 Sep 2022 15:09:08 +0100 Subject: [PATCH 060/292] Ceph: add a trailing newline to ceph.conf --- etc/kayobe/ansible/cephadm-gather-keys.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/cephadm-gather-keys.yml b/etc/kayobe/ansible/cephadm-gather-keys.yml index bfaeaaf32..0a47de8d0 100644 --- a/etc/kayobe/ansible/cephadm-gather-keys.yml +++ b/etc/kayobe/ansible/cephadm-gather-keys.yml @@ -67,8 +67,10 @@ - name: Save ceph.conf to Kayobe configuration copy: + # Include a trailing newline. # Kolla Ansible's merge_configs module does not like the leading tabs in ceph.conf. - content: "{{ cephadm_ceph_conf.stdout | regex_replace('\\t') }}" + content: | + {{ cephadm_ceph_conf.stdout | regex_replace('\t') }} dest: "{{ kayobe_env_config_path }}/kolla/config/{{ kolla_service_to_conf_dir[item.0.name] }}/ceph.conf" loop: "{{ query('subelements', kolla_ceph_services | selectattr('required'), 'keys') }}" loop_control: From 8ec7fc69117b8e0f39ebea2c37efc5a424bbc905 Mon Sep 17 00:00:00 2001 From: Piotr Parczewski Date: Thu, 29 Sep 2022 22:23:42 +0200 Subject: [PATCH 061/292] Prevent 'base' type images from syncing / publishing --- etc/kayobe/pulp.yml | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index 5805dc92a..57b61970e 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -424,24 +424,18 @@ stackhpc_release_pulp_registry_url: "{{ stackhpc_release_pulp_url }}" # List of all image names. stackhpc_pulp_images_kolla: - barbican-api - - barbican-base - barbican-keystone-listener - barbican-worker - - base - - bifrost-base - bifrost-deploy - cinder-api - cinder-backup - - cinder-base - cinder-scheduler - cinder-volume - cloudkitty-api - - cloudkitty-base - cloudkitty-processor - cron - designate-api - designate-backend-bind9 - - designate-base - designate-central - designate-mdns - designate-producer @@ -452,17 +446,14 @@ stackhpc_pulp_images_kolla: - elasticsearch-curator - fluentd - glance-api - - glance-base - grafana - haproxy - heat-api - heat-api-cfn - - heat-base - heat-engine - horizon - influxdb - ironic-api - - ironic-base - ironic-conductor - ironic-inspector - ironic-neutron-agent @@ -470,24 +461,19 @@ stackhpc_pulp_images_kolla: - iscsid - keepalived - keystone - - keystone-base - keystone-fernet - keystone-ssh - kibana - kolla-toolbox - magnum-api - - magnum-base - magnum-conductor - manila-api - - manila-base - manila-data - manila-scheduler - manila-share - - mariadb-base - mariadb-clustercheck - mariadb-server - memcached - - neutron-base - neutron-dhcp-agent - neutron-l3-agent - neutron-metadata-agent @@ -495,7 +481,6 @@ stackhpc_pulp_images_kolla: - neutron-server - neutron-sriov-agent - nova-api - - nova-base - nova-compute - nova-compute-ironic - nova-conductor @@ -505,25 +490,19 @@ stackhpc_pulp_images_kolla: - nova-serialproxy - nova-ssh - octavia-api - - octavia-base - octavia-driver-agent - octavia-health-manager - octavia-housekeeping - octavia-worker - - openstack-base - - openvswitch-base - openvswitch-db-server - openvswitch-netcontrold - openvswitch-vswitchd - - ovn-base - ovn-controller - ovn-nb-db-server - ovn-northd - ovn-sb-db-server - placement-api - - placement-base - prometheus-alertmanager - - prometheus-base - prometheus-blackbox-exporter - prometheus-cadvisor - prometheus-elasticsearch-exporter @@ -539,11 +518,9 @@ stackhpc_pulp_images_kolla: - prometheus-v2-server - rabbitmq - redis - - redis-base - redis-sentinel - skydive-agent - skydive-analyzer - - skydive-base # Whitespace-separated list of regular expressions matching Kolla image names. # Usage is similar to kolla-build CLI arguments. From a25dbaa1aeee10f82320090e9bd533a3918946b4 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Wed, 21 Sep 2022 09:31:07 +0200 Subject: [PATCH 062/292] Prevent failures when dhclient is not running --- terraform/aio/scripts/configure-local-networking.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/aio/scripts/configure-local-networking.sh b/terraform/aio/scripts/configure-local-networking.sh index 0b4869bc3..c7bd70bf8 100644 --- a/terraform/aio/scripts/configure-local-networking.sh +++ b/terraform/aio/scripts/configure-local-networking.sh @@ -55,4 +55,4 @@ if ! sudo ip a show dev breth1 | grep $public_ip/24 >/dev/null 2>&1; then fi # This prevents network.service from restarting correctly. -sudo killall dhclient +sudo killall dhclient || true From 21f170727e2786f8b30c02fdfdb26f5611b07ad4 Mon Sep 17 00:00:00 2001 From: Piotr Parczewski Date: Thu, 29 Sep 2022 15:28:10 +0200 Subject: [PATCH 063/292] Let Ansible Galaxy handle dependencies squeezer is already listed as stackhpc.pulp collection's dependency --- etc/kayobe/ansible/requirements.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index a00fa1efc..fe8432ac7 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -4,5 +4,3 @@ collections: version: 1.8.0 - name: stackhpc.pulp version: 0.3.0 - - name: pulp.squeezer - version: 0.0.11 From 1f687fc9aff2644eb15e9da1ee7e9b2a4351e54d Mon Sep 17 00:00:00 2001 From: Piotr Parczewski Date: Thu, 29 Sep 2022 15:28:47 +0200 Subject: [PATCH 064/292] Bump up the Cephadm collection version --- etc/kayobe/ansible/requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index fe8432ac7..99a0c83f7 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -1,6 +1,6 @@ --- collections: - name: stackhpc.cephadm - version: 1.8.0 + version: 1.10.0 - name: stackhpc.pulp version: 0.3.0 From 0b9630f9e5160e19e4b04a9a3cc5226556a0c679 Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Wed, 28 Sep 2022 14:37:28 +0200 Subject: [PATCH 065/292] Bump repos to the same versions as Wallaby --- etc/kayobe/pulp-repo-versions.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/etc/kayobe/pulp-repo-versions.yml b/etc/kayobe/pulp-repo-versions.yml index a4c8cf6f4..df5ff8f1a 100644 --- a/etc/kayobe/pulp-repo-versions.yml +++ b/etc/kayobe/pulp-repo-versions.yml @@ -1,25 +1,25 @@ --- # Do not edit! This file is autogenerated by Ansible. stackhpc_pulp_repo_centos_stream_8_advanced_virtualization_version: 20211122T102435 -stackhpc_pulp_repo_centos_stream_8_appstream_version: 20220607T061247 -stackhpc_pulp_repo_centos_stream_8_baseos_version: 20220607T061247 +stackhpc_pulp_repo_centos_stream_8_appstream_version: 20220817T082321 +stackhpc_pulp_repo_centos_stream_8_baseos_version: 20220817T082321 stackhpc_pulp_repo_centos_stream_8_extras_version: 20220401T032901 stackhpc_pulp_repo_centos_stream_8_nfv_extras_version: 20220609T110556 -stackhpc_pulp_repo_centos_stream_8_nfv_openvswitch_version: 20220616T202817 -stackhpc_pulp_repo_centos_stream_8_openstack_xena_version: 20220612T035123 +stackhpc_pulp_repo_centos_stream_8_nfv_openvswitch_version: 20220824T113700 +stackhpc_pulp_repo_centos_stream_8_openstack_xena_version: 20220823T042339 stackhpc_pulp_repo_centos_stream_8_opstools_version: 20220617T100837 -stackhpc_pulp_repo_centos_stream_8_powertools_version: 20220607T061247 -stackhpc_pulp_repo_centos_stream_8_storage_ceph_pacific_version: 20220525T030654 +stackhpc_pulp_repo_centos_stream_8_powertools_version: 20220817T082321 +stackhpc_pulp_repo_centos_stream_8_storage_ceph_pacific_version: 20220803T043053 stackhpc_pulp_repo_docker_ce_ubuntu_version: 20220708T132615 -stackhpc_pulp_repo_docker_version: 20220607T061247 -stackhpc_pulp_repo_elasticsearch_logstash_kibana_7_x_version: 20220525T030654 -stackhpc_pulp_repo_epel_modular_version: 20220604T032730 -stackhpc_pulp_repo_epel_version: 20220617T050552 -stackhpc_pulp_repo_grafana_version: 20220616T113902 -stackhpc_pulp_repo_mariadb_10_5_centos8_version: 20220609T110556 -stackhpc_pulp_repo_rabbitmq_erlang_version: 20220616T113902 -stackhpc_pulp_repo_rabbitmq_server_version: 20220602T033149 -stackhpc_pulp_repo_treasuredata_4_version: 20220429T160649 +stackhpc_pulp_repo_docker_version: 20220828T050510 +stackhpc_pulp_repo_elasticsearch_logstash_kibana_7_x_version: 20220825T041159 +stackhpc_pulp_repo_epel_modular_version: 20220828T050510 +stackhpc_pulp_repo_epel_version: 20220830T054227 +stackhpc_pulp_repo_grafana_version: 20220831T050502 +stackhpc_pulp_repo_mariadb_10_5_centos8_version: 20220817T082321 +stackhpc_pulp_repo_rabbitmq_erlang_version: 20220828T050510 +stackhpc_pulp_repo_rabbitmq_server_version: 20220803T043053 +stackhpc_pulp_repo_treasuredata_4_version: 20220823T042339 stackhpc_pulp_repo_ubuntu_cloud_archive_version: 20220804T040153 stackhpc_pulp_repo_ubuntu_focal_security_version: 20220804T040153 stackhpc_pulp_repo_ubuntu_focal_version: 20220804T040153 From ac99e01976f7633815ad0a35264328469ef86d25 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 5 Oct 2022 13:34:16 +0100 Subject: [PATCH 066/292] Remove duplicate opstools repo --- etc/kayobe/stackhpc.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml index e82d98bc8..e0d41155c 100644 --- a/etc/kayobe/stackhpc.yml +++ b/etc/kayobe/stackhpc.yml @@ -71,10 +71,6 @@ stackhpc_repo_centos_stream_openstack_xena_version: "{{ stackhpc_repo_distributi stackhpc_repo_centos_stream_opstools_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/centos/8-stream/opstools/x86_64/collectd-5/{{ stackhpc_repo_centos_stream_opstools_version }}" stackhpc_repo_centos_stream_opstools_version: "{{ stackhpc_repo_distribution }}" -# CentOS Stream 8 - OpsTools - collectd -stackhpc_repo_centos_stream_opstools_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/centos/8-stream/opstools/x86_64/collectd-5/{{ stackhpc_repo_centos_stream_opstools_version }}" -stackhpc_repo_centos_stream_opstools_version: "{{ stackhpc_repo_distribution }}" - # CentOS Stream 8 - PowerTools stackhpc_repo_centos_stream_powertools_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/centos/8-stream/PowerTools/x86_64/os/{{ stackhpc_repo_centos_stream_powertools_version }}" stackhpc_repo_centos_stream_powertools_version: "{{ stackhpc_repo_distribution }}" From e8aa86bb7d8ea3c976358ad714695f2da1fd5098 Mon Sep 17 00:00:00 2001 From: Piotr Parczewski Date: Thu, 29 Sep 2022 15:33:36 +0200 Subject: [PATCH 067/292] Xena: Add system logging custom playbook --- etc/kayobe/ansible/rsyslog.yml | 36 +++++++++++++++++++ .../overcloud-service-deploy/post.d/.gitkeep | 0 2 files changed, 36 insertions(+) create mode 100644 etc/kayobe/ansible/rsyslog.yml create mode 100644 etc/kayobe/hooks/overcloud-service-deploy/post.d/.gitkeep diff --git a/etc/kayobe/ansible/rsyslog.yml b/etc/kayobe/ansible/rsyslog.yml new file mode 100644 index 000000000..acc567eb7 --- /dev/null +++ b/etc/kayobe/ansible/rsyslog.yml @@ -0,0 +1,36 @@ +--- +# Custom playbook to configure rsyslog to forward operating system logs to +# fluentd, eventually reaching Kolla's Elasticsearch cluster. +# +# Variables: +# * rsyslog_group: Host pattern against which to target the playbook. Default +# is 'overcloud'. + +- name: Configure rsyslog to forward messages + hosts: "{{ rsyslog_group | default('overcloud') }}" + become: yes + + tasks: + - name: Ensure rsyslog is installed + package: + name: rsyslog + state: present + + - name: Ensure rsyslog is started and enabled + systemd: + state: started + enabled: yes + name: rsyslog + + - name: Update rsyslog configuration + lineinfile: + path: /etc/rsyslog.conf + insertafter: "^#*.* @@remote-host:514" + line: "*.* @{{ internal_net_name | net_ip }}:5140" + register: rsyslog_config + + - name: Restart rsyslog + systemd: + state: restarted + name: rsyslog + when: rsyslog_config.changed diff --git a/etc/kayobe/hooks/overcloud-service-deploy/post.d/.gitkeep b/etc/kayobe/hooks/overcloud-service-deploy/post.d/.gitkeep new file mode 100644 index 000000000..e69de29bb From 61b5070952bfe5eb61dcd6c4dde9ecbc507326a1 Mon Sep 17 00:00:00 2001 From: Piotr Parczewski Date: Mon, 3 Oct 2022 11:46:59 +0200 Subject: [PATCH 068/292] Enable Kolla features --- etc/kayobe/kolla.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index b4e8afe6b..cc60d1fe3 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -456,7 +456,7 @@ kolla_build_args: #kolla_enable_ceilometer_horizon_policy_file: #kolla_enable_ceilometer_ipmi: #kolla_enable_cells: -#kolla_enable_central_logging: +kolla_enable_central_logging: true #kolla_enable_ceph_rgw: #kolla_enable_ceph_rgw_loadbalancer: #kolla_enable_cinder: @@ -474,7 +474,7 @@ kolla_build_args: #kolla_enable_designate: #kolla_enable_destroy_images: #kolla_enable_elasticsearch: -#kolla_enable_elasticsearch_curator: +kolla_enable_elasticsearch_curator: true #kolla_enable_etcd: #kolla_enable_external_mariadb_load_balancer: #kolla_enable_fluentd: @@ -484,7 +484,7 @@ kolla_build_args: #kolla_enable_glance_image_cache: #kolla_enable_gnocchi: #kolla_enable_gnocchi_statsd: -#kolla_enable_grafana: +kolla_enable_grafana: true #kolla_enable_hacluster: #kolla_enable_haproxy: #kolla_enable_haproxy_memcached: @@ -524,7 +524,7 @@ kolla_build_args: #kolla_enable_keystone: #kolla_enable_keystone_federation: #kolla_enable_keystone_horizon_policy_file: -#kolla_enable_kibana: +kolla_enable_kibana: true #kolla_enable_kuryr: #kolla_enable_loadbalancer: #kolla_enable_magnum: @@ -573,7 +573,7 @@ kolla_build_args: #kolla_enable_ovn: #kolla_enable_ovs_dpdk: #kolla_enable_placement: -#kolla_enable_prometheus: +kolla_enable_prometheus: true #kolla_enable_prometheus_alertmanager: #kolla_enable_prometheus_blackbox_exporter: #kolla_enable_prometheus_cadvisor: From 6940e0e81da33cc949608fc2f96faceb812f65d6 Mon Sep 17 00:00:00 2001 From: Piotr Parczewski Date: Wed, 5 Oct 2022 08:56:45 +0200 Subject: [PATCH 069/292] Elasticsearch memory tuning --- etc/kayobe/environments/ci-aio/kolla/globals.yml | 3 +++ etc/kayobe/kolla/globals.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/etc/kayobe/environments/ci-aio/kolla/globals.yml b/etc/kayobe/environments/ci-aio/kolla/globals.yml index 8bbef4a78..133da0407 100644 --- a/etc/kayobe/environments/ci-aio/kolla/globals.yml +++ b/etc/kayobe/environments/ci-aio/kolla/globals.yml @@ -11,3 +11,6 @@ openstack_service_rpc_workers: "1" docker_yum_baseurl: "{{ stackhpc_repo_docker_url }}" docker_yum_gpgkey: "https://download.docker.com/linux/{% raw %}{{ ansible_facts.distribution | lower }}{% endraw %}/gpg" + +# Elasticsearch memory tuning +es_heap_size: 1g diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 79f05f90c..8adcb33a7 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -7,3 +7,6 @@ docker_yum_gpgkey: "https://download.docker.com/linux/centos/gpg" barbican_tag: xena-20220728T143658 prometheus_tag: xena-20220728T143658 {% endif %} + +# Elasticsearch memory tuning +es_heap_size: 8g From a8deca1e7293ae8086d3ae1e0e957bfc6392d84b Mon Sep 17 00:00:00 2001 From: Piotr Parczewski Date: Wed, 5 Oct 2022 11:25:31 +0200 Subject: [PATCH 070/292] Add standard alerting rules and dashboards Co-authored-by: Doug Szumski Co-authored-by: k-s-dean Co-authored-by: Pierre Riteau --- .../grafana/dashboards/ceph/ceph_mds.json | 316 + .../grafana/dashboards/ceph/ceph_osds.json | 954 ++ .../dashboards/ceph/ceph_overview.json | 2928 ++++ .../grafana/dashboards/ceph/ceph_pools.json | 752 + .../dashboards/openstack/blackbox.json | 1050 ++ .../dashboards/openstack/cadvisor.json | 1302 ++ .../dashboards/openstack/database.json | 1023 ++ .../dashboards/openstack/elasticsearch.json | 7206 ++++++++ .../grafana/dashboards/openstack/fluentd.json | 3207 ++++ .../grafana/dashboards/openstack/haproxy.json | 5377 ++++++ .../grafana/dashboards/openstack/libvirt.json | 1718 ++ .../dashboards/openstack/memcached.json | 901 + .../openstack/node_exporter_full.json | 13568 ++++++++++++++++ .../openstack/node_exporter_side_by_side.json | 5775 +++++++ .../dashboards/openstack/openstack.json | 2604 +++ .../dashboards/openstack/prometheus.json | 3432 ++++ .../openstack/prometheus_alertmanager.json | 11257 +++++++++++++ .../openstack/prometheus_benchmark.json | 5551 +++++++ .../dashboards/openstack/rabbitmq.json | 5287 ++++++ .../grafana/dashboards/openstack/redfish.json | 3769 +++++ .../grafana/grafana_home_dashboard.json | 91 + .../kolla/config/grafana/provisioning.yaml | 15 + .../kolla/config/prometheus/blackbox.rules | 77 + .../kolla/config/prometheus/cadvisor.rules | 54 + etc/kayobe/kolla/config/prometheus/ceph.rules | 222 + .../config/prometheus/elasticsearch.rules | 99 + .../kolla/config/prometheus/haproxy.rules | 36 + .../kolla/config/prometheus/mysql.rules | 81 + .../config/prometheus/notifications.tmpl | 32 + .../kolla/config/prometheus/openstack.rules | 15 + .../kolla/config/prometheus/prometheus.rules | 225 + .../kolla/config/prometheus/rabbitmq.rules | 71 + .../kolla/config/prometheus/system.rules | 79 + 33 files changed, 79074 insertions(+) create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_mds.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_osds.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_overview.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_pools.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/blackbox.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/cadvisor.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/database.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/elasticsearch.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/fluentd.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/haproxy.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/libvirt.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/memcached.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_full.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_side_by_side.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/openstack.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus_alertmanager.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus_benchmark.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/rabbitmq.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/redfish.json create mode 100644 etc/kayobe/kolla/config/grafana/grafana_home_dashboard.json create mode 100644 etc/kayobe/kolla/config/grafana/provisioning.yaml create mode 100644 etc/kayobe/kolla/config/prometheus/blackbox.rules create mode 100644 etc/kayobe/kolla/config/prometheus/cadvisor.rules create mode 100644 etc/kayobe/kolla/config/prometheus/ceph.rules create mode 100644 etc/kayobe/kolla/config/prometheus/elasticsearch.rules create mode 100644 etc/kayobe/kolla/config/prometheus/haproxy.rules create mode 100644 etc/kayobe/kolla/config/prometheus/mysql.rules create mode 100644 etc/kayobe/kolla/config/prometheus/notifications.tmpl create mode 100644 etc/kayobe/kolla/config/prometheus/openstack.rules create mode 100644 etc/kayobe/kolla/config/prometheus/prometheus.rules create mode 100644 etc/kayobe/kolla/config/prometheus/rabbitmq.rules create mode 100644 etc/kayobe/kolla/config/prometheus/system.rules diff --git a/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_mds.json b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_mds.json new file mode 100644 index 000000000..c3e6a0ec6 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_mds.json @@ -0,0 +1,316 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 179, + "iteration": 1616694078267, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 10, + "panels": [], + "title": "MDS Performance", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Reads/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(ceph_objecter_op_r{ceph_daemon=~\"($mds_servers).*\"}[10m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Read Ops", + "refId": "A" + }, + { + "expr": "sum(irate(ceph_objecter_op_w{ceph_daemon=~\"($mds_servers).*\"}[10m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Write Ops", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "MDS Workload - $mds_servers", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "Reads(-) / Writes (+)", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(ceph_mds_server_handle_client_request{ceph_daemon=~\"($mds_servers).*\"}[30m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ceph_daemon}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Client Request Load - $mds_servers", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "Client Requests", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 22, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "definition": "", + "hide": 0, + "includeAll": true, + "label": "MDS Server", + "multi": false, + "name": "mds_servers", + "options": [], + "query": "label_values(ceph_mds_inodes, ceph_daemon)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "15s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Ceph MDS", + "uid": "tbO9LAiZz", + "version": 5 +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_osds.json b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_osds.json new file mode 100644 index 000000000..64e0895bc --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_osds.json @@ -0,0 +1,954 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 191, + "iteration": 1616693984817, + "links": [], + "panels": [ + { + "aliasColors": { + "@95%ile": "#e0752d" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg (irate(ceph_osd_op_r_latency_sum[5m]) / on (ceph_daemon) irate(ceph_osd_op_r_latency_count[5m]) * 1000)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "AVG read", + "refId": "A" + }, + { + "expr": "max (irate(ceph_osd_op_r_latency_sum[5m]) / on (ceph_daemon) irate(ceph_osd_op_r_latency_count[5m]) * 1000)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "MAX read", + "refId": "B" + }, + { + "expr": "quantile(0.95,\n (irate(ceph_osd_op_r_latency_sum[5m]) / on (ceph_daemon) irate(ceph_osd_op_r_latency_count[5m]) * 1000)\n)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "@95%ile", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "OSD Read Latencies", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": "$datasource", + "description": "This table shows the osd's that are delivering the 10 highest read latencies within the cluster", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 15, + "links": [], + "options": {}, + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": null, + "desc": false + }, + "styles": [ + { + "alias": "OSD ID", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "ceph_daemon", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Latency (ms)", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value", + "thresholds": [], + "type": "number", + "unit": "none" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "hidden", + "unit": "short" + } + ], + "targets": [ + { + "expr": "topk(10,\n (sort(\n (irate(ceph_osd_op_r_latency_sum[5m]) / on (ceph_daemon) irate(ceph_osd_op_r_latency_count[5m]) * 1000)\n ))\n)\n\n", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Highest READ Latencies", + "transform": "table", + "type": "table" + }, + { + "aliasColors": { + "@95%ile write": "#e0752d" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg (rate(ceph_osd_op_w_latency_sum[10m]) / on (ceph_daemon) rate(ceph_osd_op_w_latency_count[10m]) * 1000)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "AVG write", + "refId": "A" + }, + { + "expr": "max (irate(ceph_osd_op_w_latency_sum[5m]) / on (ceph_daemon) irate(ceph_osd_op_w_latency_count[5m]) * 1000)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "MAX write", + "refId": "B" + }, + { + "expr": "quantile(0.95,\n (irate(ceph_osd_op_w_latency_sum[5m]) / on (ceph_daemon) irate(ceph_osd_op_w_latency_count[5m]) * 1000)\n)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "@95%ile write", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "OSD Write Latencies", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": "$datasource", + "description": "This table shows the osd's that are delivering the 10 highest write latencies within the cluster", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 16, + "links": [], + "options": {}, + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": null, + "desc": false + }, + "styles": [ + { + "alias": "OSD ID", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "ceph_daemon", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Latency (ms)", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value", + "thresholds": [], + "type": "number", + "unit": "none" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "hidden", + "unit": "short" + } + ], + "targets": [ + { + "expr": "topk(10,\n (sort(\n (irate(ceph_osd_op_w_latency_sum[5m]) / on (ceph_daemon) irate(ceph_osd_op_w_latency_count[5m]) * 1000)\n ))\n)\n\n", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Highest WRITE Latencies", + "transform": "table", + "type": "table" + }, + { + "cacheTimeout": null, + "columns": [], + "datasource": "$datasource", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 4, + "x": 0, + "y": 8 + }, + "id": 2, + "links": [], + "options": {}, + "pageSize": null, + "pluginVersion": "6.6.1", + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "align": "right", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count by (device_class) (ceph_osd_metadata)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{device_class}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "OSD Types Summary", + "transform": "table", + "type": "table" + }, + { + "cacheTimeout": null, + "columns": [], + "datasource": "$datasource", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 4, + "x": 4, + "y": 8 + }, + "hideTimeOverride": true, + "id": 4, + "links": [], + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "align": "right", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(ceph_bluefs_wal_total_bytes)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "bluestore", + "refId": "A", + "step": 240 + }, + { + "expr": "count(ceph_osd_metadata) - count(ceph_bluefs_wal_total_bytes)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "filestore", + "refId": "B", + "step": 240 + }, + { + "expr": "absent(ceph_bluefs_wal_total_bytes)*count(ceph_osd_metadata)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "filestore", + "refId": "C", + "step": 240 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "OSD Objectstore Types", + "transform": "timeseries_to_columns", + "type": "table" + }, + { + "cacheTimeout": null, + "columns": [], + "datasource": "$datasource", + "description": "The pie chart shows the various OSD sizes used within the cluster", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 4, + "x": 8, + "y": 8 + }, + "hideTimeOverride": true, + "id": 8, + "links": [], + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "align": "right", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(ceph_osd_stat_bytes < 1099511627776)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "<1 TB", + "refId": "A", + "step": 2 + }, + { + "expr": "count(ceph_osd_stat_bytes >= 1099511627776 < 2199023255552)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "<2 TB", + "refId": "B", + "step": 2 + }, + { + "expr": "count(ceph_osd_stat_bytes >= 2199023255552 < 3298534883328)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "<3TB", + "refId": "C", + "step": 2 + }, + { + "expr": "count(ceph_osd_stat_bytes >= 3298534883328 < 4398046511104)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "<4TB", + "refId": "D", + "step": 2 + }, + { + "expr": "count(ceph_osd_stat_bytes >= 4398046511104 < 6597069766656)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "<6TB", + "refId": "E", + "step": 2 + }, + { + "expr": "count(ceph_osd_stat_bytes >= 6597069766656 < 8796093022208)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "<8TB", + "refId": "F", + "step": 2 + }, + { + "expr": "count(ceph_osd_stat_bytes >= 8796093022208 < 10995116277760)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "<10TB", + "refId": "G", + "step": 2 + }, + { + "expr": "count(ceph_osd_stat_bytes >= 10995116277760 < 13194139533312)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "<12TB", + "refId": "H", + "step": 2 + }, + { + "expr": "count(ceph_osd_stat_bytes >= 13194139533312)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "12TB+", + "refId": "I", + "step": 2 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "OSD Size Summary", + "transform": "timeseries_to_columns", + "type": "table" + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Each bar indicates the number of OSD's that have a PG count in a specific range as shown on the x axis.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "ceph_osd_numpg\n", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "legendFormat": "PGs per OSD", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Distribution of PGs per OSD", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": 20, + "mode": "histogram", + "name": null, + "show": true, + "values": [ + "total" + ] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "# of OSDs", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 20, + "panels": [], + "title": "R/W Profile", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Show the read/write workload profile overtime", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 17 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(ceph_pool_rd[5m])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Reads", + "refId": "A" + }, + { + "expr": "round(sum(irate(ceph_pool_wr[5m])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Writes", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Read/Write Profile", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 22, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Ceph OSD Overview", + "uid": "lo02I1Aiz", + "version": 9 +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_overview.json b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_overview.json new file mode 100644 index 000000000..4693e139f --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_overview.json @@ -0,0 +1,2928 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Ceph Cluster overview.\r\n", + "editable": true, + "gnetId": 2842, + "graphTooltip": 0, + "id": 97, + "iteration": 1616693613290, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 37, + "panels": [], + "repeat": null, + "title": "CLUSTER STATE", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 21, + "interval": "1m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "ceph_health_status", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A", + "step": 300 + } + ], + "thresholds": "1,2", + "title": "Status", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + }, + { + "op": "=", + "text": "WARNING", + "value": "1" + }, + { + "op": "=", + "text": "HEALTHY", + "value": "0" + }, + { + "op": "=", + "text": "ERROR", + "value": "2" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 14, + "interval": "1m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(ceph_mon_quorum_status)", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 300 + } + ], + "thresholds": "2,3", + "title": "Monitors In Quorum", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 22, + "interval": "1m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(ceph_pool_bytes_used)", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 300 + } + ], + "thresholds": "", + "title": "Pools", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 33, + "interval": "1m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "ceph_cluster_total_bytes", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 300 + } + ], + "thresholds": "0.025,0.1", + "title": "Cluster Capacity", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 34, + "interval": "1m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "ceph_cluster_total_used_bytes", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 300 + } + ], + "thresholds": "0.025,0.1", + "title": "Used Capacity", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "decimals": 2, + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 23, + "interval": "1m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "((ceph_cluster_total_bytes-ceph_cluster_total_used_bytes)/ceph_cluster_total_bytes)*100", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 300 + } + ], + "thresholds": "10,30", + "title": "Available Capacity", + "type": "singlestat", + "valueFontSize": "70%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 38, + "panels": [], + "repeat": null, + "title": "OSD STATE", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 6 + }, + "id": 26, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(ceph_osd_in)", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 300 + } + ], + "thresholds": "", + "title": "OSDs IN", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 40, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 6 + }, + "id": 27, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(ceph_osd_up) - count(ceph_osd_in)", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 300 + } + ], + "thresholds": "1,1", + "title": "OSDs OUT", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 4, + "y": 6 + }, + "id": 28, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(ceph_osd_up)", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 300 + } + ], + "thresholds": "", + "title": "OSDs UP", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 40, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 6, + "y": 6 + }, + "id": 29, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(ceph_osd_up == 0) OR vector(0)", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 300 + } + ], + "thresholds": "1,1", + "title": "OSDs DOWN", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 6 + }, + "id": 30, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(ceph_osd_numpg)", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 300 + } + ], + "thresholds": "250,300", + "title": "Agerage PGs per OSD", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "editable": true, + "error": false, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 6 + }, + "id": 31, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(ceph_osd_apply_latency_ms)", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 300 + } + ], + "thresholds": "10,50", + "title": "Average OSD Apply Latency", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "editable": true, + "error": false, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 6 + }, + "id": 32, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(ceph_osd_commit_latency_ms)", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 300 + } + ], + "thresholds": "10,50", + "title": "Average OSD Commit Latency", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "editable": true, + "error": false, + "format": "s", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 6 + }, + "id": 24, + "interval": "1m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "repeat": null, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(ceph_monitor_latency_seconds)", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 300 + } + ], + "thresholds": "70,80", + "title": "Average Monitor Latency", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 39, + "panels": [], + "repeat": null, + "title": "CLUSTER", + "type": "row" + }, + { + "aliasColors": { + "Available": "#EAB839", + "Total Capacity": "#447EBC", + "Used": "#BF1B00", + "total_avail": "#6ED0E0", + "total_space": "#7EB26D", + "total_used": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 4, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 10 + }, + "height": "300", + "hiddenSeries": false, + "id": 1, + "interval": "$interval", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Total Capacity", + "fill": 0, + "linewidth": 3, + "stack": false + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceph_cluster_total_bytes-ceph_cluster_total_used_bytes", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Available", + "refId": "A", + "step": 300 + }, + { + "expr": "ceph_cluster_total_used_bytes", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Used", + "refId": "B", + "step": 300 + }, + { + "expr": "ceph_cluster_total_bytes", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Total Capacity", + "refId": "C", + "step": 300 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Capacity", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Total Capacity": "#7EB26D", + "Used": "#BF1B00", + "total_avail": "#6ED0E0", + "total_space": "#7EB26D", + "total_used": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 0, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 10 + }, + "height": "300", + "hiddenSeries": false, + "id": 3, + "interval": "$interval", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(ceph_osd_op_w[$interval]))", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Write", + "refId": "A", + "step": 300 + }, + { + "expr": "sum(rate(ceph_osd_op_r[$interval]))", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Read", + "refId": "B", + "step": 300 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "IOPS", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 10 + }, + "height": "300", + "hiddenSeries": false, + "id": 7, + "interval": "$interval", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(ceph_osd_op_w_in_bytes[$interval]))", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Write", + "refId": "A", + "step": 300 + }, + { + "expr": "sum(irate(ceph_osd_op_r_out_bytes[$interval]))", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Read", + "refId": "B", + "step": 300 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Throughput", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 40, + "panels": [], + "repeat": null, + "title": "LATENCY", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 16, + "x": 0, + "y": 19 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(ceph_osd_apply_latency_ms)", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "apply", + "metric": "ceph_osd_perf_apply_latency_seconds", + "refId": "A", + "step": 4 + }, + { + "expr": "avg(ceph_osd_commit_latency_ms)", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "commit", + "metric": "ceph_osd_perf_commit_latency_seconds", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "OSD Apply + Commit Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": null, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 19 + }, + "hiddenSeries": false, + "id": 36, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(ceph_monitor_latency_seconds)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "latency", + "metric": "ceph_monitor_latency_seconds", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Monitor Latency (Currently not available with Ceph MGR)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 41, + "panels": [], + "repeat": null, + "title": "OBJECTS", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 27 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^Total.*$/", + "stack": false + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceph_cluster_total_objects", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Total", + "refId": "A", + "step": 300 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Objects in the Cluster", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 31 + }, + "hiddenSeries": false, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^Total.*$/", + "stack": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "ceph_pg_active", + "format": "time_series", + "hide": false, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Active", + "refId": "M" + }, + { + "expr": "ceph_pg_clean", + "format": "time_series", + "hide": false, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Clean", + "refId": "U" + }, + { + "expr": "ceph_pg_peering", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Peering", + "refId": "I" + }, + { + "expr": "ceph_pg_degraded", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Degraded", + "refId": "B", + "step": 300 + }, + { + "expr": "ceph_pg_stale", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Stale", + "refId": "C", + "step": 300 + }, + { + "expr": "ceph_unclean_pgs", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Unclean", + "refId": "D", + "step": 300 + }, + { + "expr": "ceph_pg_undersized", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Undersized", + "refId": "E", + "step": 300 + }, + { + "expr": "ceph_pg_incomplete", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Incomplete", + "refId": "G" + }, + { + "expr": "ceph_pg_forced_backfill", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Forced Backfill", + "refId": "H" + }, + { + "expr": "ceph_pg_inconsistent", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Inconsistent", + "refId": "F" + }, + { + "expr": "ceph_pg_forced_recovery", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Forced Recovery", + "refId": "J" + }, + { + "expr": "ceph_pg_creating", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Creating", + "refId": "K" + }, + { + "expr": "ceph_pg_wait_backfill", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Wait Backfill", + "refId": "L" + }, + { + "expr": "ceph_pg_deep", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Deep", + "refId": "N" + }, + { + "expr": "ceph_pg_scrubbing", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Scrubbing", + "refId": "O" + }, + { + "expr": "ceph_pg_recovering", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Recovering", + "refId": "P" + }, + { + "expr": "ceph_pg_repair", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Repair", + "refId": "Q" + }, + { + "expr": "ceph_pg_down", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Down", + "refId": "R" + }, + { + "expr": "ceph_pg_peered", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Peered", + "refId": "S" + }, + { + "expr": "ceph_pg_backfill", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Backfill", + "refId": "T" + }, + { + "expr": "ceph_pg_remapped", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Remapped", + "refId": "V" + }, + { + "expr": "ceph_pg_backfill_toofull", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Backfill Toofull", + "refId": "W" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "PGs", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 41 + }, + "hiddenSeries": false, + "id": 43, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^Total.*$/", + "stack": false + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceph_pg_active", + "format": "time_series", + "hide": true, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Active", + "refId": "M" + }, + { + "expr": "ceph_pg_clean", + "format": "time_series", + "hide": true, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Clean", + "refId": "U" + }, + { + "expr": "ceph_pg_peering", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Peering", + "refId": "I" + }, + { + "expr": "ceph_pg_degraded", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Degraded", + "refId": "B", + "step": 300 + }, + { + "expr": "ceph_pg_stale", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Stale", + "refId": "C", + "step": 300 + }, + { + "expr": "ceph_unclean_pgs", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Unclean", + "refId": "D", + "step": 300 + }, + { + "expr": "ceph_pg_undersized", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Undersized", + "refId": "E", + "step": 300 + }, + { + "expr": "ceph_pg_incomplete", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Incomplete", + "refId": "G" + }, + { + "expr": "ceph_pg_forced_backfill", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Forced Backfill", + "refId": "H" + }, + { + "expr": "ceph_pg_inconsistent", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Inconsistent", + "refId": "F" + }, + { + "expr": "ceph_pg_forced_recovery", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Forced Recovery", + "refId": "J" + }, + { + "expr": "ceph_pg_creating", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Creating", + "refId": "K" + }, + { + "expr": "ceph_pg_wait_backfill", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Wait Backfill", + "refId": "L" + }, + { + "expr": "ceph_pg_deep", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Deep", + "refId": "N" + }, + { + "expr": "ceph_pg_scrubbing", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Scrubbing", + "refId": "O" + }, + { + "expr": "ceph_pg_recovering", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Recovering", + "refId": "P" + }, + { + "expr": "ceph_pg_repair", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Repair", + "refId": "Q" + }, + { + "expr": "ceph_pg_down", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Down", + "refId": "R" + }, + { + "expr": "ceph_pg_peered", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Peered", + "refId": "S" + }, + { + "expr": "ceph_pg_backfill", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Backfill", + "refId": "T" + }, + { + "expr": "ceph_pg_remapped", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Remapped", + "refId": "V" + }, + { + "expr": "ceph_pg_backfill_toofull", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Backfill Toofull", + "refId": "W" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Bad PGs", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^Total.*$/", + "stack": false + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceph_pg_degraded", + "format": "time_series", + "hide": false, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Degraded", + "refId": "F", + "step": 300 + }, + { + "expr": "ceph_pg_stale", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Stale", + "refId": "A", + "step": 300 + }, + { + "expr": "ceph_pg_undersized", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Undersized", + "refId": "B", + "step": 300 + }, + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "H" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Stuck PGs", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 42, + "panels": [], + "repeat": null, + "title": "RECOVERY", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 58 + }, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(ceph_osd_recovery_ops[$interval]))", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "OPS", + "refId": "A", + "step": 300 + }, + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Operations", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "", + "schemaVersion": 22, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "auto": true, + "auto_count": 10, + "auto_min": "1m", + "current": { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + "datasource": null, + "hide": 0, + "includeAll": false, + "label": "Interval", + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": false, + "text": "5s", + "value": "5s" + }, + { + "selected": false, + "text": "10s", + "value": "10s" + }, + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "5s,10s,30s,1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Ceph Cluster Overview", + "uid": "oVDYSH1Wz", + "version": 6 +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_pools.json b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_pools.json new file mode 100644 index 000000000..847350a0f --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_pools.json @@ -0,0 +1,752 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 182, + "iteration": 1616693919192, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 4, + "fillGradient": 0, + "gridPos": { + "h": 14, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 1, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.* read/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk($topk,rate(ceph_pool_rd[5m]) + on(pool_id) group_left(instance,name) ceph_pool_metadata) ", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{name}} - read", + "refId": "F" + }, + { + "expr": "topk($topk,rate(ceph_pool_wr[5m]) + on(pool_id) group_left(instance,name) ceph_pool_metadata) ", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{name}} - write", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Top $topk Client IOPS by Pool", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "Read (-) / Write (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 3, + "fillGradient": 0, + "gridPos": { + "h": 14, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.* read/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk($topk,rate(ceph_pool_rd_bytes[5m]) + on(pool_id) group_left(instance,name) ceph_pool_metadata)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{name}} - read", + "refId": "A", + "textEditor": true + }, + { + "expr": "topk($topk,rate(ceph_pool_wr_bytes[5m]) + on(pool_id) group_left(instance,name) ceph_pool_metadata)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{name}} - write", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Top $topk Client Throughput by Pool", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": "Read (-) / Writes (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [ + { + "text": "Current", + "value": "current" + } + ], + "datasource": "$datasource", + "fontSize": "100%", + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "id": 3, + "links": [], + "maxPerRow": 2, + "options": {}, + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 5, + "desc": true + }, + "styles": [ + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "id", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "instance", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "job", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Pool Name", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "name", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Pool ID", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "pool_id", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS (R+W)", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value", + "thresholds": [], + "type": "number", + "unit": "none" + } + ], + "targets": [ + { + "expr": "topk($topk,((irate(ceph_pool_rd[5m]) + irate(ceph_pool_wr[5m])) + on(pool_id) group_left(instance,name) ceph_pool_metadata))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "refId": "A", + "textEditor": true + } + ], + "title": "Top $topk Pools by Client IOPS", + "transform": "table", + "type": "table" + }, + { + "columns": [ + { + "text": "Current", + "value": "current" + } + ], + "datasource": "$datasource", + "fontSize": "100%", + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 14 + }, + "id": 4, + "links": [], + "maxPerRow": 2, + "options": {}, + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 5, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "id", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "instance", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "job", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Pool Name", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "name", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Pool ID", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "pool_id", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value", + "thresholds": [], + "type": "number", + "unit": "decbytes" + } + ], + "targets": [ + { + "expr": "topk($topk,(irate(ceph_pool_rd_bytes[5m]) + irate(ceph_pool_wr_bytes[5m])) + on(pool_id) group_left(instance,name) ceph_pool_metadata) ", + "format": "table", + "instant": true, + "intervalFactor": 2, + "refId": "A", + "textEditor": true + } + ], + "title": "Top $topk Pools by Throughput", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "$datasource", + "fontSize": "100%", + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 14 + }, + "id": 5, + "links": [], + "maxPerRow": 3, + "options": {}, + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 5, + "desc": true + }, + "styles": [ + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "instance", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "job", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Pool Name", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "name", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Pool ID", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "pool_id", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Capacity Used", + "align": "auto", + "colorMode": "value", + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value", + "thresholds": [ + "70", + "85" + ], + "type": "number", + "unit": "percentunit" + } + ], + "targets": [ + { + "expr": "topk(1,((ceph_pool_stored / (ceph_pool_stored + ceph_pool_max_avail)) * on(pool_id) group_left(name) ceph_pool_metadata))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "D" + } + ], + "title": "Top $topk Pools By Capacity Used", + "transform": "table", + "type": "table" + } + ], + "refresh": false, + "schemaVersion": 22, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "text": "15", + "value": "15" + }, + "hide": 0, + "label": "Top K", + "name": "topk", + "options": [ + { + "text": "15", + "value": "15" + } + ], + "query": "15", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "15s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Ceph Pools Overview", + "uid": "z99hzWtmk", + "version": 3 +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/blackbox.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/blackbox.json new file mode 100644 index 000000000..dffd96a4c --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/blackbox.json @@ -0,0 +1,1050 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Prometheus Blackbox Exporter Overview", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 7587, + "graphTooltip": 0, + "iteration": 1651051627287, + "links": [], + "liveNow": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 138, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "probe_duration_seconds{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Global Probe Duration", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 15, + "panels": [], + "repeat": "target", + "title": "$target status", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "text": "DOWN" + }, + "1": { + "text": "UP" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "#299c46", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 9 + }, + "id": 2, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "repeatDirection": "v", + "targets": [ + { + "expr": "probe_success{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Status", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 10, + "x": 4, + "y": 9 + }, + "hiddenSeries": false, + "id": 25, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "probe_http_duration_seconds{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ phase }}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "HTTP Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 10, + "x": 14, + "y": 9 + }, + "hiddenSeries": false, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "probe_duration_seconds{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "seconds", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Probe Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "0": { + "text": "N/A" + }, + "1": { + "text": "YES" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 201 + }, + { + "color": "#d44a3a", + "value": 399 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 11 + }, + "id": 20, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "repeatDirection": "h", + "targets": [ + { + "expr": "probe_http_status_code{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "HTTP Status Code", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 13 + }, + "id": 27, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "probe_http_version{instance=~\"$target\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "HTTP Version", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "text": "NO" + }, + "1": { + "text": "YES" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 + }, + { + "color": "#299c46", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 15 + }, + "id": 18, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "repeatDirection": "v", + "targets": [ + { + "expr": "probe_http_ssl{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "SSL", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "0": { + "text": "NO" + }, + "1": { + "text": "YES" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 + }, + { + "color": "#299c46", + "value": 1209600 + } + ] + }, + "unit": "dtdurations" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 10, + "x": 4, + "y": 15 + }, + "id": 19, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "repeatDirection": "h", + "targets": [ + { + "expr": "probe_ssl_earliest_cert_expiry{instance=~\"$target\"} - time()", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "SSL Expiry", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 5, + "x": 14, + "y": 15 + }, + "id": 23, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "avg(probe_duration_seconds{instance=~\"$target\"})", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Average Probe Duration", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 5, + "x": 19, + "y": 15 + }, + "id": 24, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "repeatDirection": "h", + "targets": [ + { + "expr": "avg(probe_dns_lookup_time_seconds{instance=~\"$target\"})", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Average DNS Lookup", + "type": "stat" + } + ], + "refresh": "", + "schemaVersion": 34, + "style": "dark", + "tags": [ + "blackbox", + "prometheus" + ], + "templating": { + "list": [ + { + "auto": true, + "auto_count": 10, + "auto_min": "10s", + "current": { + "selected": false, + "text": "10s", + "value": "10s" + }, + "hide": 0, + "label": "Interval", + "name": "interval", + "options": [ + { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": false, + "text": "5s", + "value": "5s" + }, + { + "selected": true, + "text": "10s", + "value": "10s" + }, + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "5s,10s,30s,1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "target", + "options": [], + "query": { + "query": "label_values(probe_success, instance)", + "refId": "Prometheus-target-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Prometheus Blackbox Exporter", + "uid": "xtkCtBkiz", + "version": 3, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/cadvisor.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/cadvisor.json new file mode 100644 index 000000000..3ac11afb4 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/cadvisor.json @@ -0,0 +1,1302 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "A simple overview of the most important Docker host and container metrics. (cAdvisor/Prometheus)", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 893, + "graphTooltip": 1, + "id": 101, + "iteration": 1658138926032, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 43, + "panels": [], + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 31, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "count(rate(container_last_seen{name=~\".+\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 1800 + }, + { + "expr": "", + "refId": "B" + } + ], + "title": "Global Container Count", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 38, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "count(container_last_seen{job=\"$job\", instance=\"$node\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 1800 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "", + "refId": "B" + } + ], + "title": "Selected Container Count", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 52, + "panels": [], + "title": "Host Summary", + "type": "row" + }, + { + "aliasColors": { + "{id=\"/\",instance=\"cadvisor:8080\",job=\"prometheus\"}": "#BA43A9" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 6 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "avg by (instance) (irate(container_cpu_usage_seconds_total{name=~\".+\",job=\"$job\", instance=\"$node\"}[$interval])) * 100", + "hide": false, + "interval": "", + "intervalFactor": 5, + "legendFormat": "{{ name }}", + "metric": "", + "refId": "C", + "step": 600 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Per Host CPU Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": false, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:486", + "format": "percent", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:487", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "{id=\"/\",instance=\"cadvisor:8080\",job=\"prometheus\"}": "#BA43A9" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 6 + }, + "hiddenSeries": false, + "id": 48, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "sum(container_memory_usage_bytes{name=~\".+\",job=\"$job\", instance=\"$node\"}) by (instance)", + "hide": false, + "interval": "", + "intervalFactor": 5, + "legendFormat": "{{ instance }}", + "metric": "", + "refId": "C", + "step": 600 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Per Host Memory Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": false, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:812", + "format": "decbytes", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:813", + "format": "short", + "logBase": 1, + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 50, + "panels": [], + "title": "Network summary", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "rate(container_network_receive_bytes_total{id=\"/\",job=\"$job\", instance=\"$node\"}[$interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "rx - {{interface}}", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "- rate(container_network_transmit_bytes_total{id=\"/\",job=\"$job\", instance=\"$node\"}[$interval])", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "tx - {{interface}}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network Traffic ", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 1, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:972", + "format": "Bps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:973", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 47, + "panels": [], + "title": "CPU and Memory per Container", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 14, + "w": 12, + "x": 0, + "y": 22 + }, + "hiddenSeries": false, + "id": 1, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": 250, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{name=~\".+\",job=\"$job\", instance=\"$node\"}[$interval])) by (name) * 100", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "F", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU Usage per Container", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:650", + "format": "percent", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:651", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 3, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 14, + "w": 12, + "x": 12, + "y": 22 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": 250, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum(container_memory_usage_bytes{name=~\".+\",job=\"$job\", instance=\"$node\"}) by (name)", + "hide": false, + "legendFormat": "{{ name }}", + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Usage per Container", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": "", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 45, + "panels": [], + "title": "Network details", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 37 + }, + "hiddenSeries": false, + "id": 39, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "rate(container_network_receive_bytes_total{id=\"/\",job=\"$job\", instance=\"$node\"}[$interval])", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{interface}} {{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Receive Network Traffic", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 1, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1316", + "format": "Bps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1317", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 37 + }, + "hiddenSeries": false, + "id": 41, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "rate(container_network_transmit_bytes_total{id=\"/\",job=\"$job\", instance=\"$node\"}[$interval])", + "hide": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{interface}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "rate(container_network_transmit_bytes_total{id=\"/\",job=\"$job\", instance=\"$node\"}[$interval])", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{interface}} {{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Transmit Network Traffic", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 1, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2071", + "format": "Bps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2072", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": false, + "schemaVersion": 36, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": ".+", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(container_cpu_load_average_10s, name)", + "hide": 0, + "includeAll": true, + "label": "Container Name", + "multi": true, + "name": "name", + "options": [], + "query": { + "query": "label_values(container_cpu_load_average_10s, name)", + "refId": "Prometheus-name-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query", + "useTags": false + }, + { + "auto": true, + "auto_count": 50, + "auto_min": "50s", + "current": { + "selected": false, + "text": "5m", + "value": "5m" + }, + "hide": 0, + "includeAll": false, + "label": "Interval", + "multi": false, + "name": "interval", + "options": [ + { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "2m", + "value": "2m" + }, + { + "selected": false, + "text": "3m", + "value": "3m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "7m", + "value": "7m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "30s,1m,2m,3m,5m,7m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "queryValue": "", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "current": { + "selected": false, + "text": "cadvisor", + "value": "cadvisor" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(container_cpu_usage_seconds_total, job)", + "hide": 0, + "includeAll": false, + "label": "job", + "multi": false, + "name": "job", + "options": [], + "query": { + "query": "label_values(container_cpu_usage_seconds_total, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "kef1p-phycon0001", + "value": "kef1p-phycon0001" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(container_cpu_usage_seconds_total{job=\"$job\"}, instance)", + "hide": 0, + "includeAll": false, + "label": "Node", + "multi": false, + "name": "node", + "options": [], + "query": { + "query": "label_values(container_cpu_usage_seconds_total{job=\"$job\"}, instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Control Plane Containers (CAdvisor) - update", + "uid": "DV5zI13Wi", + "version": 3, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/database.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/database.json new file mode 100644 index 000000000..70fb95e08 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/database.json @@ -0,0 +1,1023 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Basic Mysql dashboard for the prometheus exporter ", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 6239, + "graphTooltip": 0, + "id": 89, + "iteration": 1658139364595, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 17, + "panels": [], + "title": "Global status", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#bf1b00", + "value": null + }, + { + "color": "#508642", + "value": 1 + }, + { + "color": "#ef843c", + "value": 2 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 11, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "min(mysql_up)", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "refId": "A" + }, + { + "expr": "mysql_galera_status_info", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "B" + } + ], + "title": "Galera status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 25200 + }, + { + "color": "#508642", + "value": 32400 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 15, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "min(mysql_global_status_uptime)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1 + }, + "hiddenSeries": false, + "id": 29, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "mysql_global_status_max_used_connections", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "current {{ instance }}", + "refId": "A" + }, + { + "expr": "min(mysql_global_variables_max_connections)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Max allowed", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Mysql Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 19, + "panels": [], + "title": "I/O", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "write", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(mysql_global_status_innodb_data_reads[10m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "reads {{ instance }}", + "refId": "A" + }, + { + "expr": "irate(mysql_global_status_innodb_data_written[10m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "write {{ instance }}", + "refId": "B" + }, + { + "expr": "", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "write {{ instance }}", + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "mysql disk reads vs writes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 9 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/sent/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(mysql_global_status_bytes_received[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "received {{ instance }}", + "refId": "A" + }, + { + "expr": "irate(mysql_global_status_bytes_sent[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "sent {{ instance }}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "mysql network received vs sent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(mysql_global_status_commands_total[5m])) by (command)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ command }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Query rates", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 18 + }, + "hiddenSeries": false, + "id": 25, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "mysql_global_status_threads_running ", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Running Threads", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": "15", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 21, + "panels": [], + "title": "Errors", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The number of connections that were aborted because the client died without closing the connection properly. See Section B.5.2.10, “Communication Errors and Aborted Connections”.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 29 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(mysql_global_status_aborted_clients[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Aborted clients", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The number of failed attempts to connect to the MySQL server. See Section B.5.2.10, “Communication Errors and Aborted Connections”.\n\nFor additional connection-related information, check the Connection_errors_xxx status variables and the host_cache table.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 29 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": false, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(mysql_global_status_aborted_connects[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "mysql aborted Connects", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "schemaVersion": 36, + "style": "dark", + "tags": [ + "Databases", + "backgroundservices" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "10.3.28", + "value": "10.3.28" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(mysql_version_info, innodb_version)", + "hide": 0, + "includeAll": false, + "label": "MariaDB Version", + "multi": false, + "name": "release", + "options": [], + "query": { + "query": "label_values(mysql_version_info, innodb_version)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Database", + "uid": "6-kPlS7il", + "version": 3, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/elasticsearch.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/elasticsearch.json new file mode 100644 index 000000000..e4522f306 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/elasticsearch.json @@ -0,0 +1,7206 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "ElasticSearch cluster stats", + "editable": true, + "gnetId": 2322, + "graphTooltip": 1, + "id": 170, + "iteration": 1616778067978, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "OS" + ], + "targetBlank": true, + "title": "OS", + "type": "dashboards" + }, + { + "asDropdown": true, + "icon": "external link", + "keepTime": true, + "tags": [ + "MySQL" + ], + "targetBlank": true, + "title": "MySQL", + "type": "dashboards" + }, + { + "asDropdown": true, + "icon": "external link", + "keepTime": true, + "tags": [ + "MongoDB" + ], + "targetBlank": true, + "title": "MongoDB", + "type": "dashboards" + }, + { + "asDropdown": true, + "icon": "external link", + "keepTime": true, + "tags": [ + "App" + ], + "targetBlank": true, + "title": "App", + "type": "dashboards" + } + ], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 90, + "panels": [], + "repeat": null, + "title": "KPI", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#d44a3a", + "rgba(237, 129, 40, 0.89)", + "#299c46" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 1 + }, + "height": "50", + "id": 53, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "topk(1, elasticsearch_cluster_health_status{cluster=\"$cluster\",color=\"red\"}==1 or (elasticsearch_cluster_health_status{cluster=\"$cluster\",color=\"green\"}==1)+4 or (elasticsearch_cluster_health_status{cluster=\"$cluster\",color=\"yellow\"}==1)+22)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "2,4", + "title": "Cluster health", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + }, + { + "op": "=", + "text": "Green", + "value": "5" + }, + { + "op": "=", + "text": "Yellow", + "value": "3" + }, + { + "op": "=", + "text": "Red", + "value": "1" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "decimals": null, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 4, + "y": 1 + }, + "height": "50", + "id": 81, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "__name__", + "targets": [ + { + "expr": "count(elasticsearch_breakers_tripped{cluster=\"$cluster\",name=~\"$name\"}>0)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "1,2", + "title": "Tripped for breakers", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "N/A" + }, + { + "op": "=", + "text": "0", + "value": "no value" + }, + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 6, + "y": 1 + }, + "height": "50", + "id": 51, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum (elasticsearch_process_cpu_percent{cluster=\"$cluster\",name=~\"$name\"} ) / count (elasticsearch_process_cpu_percent{cluster=\"$cluster\",name=~\"$name\"} )", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "70,80", + "title": "CPU usage Avg.", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 10, + "y": 1 + }, + "height": "50", + "id": 50, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum (elasticsearch_jvm_memory_used_bytes{cluster=\"$cluster\",name=~\"$name\"}) / sum (elasticsearch_jvm_memory_max_bytes{cluster=\"$cluster\",name=~\"$name\"}) * 100", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "70,80", + "title": "JVM memory used Avg.", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "description": "Number of nodes in the cluster", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 14, + "y": 1 + }, + "height": "50", + "id": 10, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "topk(1, elasticsearch_cluster_health_number_of_nodes{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "", + "title": "Nodes", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "description": "Number of data nodes in the cluster", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 16, + "y": 1 + }, + "height": "50", + "id": 9, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "topk(1, elasticsearch_cluster_health_number_of_data_nodes{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "", + "title": "Data nodes", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "description": "Cluster level changes which have not yet been executed", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 18, + "y": 1 + }, + "height": "50", + "hideTimeOverride": true, + "id": 16, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "topk(1, elasticsearch_cluster_health_number_of_pending_tasks{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "1,5", + "title": "Pending tasks", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 1 + }, + "height": "50", + "id": 89, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum (elasticsearch_process_open_files_count{cluster=\"$cluster\",name=~\"$name\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "", + "title": "Open file descriptors per cluster", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [], + "valueName": "current" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 91, + "panels": [], + "repeat": null, + "title": "Shards", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "description": "The number of primary shards in your cluster. This is an aggregate total across all indices.", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 5 + }, + "height": "50", + "id": 11, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "maxPerRow": 6, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "repeat": "shard_type", + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "topk(1, elasticsearch_cluster_health_active_primary_shards{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "", + "title": "Active primary shards", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "description": "Aggregate total of all shards across all indices, which includes replica shards", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 5 + }, + "height": "50", + "id": 39, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "maxPerRow": 6, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "topk(1, elasticsearch_cluster_health_active_shards{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "", + "title": "Active shards", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "description": "Count of shards that are being freshly created", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 5 + }, + "height": "50", + "id": 40, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "maxPerRow": 6, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "topk(1, elasticsearch_cluster_health_initializing_shards{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "", + "title": "Initializing shards", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "description": "The number of shards that are currently moving from one node to another node.", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 5 + }, + "height": "50", + "id": 41, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "maxPerRow": 6, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "topk(1, elasticsearch_cluster_health_relocating_shards{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "", + "title": "Relocating shards", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "description": "Shards delayed to reduce reallocation overhead", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 5 + }, + "height": "50", + "id": 42, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "maxPerRow": 6, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "topk(1, elasticsearch_cluster_health_delayed_unassigned_shards{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "", + "title": "Delayed shards", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "description": "The number of shards that exist in the cluster state, but cannot be found in the cluster itself", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 5 + }, + "height": "50", + "id": 82, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "maxPerRow": 6, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "topk(1, elasticsearch_cluster_health_unassigned_shards{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "", + "title": "Unassigned shards", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 92, + "panels": [], + "repeat": null, + "title": "JVM Garbage Collection", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 9 + }, + "height": "400", + "hiddenSeries": false, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(elasticsearch_jvm_gc_collection_seconds_count{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}} - {{gc}}", + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GC count", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "GCs", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 9 + }, + "height": "400", + "hiddenSeries": false, + "id": 27, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(elasticsearch_jvm_gc_collection_seconds_sum{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}} - {{gc}}", + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GC time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "Time", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 93, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 77, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(elasticsearch_indices_translog_operations{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total translog operations", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 78, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(elasticsearch_indices_translog_size_in_bytes{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total translog size in bytes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "title": "Translog", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 94, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 79, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_breakers_tripped{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: {{breaker}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Tripped for breakers", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 80, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_breakers_estimated_size_bytes{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: {{breaker}}", + "refId": "A" + }, + { + "expr": "elasticsearch_breakers_limit_size_bytes{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: limit for {{breaker}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Estimated size in bytes of breaker", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "title": "Breakers", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 95, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 34 + }, + "height": "400", + "id": 30, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_os_load1{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "load1: {{name}}", + "metric": "", + "refId": "A", + "step": 20 + }, + { + "expr": "elasticsearch_os_load5{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "load5: {{name}}", + "metric": "", + "refId": "B", + "step": 20 + }, + { + "expr": "elasticsearch_os_load15{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "load15: {{name}}", + "metric": "", + "refId": "C", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Load average", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "CPU usage", + "logBase": 1, + "max": 100, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 34 + }, + "height": "400", + "id": 88, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_process_cpu_percent{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": "CPU usage", + "logBase": 1, + "max": 100, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 41 + }, + "height": "400", + "id": 31, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_jvm_memory_used_bytes{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}} used: {{area}}", + "metric": "", + "refId": "A", + "step": 20 + }, + { + "expr": "elasticsearch_jvm_memory_max_bytes{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}} max: {{area}}", + "refId": "C", + "step": 20 + }, + { + "expr": "elasticsearch_jvm_memory_pool_peak_used_bytes{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}} peak used pool: {{pool}}", + "refId": "D", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "JVM memory usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Memory", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 41 + }, + "height": "400", + "id": 54, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_jvm_memory_committed_bytes{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}} committed: {{area}}", + "refId": "B", + "step": 20 + }, + { + "expr": "elasticsearch_jvm_memory_max_bytes{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}} max: {{area}}", + "refId": "C", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "JVM memory committed", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Memory", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "title": "CPU and Memory", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 96, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 42 + }, + "height": "400", + "id": 32, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1-(elasticsearch_filesystem_data_available_bytes{cluster=\"$cluster\",name=~\"$name\"}/elasticsearch_filesystem_data_size_bytes{cluster=\"$cluster\",name=~\"$name\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}: {{path}}", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "thresholds": [ + { + "colorMode": "custom", + "fill": true, + "fillColor": "rgba(216, 200, 27, 0.27)", + "op": "gt", + "value": 0.8 + }, + { + "colorMode": "custom", + "fill": true, + "fillColor": "rgba(234, 112, 112, 0.22)", + "op": "gt", + "value": 0.9 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "Disk Usage %", + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 42 + }, + "height": "400", + "id": 47, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "sent", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(elasticsearch_transport_tx_size_bytes_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: sent ", + "refId": "D", + "step": 20 + }, + { + "expr": "-irate(elasticsearch_transport_rx_size_bytes_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: received", + "refId": "C", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": "Bytes/sec", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "pps", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "title": "Disk and Network", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 97, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 43 + }, + "height": "400", + "id": 1, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_docs{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Documents count on node", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 43 + }, + "height": "400", + "id": 24, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "irate(elasticsearch_indices_indexing_index_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Documents indexed rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "index calls/s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "Count of deleted documents on this node", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 11, + "w": 8, + "x": 0, + "y": 50 + }, + "height": "400", + "id": 25, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "irate(elasticsearch_indices_docs_deleted{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Documents deleted rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Documents/s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 11, + "w": 8, + "x": 8, + "y": 50 + }, + "height": "400", + "id": 26, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(elasticsearch_indices_merges_docs_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Documents merged rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "short", + "label": "Documents/s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 11, + "w": 8, + "x": 16, + "y": 50 + }, + "height": "400", + "id": 52, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "irate(elasticsearch_indices_merges_total_size_bytes_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Documents merged bytes", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "decbytes", + "label": "Bytes/s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "title": "Documents", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 98, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 51 + }, + "height": "400", + "id": 33, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(elasticsearch_indices_search_query_time_seconds{cluster=\"$cluster\",name=~\"$name\"}[$interval]) ", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Query time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "Time", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 51 + }, + "height": "400", + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(elasticsearch_indices_indexing_index_time_seconds_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Indexing time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "Time", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 58 + }, + "height": "400", + "id": 3, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(elasticsearch_indices_merges_total_time_seconds_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Merging time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "Time", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 58 + }, + "height": "400", + "id": 87, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(elasticsearch_indices_store_throttle_time_seconds_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Throttle time for index store", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "Time", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "title": "Times", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 99, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 59 + }, + "height": "400", + "id": 48, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(elasticsearch_indices_indexing_index_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}: indexing", + "metric": "", + "refId": "A", + "step": 10 + }, + { + "expr": "rate(elasticsearch_indices_search_query_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: query", + "refId": "B", + "step": 10 + }, + { + "expr": "rate(elasticsearch_indices_search_fetch_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: fetch", + "refId": "C", + "step": 10 + }, + { + "expr": "rate(elasticsearch_indices_merges_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: merges", + "refId": "D", + "step": 10 + }, + { + "expr": "rate(elasticsearch_indices_refresh_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: refresh", + "refId": "E", + "step": 10 + }, + { + "expr": "rate(elasticsearch_indices_flush_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: flush", + "refId": "F", + "step": 10 + }, + { + "expr": "rate(elasticsearch_indices_get_exists_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: get_exists", + "refId": "G", + "step": 10 + }, + { + "expr": "rate(elasticsearch_indices_get_missing_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: get_missing", + "refId": "H", + "step": 10 + }, + { + "expr": "rate(elasticsearch_indices_get_tota{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: get", + "refId": "I", + "step": 10 + }, + { + "expr": "rate(elasticsearch_indices_indexing_delete_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: indexing_delete", + "refId": "J", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total Operations rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Operations/s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 66 + }, + "height": "400", + "id": 49, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(elasticsearch_indices_indexing_index_time_seconds_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}: indexing", + "metric": "", + "refId": "A", + "step": 10 + }, + { + "expr": "irate(elasticsearch_indices_search_query_time_seconds{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: query", + "refId": "B", + "step": 10 + }, + { + "expr": "irate(elasticsearch_indices_search_fetch_time_seconds{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: fetch", + "refId": "C", + "step": 10 + }, + { + "expr": "irate(elasticsearch_indices_merges_total_time_seconds_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: merges", + "refId": "D", + "step": 10 + }, + { + "expr": "irate(elasticsearch_indices_refresh_time_seconds_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: refresh", + "refId": "E", + "step": 10 + }, + { + "expr": "irate(elasticsearch_indices_flush_time_seconds{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: flush", + "refId": "F", + "step": 10 + }, + { + "expr": "irate(elasticsearch_indices_get_exists_time_seconds{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: get_exists", + "refId": "G", + "step": 10 + }, + { + "expr": "irate(elasticsearch_indices_get_time_seconds{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: get_time", + "refId": "H", + "step": 10 + }, + { + "expr": "irate(elasticsearch_indices_get_missing_time_seconds{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: get_missing", + "refId": "I", + "step": 10 + }, + { + "expr": "irate(elasticsearch_indices_indexing_delete_time_seconds_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: indexing_delete", + "refId": "J", + "step": 10 + }, + { + "expr": "irate(elasticsearch_indices_get_time_seconds{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: get", + "refId": "K", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total Operations time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "Time", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "title": "Total Operations stats", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 100, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 20, + "w": 6, + "x": 0, + "y": 67 + }, + "id": 45, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": true, + "min": true, + "show": true, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(elasticsearch_thread_pool_rejected_count{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: {{ type }}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Thread Pool operations rejected", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 20, + "w": 6, + "x": 6, + "y": 67 + }, + "id": 46, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_thread_pool_active_count{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: {{ type }}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Thread Pool operations queued", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 20, + "w": 6, + "x": 12, + "y": 67 + }, + "height": "", + "id": 43, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_thread_pool_active_count{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: {{ type }}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Thread Pool threads active", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 20, + "w": 6, + "x": 18, + "y": 67 + }, + "id": 44, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(elasticsearch_thread_pool_completed_count{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: {{ type }}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Thread Pool operations completed", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "title": "Thread Pool", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 101, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 68 + }, + "height": "400", + "id": 4, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_fielddata_memory_size_bytes{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Field data memory size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Memory", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 68 + }, + "height": "400", + "id": 34, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(elasticsearch_indices_fielddata_evictions{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Field data evictions", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Evictions/s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 11, + "w": 8, + "x": 0, + "y": 75 + }, + "height": "400", + "id": 35, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_query_cache_memory_size_bytes{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Query cache size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Size", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 11, + "w": 8, + "x": 8, + "y": 75 + }, + "height": "400", + "id": 36, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(elasticsearch_indices_query_cache_evictions{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Query cache evictions", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Evictions/s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 11, + "w": 8, + "x": 16, + "y": 75 + }, + "height": "400", + "id": 84, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(elasticsearch_indices_filter_cache_evictions{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Evictions from filter cache", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Evictions/s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "title": "Caches", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 102, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 76 + }, + "id": 85, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_segments_count{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Count of index segments", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 83 + }, + "id": 86, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_segments_memory_bytes{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Current memory size of segments in bytes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "title": "Segments", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 103, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 84 + }, + "id": 75, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_docs_primary{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{index}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Count of documents with only primary shards", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 91 + }, + "id": 83, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_store_size_bytes_primary{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{index}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total size of stored index data in bytes with only primary shards on all nodes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 98 + }, + "id": 76, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_store_size_bytes_total{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{index}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total size of stored index data in bytes with all shards on all nodes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "title": "Indices: Count of documents and Total size", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 104, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 99 + }, + "id": 61, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_segment_index_writer_memory_bytes_primary{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{index}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Index writer with only primary shards on all nodes in bytes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 106 + }, + "id": 62, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_segment_index_writer_memory_bytes_total{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{index}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Index writer with all shards on all nodes in bytes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "title": "Indices: Index writer", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 105, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 107 + }, + "id": 55, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_segment_count_primary{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{index}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Segments with only primary shards on all nodes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 114 + }, + "id": 56, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_segment_count_total{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{index}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Segments with all shards on all nodes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 121 + }, + "id": 65, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_segment_memory_bytes_primary{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{index}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Size of segments with only primary shards on all nodes in bytes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 128 + }, + "id": 66, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_segment_memory_bytes_total{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{index}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Size of segments with all shards on all nodes in bytes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "title": "Indices: Segments", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 106, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 129 + }, + "id": 57, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_segment_doc_values_memory_bytes_primary{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{index}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Doc values with only primary shards on all nodes in bytes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 136 + }, + "id": 58, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_segment_doc_values_memory_bytes_total{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{index}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Doc values with all shards on all nodes in bytes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "title": "Indices: Doc values", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 107, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 137 + }, + "id": 59, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_segment_fields_memory_bytes_primary{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{index}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Size of fields with only primary shards on all nodes in bytes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 144 + }, + "id": 60, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_segment_fields_memory_bytes_total{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{index}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Size of fields with all shards on all nodes in bytes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "title": "Indices: Fields", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 108, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 145 + }, + "id": 63, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_segment_fixed_bit_set_memory_bytes_primary{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{index}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Size of fixed bit with only primary shards on all nodes in bytes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 152 + }, + "id": 64, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_segment_fixed_bit_set_memory_bytes_total{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{index}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Size of fixed bit with all shards on all nodes in bytes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "title": "Indices: Fixed bit", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 109, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 153 + }, + "id": 67, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_segment_norms_memory_bytes_primary{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{index}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Size of norms with only primary shards on all nodes in bytes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 160 + }, + "id": 68, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_segment_norms_memory_bytes_total{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{index}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Size of norms with all shards on all nodes in bytes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "title": "Indices: Norms", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 110, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 161 + }, + "id": 69, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_segment_points_memory_bytes_primary{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{index}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Size of points with only primary shards on all nodes in bytes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 168 + }, + "id": 70, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_segment_points_memory_bytes_total{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{index}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Size of points with all shards on all nodes in bytes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "title": "Indices: Points", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 111, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 169 + }, + "id": 71, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_segment_terms_memory_primary{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{index}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Size of terms with only primary shards on all nodes in bytes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 176 + }, + "id": 72, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_segment_terms_memory_total{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{index}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Number of terms with all shards on all nodes in bytes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "title": "Indices: Terms", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 112, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 177 + }, + "id": 73, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_segment_version_map_memory_bytes_primary{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{index}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Size of version map with only primary shards on all nodes in bytes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 184 + }, + "id": 74, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_segment_version_map_memory_bytes_total{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{index}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Size of version map with all shards on all nodes in bytes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "title": "Indices: Version map", + "type": "row" + } + ], + "refresh": false, + "schemaVersion": 22, + "style": "dark", + "tags": [ + "elasticsearch", + "App" + ], + "templating": { + "list": [ + { + "auto": true, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + "hide": 0, + "label": "Interval", + "name": "interval", + "options": [ + { + "selected": true, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "5m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "allValue": null, + "current": { + "text": "kolla_logging", + "value": "kolla_logging" + }, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Сluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(elasticsearch_indices_docs,cluster)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": null, + "tags": [], + "tagsQuery": null, + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Node name", + "multi": true, + "name": "name", + "options": [], + "query": "label_values(elasticsearch_indices_docs{cluster=\"$cluster\", name!=\"\"},name)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": null, + "tags": [], + "tagsQuery": null, + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "10.205.1.4:9108", + "value": "10.205.1.4:9108" + }, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Source of metrics", + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(elasticsearch_indices_docs{cluster=\"$cluster\", name!=\"\"},instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": null, + "tags": [], + "tagsQuery": null, + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "ElasticSearch", + "uid": "LNua-zOMC", + "version": 1 +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/fluentd.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/fluentd.json new file mode 100644 index 000000000..2339db271 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/fluentd.json @@ -0,0 +1,3207 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "fluentd 1.x deployed in kubernetes and scraped by prometheus", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 13042, + "graphTooltip": 1, + "id": 95, + "iteration": 1658139146884, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 19, + "panels": [], + "title": "General", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "If you see errors then you probbaly have serious issues with log processing, see https://docs.fluentd.org/buffer#handling-unrecoverable-errors\n\nRetries are normal but should occur only from time to time, otherwise check for network errors or destination is too slow and requires additional tuning per given provider.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:6571", + "alias": "/Error.*/", + "color": "#E02F44" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "sum(rate(fluentd_output_status_retry_count[5m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Retry rate", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum(rate(fluentd_output_status_num_errors[5m]))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Error rate", + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Fluentd output error/retry rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Input and output total rates", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 1 + }, + "hiddenSeries": false, + "id": 44, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(fluentd_input_status_num_records_total[5m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "input", + "refId": "A" + }, + { + "expr": "sum(rate(fluentd_output_status_write_count[5m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "output", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Input / output rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "This should not reach 0 otherwise logs are blocked from processing or even dropped", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 1 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(fluentd_output_status_buffer_available_space_ratio)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "lowest across all hosts", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "fluentd output status buffer available space ratio", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "percent", + "logBase": 1, + "max": "100", + "min": "0", + "show": true + }, + { + "decimals": 0, + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "total flush time", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 6 + }, + "hiddenSeries": false, + "id": 21, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "count", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(fluentd_output_status_flush_time_count[5m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "time", + "refId": "A" + }, + { + "expr": "sum(rate(fluentd_output_status_slow_flush_count[5m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "count", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "fluentd output status flush time count rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "decimals": 0, + "format": "short", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Current total size of stage and queue buffers.\nfluentd_output_status_buffer_total_bytes", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 6 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(fluentd_output_status_buffer_total_bytes) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ type }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current total size of stage and queue buffers", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 6 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(fluentd_output_status_buffer_queue_length)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "total", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Fluentd output buffer queue", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 41, + "panels": [], + "title": "Buffer Stage", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fluentd_output_status_buffer_stage_length", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(fluentd_output_status_buffer_stage_length) by (pod, type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current length of stage buffers", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fluentd_output_status_buffer_stage_byte_size", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 12 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(fluentd_output_status_buffer_stage_byte_size) by (pod, type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current total size of stage buffers", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 43, + "panels": [], + "title": "Buffer Queue", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 19 + }, + "hiddenSeries": false, + "id": 50, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max_over_time(fluentd_output_status_buffer_queue_length[5m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Maximum buffer length in last 5min", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 19 + }, + "hiddenSeries": false, + "id": 25, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max_over_time(fluentd_output_status_buffer_total_bytes[5m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Maximum buffer bytes in last 5min", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fluentd_output_status_buffer_queue_length", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 25 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(fluentd_output_status_buffer_queue_length) by (pod, type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current length of queue buffers", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fluentd_output_status_queue_byte_size", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 25 + }, + "hiddenSeries": false, + "id": 51, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(fluentd_output_status_queue_byte_size) by (pod, type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current total size of queue buffers", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 46, + "panels": [], + "title": "Buffer Space", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fluentd_output_status_buffer_available_space_ratio", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(fluentd_output_status_buffer_available_space_ratio) by (pod, type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Ratio of available space in buffer", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "logBase": 1, + "max": "100", + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 53, + "panels": [], + "title": "Buffer Retries", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fluentd_output_status_retry_count", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 39 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(fluentd_output_status_retry_count[5m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current retry counts", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fluentd_output_status_emit_records", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 39 + }, + "hiddenSeries": false, + "id": 33, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(fluentd_output_status_emit_records[5m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current emit records", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fluentd_output_status_emit_count", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 45 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(fluentd_output_status_emit_count[5m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current emit counts rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fluentd_output_status_rollback_count", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 45 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(fluentd_output_status_rollback_count[5m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current rollback counts", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fluentd_output_status_write_count", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 51 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(fluentd_output_status_write_count[5m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current write counts", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fluentd_output_status_slow_flush_count", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 51 + }, + "hiddenSeries": false, + "id": 37, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(fluentd_output_status_slow_flush_count[5m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current slow flush counts", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fluentd_output_status_retry_wait", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 57 + }, + "hiddenSeries": false, + "id": 38, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(fluentd_output_status_retry_wait[5m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current retry wait", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fluentd_output_status_flush_time_count", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 57 + }, + "hiddenSeries": false, + "id": 36, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(fluentd_output_status_flush_time_count[5m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total flush time", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 57, + "panels": [], + "title": "Buffer Errors", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fluentd_output_status_num_errors", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 64 + }, + "hiddenSeries": false, + "id": 31, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(fluentd_output_status_num_errors[5m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current number of errors rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fluentd_output_status_buffer_newest_timekey", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 64 + }, + "hiddenSeries": false, + "id": 27, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(fluentd_output_status_buffer_newest_timekey) by (pod, type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Newest timekey in buffer", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 70 + }, + "id": 55, + "panels": [], + "title": "Buffer timekeys", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 71 + }, + "hiddenSeries": false, + "id": 29, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "fluentd_output_status_buffer_newest_timekey - fluentd_output_status_buffer_oldest_timekey", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Timekey diff", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fluentd_output_status_buffer_oldest_timekey", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 71 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(fluentd_output_status_buffer_oldest_timekey) by (pod, type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Oldest timekey in buffer", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 77 + }, + "id": 17, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fluentd_input_status_num_records_total", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 78 + }, + "hiddenSeries": false, + "id": 39, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.4.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "sum(rate(fluentd_input_status_num_records_total{cluster=\"$cluster\", instance=\"$instance\"}[5m])) ", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "total", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Input entries rate (total)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fluentd_input_status_num_records_total", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 78 + }, + "hiddenSeries": false, + "id": 47, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.4.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(fluentd_input_status_num_records_total[5m])) by (hostname)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{hostname}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Input entries rate per hostname", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fluentd_input_status_num_records_total", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 84 + }, + "hiddenSeries": false, + "id": 60, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.4.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(fluentd_input_status_num_records_total[5m])) by (namespace)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Input entries rate per namespace", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fluentd_input_status_num_records_total", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 84 + }, + "hiddenSeries": false, + "id": 48, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.4.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(fluentd_input_status_num_records_total[5m])) by (instance)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Input entries rate per instance", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fluentd_input_status_num_records_total", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 6, + "y": 90 + }, + "hiddenSeries": false, + "id": 49, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.4.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(fluentd_input_status_num_records_total[5m])) by (tag)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{tag}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Input entries rate per tag", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "Input details - Not implemented", + "type": "row" + } + ], + "refresh": false, + "schemaVersion": 36, + "style": "dark", + "tags": [ + "fluentd", + "logging" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "kolla_logging", + "value": "kolla_logging" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(cluster)", + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(cluster)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Fluentd 1.x - Update", + "uid": "bNn5LUtizs2", + "version": 1, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/haproxy.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/haproxy.json new file mode 100644 index 000000000..82f50ffa6 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/haproxy.json @@ -0,0 +1,5377 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "HAProxy backend servers", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 2428, + "graphTooltip": 1, + "id": 92, + "iteration": 1658139194737, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 86, + "panels": [], + "title": "Basic Backend status", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 0, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 7, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 85, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 3, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Backends Down", + "color": "#890F02" + }, + { + "alias": "Backends Down", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "count(haproxy_backend_up{backend=~\"$backend\", instance=~\"$host\"} == 1)", + "hide": false, + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "Backends Up", + "refId": "A", + "step": 60 + }, + { + "expr": "count(haproxy_backend_up{backend=~\"$backend\",instance=~\"$host\"} == 0)", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "Backends Down", + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Backends Up / Down", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 87, + "panels": [], + "title": "Basic General Info", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 6 + }, + "hiddenSeries": false, + "id": 83, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Back.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*1.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*2.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*3.*/", + "color": "#1F78C1" + }, + { + "alias": "/.*4.*/", + "color": "#CCA300" + }, + { + "alias": "/.*5.*/", + "color": "#890F02" + }, + { + "alias": "/.*other.*/", + "color": "#806EB7" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(haproxy_frontend_http_responses_total{frontend=~\"$frontend\",code=~\"$code\",instance=~\"$host\"}[5m])) by (code)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Frontend {{ code }}", + "metric": "", + "refId": "A", + "step": 30 + }, + { + "expr": "sum(irate(haproxy_backend_http_responses_total{backend=~\"$backend\",code=~\"$code\",instance=~\"$host\"}[5m])) by (code)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Backend {{ code }}", + "metric": "", + "refId": "B", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total responses by HTTP code", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 1, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 75, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*OUT.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(haproxy_frontend_bytes_in_total{frontend=~\"$frontend\",instance=~\"$host\"}[5m])*8) by (instance)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "IN Front", + "metric": "", + "refId": "A", + "step": 30 + }, + { + "expr": "sum(irate(haproxy_frontend_bytes_out_total{frontend=~\"$frontend\",instance=~\"$host\"}[5m])*8) by (instance)", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "OUT Front", + "refId": "B", + "step": 60 + }, + { + "expr": "sum(irate(haproxy_backend_bytes_in_total{backend=~\"$backend\",instance=~\"$host\"}[5m])*8) by (instance)", + "intervalFactor": 2, + "legendFormat": "IN Back", + "refId": "C", + "step": 240 + }, + { + "expr": "sum(irate(haproxy_backend_bytes_out_total{backend=~\"$backend\",instance=~\"$host\"}[5m])*8) by (instance)", + "intervalFactor": 2, + "legendFormat": "OUT Back", + "refId": "D", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current total of incoming / outgoing bytes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 79, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Back.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(haproxy_frontend_connections_total{frontend=~\"$frontend\",instance=~\"$host\"}[5m])) by (instance)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Front", + "metric": "", + "refId": "A", + "step": 30 + }, + { + "expr": "sum(irate(haproxy_backend_connections_total{backend=~\"$backend\",instance=~\"$host\"}[5m])) by (instance)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Back", + "metric": "", + "refId": "B", + "step": 30 + }, + { + "expr": "sum(irate(haproxy_backend_connection_errors_total{backend=~\"$backend\",instance=~\"$host\"}[5m])) by (instance)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Back errors", + "metric": "", + "refId": "C", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total number of connections", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 26 + }, + "hiddenSeries": false, + "id": 81, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Response.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*Backend.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(haproxy_frontend_http_requests_total{frontend=~\"$frontend\",instance=~\"$host\"}[5m])) by (instance)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Requests", + "metric": "", + "refId": "A", + "step": 30 + }, + { + "expr": "sum(irate(haproxy_backend_response_errors_total{backend=~\"$backend\",instance=~\"$host\"}[5m])) by (instance)", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "Response errors", + "refId": "B", + "step": 60 + }, + { + "expr": "sum(irate(haproxy_frontend_request_errors_total{frontend=~\"$frontend\",instance=~\"$host\"}[5m])) by (instance)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Requests errors", + "metric": "", + "refId": "C", + "step": 30 + }, + { + "expr": "sum(irate(haproxy_backend_redispatch_warnings_total{backend=~\"$backend\",instance=~\"$host\"}[5m])) by (instance)", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "Backend redispatch", + "refId": "D", + "step": 60 + }, + { + "expr": "sum(irate(haproxy_backend_retry_warnings_total{backend=~\"$backend\",instance=~\"$host\"}[5m])) by (instance)", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "Backend retry", + "refId": "E", + "step": 60 + }, + { + "expr": "sum(irate(haproxy_frontend_requests_denied_total{frontend=~\"$frontend\",instance=~\"$host\"}[5m])) by (instance)", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "Request denied", + "refId": "F", + "step": 60 + }, + { + "expr": "sum(haproxy_backend_current_queue{backend=~\"$backend\",instance=~\"$host\"}) by (instance)", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "Backend Queued", + "refId": "G", + "step": 60 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total requests / responses", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 26 + }, + "hiddenSeries": false, + "id": 84, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Back.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(haproxy_frontend_current_sessions{frontend=~\"$frontend\",instance=~\"$host\"}) by (instance)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Frontend current sessions", + "metric": "", + "refId": "B", + "step": 30 + }, + { + "expr": "sum(haproxy_frontend_current_session_rate{frontend=~\"$frontend\",instance=~\"$host\"}) by (instance)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Frontend current session rate", + "metric": "", + "refId": "C", + "step": 30 + }, + { + "expr": "sum(haproxy_backend_current_sessions{backend=~\"$backend\",instance=~\"$host\"}) by (instance)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Backend current sessions", + "metric": "", + "refId": "A", + "step": 30 + }, + { + "expr": "sum(haproxy_backend_current_session_rate{backend=~\"$backend\",instance=~\"$host\"}) by (instance)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Backend current session rate", + "metric": "", + "refId": "D", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Sessions", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 88, + "panels": [], + "title": "Throughput", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 1, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 37 + }, + "hiddenSeries": false, + "id": 1, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*OUT.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(haproxy_backend_bytes_in_total{backend=~\"$backend\",instance=~\"$host\"}[5m])*8", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "IN-{{ backend }}", + "metric": "haproxy_backend_", + "refId": "A", + "step": 30 + }, + { + "expr": "irate(haproxy_backend_bytes_out_total{backend=~\"$backend\",instance=~\"$host\"}[5m])*8", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "OUT-{{ backend }}", + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Back - Current total of incoming / outgoing bytes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 1, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 37 + }, + "hiddenSeries": false, + "id": 42, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*OUT.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(haproxy_frontend_bytes_in_total{frontend=~\"$frontend\",instance=~\"$host\"}[5m])*8", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "IN-{{ frontend }}", + "metric": "", + "refId": "A", + "step": 30 + }, + { + "expr": "irate(haproxy_frontend_bytes_out_total{frontend=~\"$frontend\",instance=~\"$host\"}[5m])*8", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "OUT-{{ frontend }}", + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Front - Current total of incoming / outgoing bytes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 95, + "panels": [], + "title": "By Server Throughput", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 1, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 15, + "w": 24, + "x": 0, + "y": 48 + }, + "hiddenSeries": false, + "id": 54, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*OUT.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(haproxy_server_bytes_in_total{backend=~\"$backend\",server=~\"$server\",instance=~\"$host\"}[5m])*8", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "IN-{{ backend }} / {{ server }}", + "metric": "", + "refId": "A", + "step": 30 + }, + { + "expr": "irate(haproxy_server_bytes_out_total{backend=~\"$backend\",server=~\"$server\",instance=~\"$host\"}[5m])*8", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "OUT-{{ backend }} / {{ server }}", + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Current total of incoming / outgoing bytes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 89, + "panels": [], + "title": "Connections", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 64 + }, + "hiddenSeries": false, + "id": 27, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Error.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(haproxy_backend_connections_total{backend=~\"$backend\",instance=~\"$host\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ backend }}", + "metric": "", + "refId": "B", + "step": 30 + }, + { + "expr": "irate(haproxy_backend_connection_errors_total{backend=~\"$backend\",instance=~\"$host\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{ backend }} Error", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Back - Total number of connections OK / Error", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 64 + }, + "hiddenSeries": false, + "id": 43, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(haproxy_frontend_connections_total{frontend=~\"$frontend\",instance=~\"$host\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ frontend }}", + "metric": "haproxy_backe", + "refId": "B", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Front - Total number of connections", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 75 + }, + "id": 90, + "panels": [], + "title": "Queue", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 76 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(haproxy_backend_max_queue{backend=~\"$backend\",instance=~\"$host\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ backend }}", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Back - Maximum observed number of queued requests not assigned to any server", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 76 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "haproxy_backend_current_queue{backend=~\"$backend\",instance=~\"$host\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ backend }}", + "metric": "haproxy_backend_current_queue", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Back - Current number of queued requests not assigned to any server", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 86 + }, + "id": 91, + "panels": [], + "title": "Requests / Responses", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 87 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Error.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(haproxy_backend_redispatch_warnings_total{backend=~\"$backend\", instance=~\"$host\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Redispatch {{ backend }}", + "metric": "", + "refId": "A", + "step": 30 + }, + { + "expr": "irate(haproxy_backend_retry_warnings_total{backend=~\"$backend\", instance=~\"$host\"}[5m])", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "Retry {{ backend }}", + "refId": "B", + "step": 60 + }, + { + "expr": "irate(haproxy_backend_response_errors_total{backend=~\"$backend\", instance=~\"$host\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Error {{ backend }}", + "refId": "C", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Back - Total of redispatch / error / retry warnings", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 87 + }, + "hiddenSeries": false, + "id": 46, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Error.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*Denied.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(haproxy_frontend_http_requests_total{frontend=~\"$frontend\",instance=~\"$host\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ frontend }}", + "metric": "", + "refId": "A", + "step": 30 + }, + { + "expr": "irate(haproxy_frontend_request_errors_total{frontend=~\"$frontend\", instance=~\"$host\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{ frontend }} Error", + "refId": "B", + "step": 240 + }, + { + "expr": "irate(haproxy_frontend_requests_denied_total{frontend=~\"$frontend\", instance=~\"$host\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{ frontend }} Denied", + "refId": "C", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Front - Total HTTP requests OK / Error / Denied", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 99 + }, + "id": 92, + "panels": [], + "title": "Responses by HTTP code", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 100 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(haproxy_backend_http_responses_total{backend=~\"$backend\", code=~\"$code\",instance=~\"$host\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ code }} {{ backend }}", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Back - Total of HTTP responses", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 100 + }, + "hiddenSeries": false, + "id": 47, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(haproxy_frontend_http_responses_total{frontend=~\"$frontend\", code=~\"$code\",instance=~\"$host\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ code }} {{ frontend }} ", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Front - Total of HTTP responses", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 111 + }, + "id": 93, + "panels": [], + "title": "Sessions", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 112 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "haproxy_backend_current_sessions{backend=~\"$backend\",instance=~\"$host\"}", + "hide": false, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ backend }}", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Back - Current number of active sessions", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 112 + }, + "hiddenSeries": false, + "id": 45, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "haproxy_frontend_current_sessions{frontend=~\"$frontend\",instance=~\"$host\"}", + "hide": false, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ frontend }}", + "metric": "haproxy_backend_current_queue", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Front - Current number of active sessions", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 122 + }, + "hiddenSeries": false, + "id": 29, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "haproxy_backend_current_session_rate{backend=~\"$backend\",instance=~\"$host\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ backend }}", + "metric": "haproxy_backend_current_queue", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Back - Current number of sessions rate per second over last elapsed second", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 122 + }, + "hiddenSeries": false, + "id": 44, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "haproxy_frontend_current_session_rate{frontend=~\"$frontend\",instance=~\"$host\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ frontend }}", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Front - Current number of sessions rate per second over last elapsed second", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 132 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 4, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*limit.*/", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "haproxy_backend_max_sessions{backend=~\"$backend\",instance=~\"$host\"}", + "hide": false, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ backend }}", + "metric": "", + "refId": "A", + "step": 30 + }, + { + "expr": "haproxy_backend_limit_sessions{backend=~\"$backend\",instance=~\"$host\"}", + "intervalFactor": 2, + "legendFormat": "{{ backend }} limit", + "refId": "B", + "step": 120 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Back - Maximum observed number of active sessions and limit", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 132 + }, + "hiddenSeries": false, + "id": 51, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 4, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*limit.*/", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "haproxy_frontend_max_sessions{frontend=~\"$frontend\",instance=~\"$host\"}", + "hide": false, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ frontend }}", + "metric": "", + "refId": "A", + "step": 30 + }, + { + "expr": "haproxy_frontend_limit_sessions{frontend=~\"$frontend\",instance=~\"$host\"}", + "intervalFactor": 2, + "legendFormat": "{{ frontend }} limit", + "refId": "B", + "step": 120 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Front - Maximum observed number of active sessions and limit", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 142 + }, + "hiddenSeries": false, + "id": 33, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "haproxy_backend_max_session_rate{backend=~\"$backend\",instance=~\"$host\"}", + "hide": false, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ backend }}", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Back - Maximum number of sessions rate per second", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 142 + }, + "hiddenSeries": false, + "id": 69, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 3, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*limit.*/", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "haproxy_frontend_max_session_rate{frontend=~\"$frontend\",instance=~\"$host\"}", + "hide": false, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ frontend }}", + "metric": "", + "refId": "A", + "step": 30 + }, + { + "expr": "haproxy_frontend_limit_session_rate{frontend=~\"$frontend\",instance=~\"$host\"}", + "intervalFactor": 2, + "legendFormat": "{{ frontend }} limit", + "refId": "B", + "step": 120 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Front - Maximum number of sessions rate per second and limit", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 152 + }, + "id": 94, + "panels": [], + "title": "Health and Weight", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 0, + "description": "", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 153 + }, + "hiddenSeries": false, + "id": 38, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "haproxy_backend_up{backend=~\"$backend\", instance=~\"$host\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ backend }}", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Back - Current health status of the backend", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 153 + }, + "hiddenSeries": false, + "id": 39, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "haproxy_backend_weight{backend=~\"$backend\", instance=~\"$host\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ backend }}", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Back - Total weight of the servers in the backend / Active servers backend", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 161 + }, + "id": 96, + "panels": [], + "title": "By Server Connections", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 162 + }, + "hiddenSeries": false, + "id": 58, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "irate(haproxy_server_sessions_total[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ backend }} / {{ server }}", + "metric": "haproxy_backe", + "refId": "B", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Total number of connections", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 173 + }, + "id": 97, + "panels": [], + "title": "By Server Queue", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 174 + }, + "hiddenSeries": false, + "id": 65, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(haproxy_server_max_queue{backend=~\"$backend\",server=~\"$server\",instance=~\"$host\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ backend }} / {{ server }}", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Maximum observed number of queued requests assigned to this server", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 174 + }, + "hiddenSeries": false, + "id": 59, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(haproxy_server_current_queue{backend=~\"$backend\",server=~\"$server\",instance=~\"$host\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ backend }} / {{ server }}", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Current number of queued requests assigned to this server", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 184 + }, + "id": 98, + "panels": [], + "title": "By Server Requests / Responses", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 185 + }, + "hiddenSeries": false, + "id": 74, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Error.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(haproxy_server_http_responses_total{backend=~\"$backend\",server=~\"$server\",instance=~\"$host\"}[5m])) by (server)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ backend }} {{ server }}", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Total of HTTP responses OK", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 185 + }, + "hiddenSeries": false, + "id": 71, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Error.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(haproxy_server_retry_warnings_total{backend=~\"$backend\", instance=~\"$host\"}[5m])", + "hide": false, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Retry {{ backend }} {{ server }}", + "metric": "", + "refId": "A", + "step": 30 + }, + { + "expr": "haproxy_server_redispatch_warnings_total{backend=~\"$backend\",server=~\"$server\",instance=~\"$host\"}", + "hide": false, + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "Redispatch {{ backend }} {{ server }}", + "refId": "B", + "step": 60 + }, + { + "expr": "irate(haproxy_server_response_errors_total{backend=~\"$backend\",server=~\"$server\",instance=~\"$host\"}[5m])", + "hide": false, + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "Error {{ backend }} {{ server }}", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Total of retry / error / redispatch warnings", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 197 + }, + "id": 99, + "panels": [], + "title": "By Server Responses by HTTP code", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 198 + }, + "height": "400px", + "hiddenSeries": false, + "id": 64, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(haproxy_server_http_responses_total{backend=~\"$backend\",server=~\"$server\",code=~\"$code\",instance=~\"$host\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ backend }} {{ server }} {{ code }}", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Total of HTTP responses", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 208 + }, + "id": 100, + "panels": [], + "title": "By Server Sessions", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 209 + }, + "hiddenSeries": false, + "id": 61, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "haproxy_server_current_sessions{backend=~\"$backend\",instance=~\"$host\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ backend }} / {{ server }}", + "metric": "haproxy_backend_current_queue", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Current number of active sessions", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 209 + }, + "hiddenSeries": false, + "id": 60, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "haproxy_server_current_session_rate{backend=~\"$backend\",instance=~\"$host\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ backend }} / {{ server }}", + "metric": "haproxy_backend_current_queue", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Current number of sessions rate per second over last elapsed second", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 218 + }, + "hiddenSeries": false, + "id": 66, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "haproxy_server_max_session_rate{backend=~\"$backend\",server=~\"$server\",instance=~\"$host\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ backend }} / {{ server }}", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Maximum observed number of sessions rate per second", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 218 + }, + "hiddenSeries": false, + "id": 67, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "haproxy_server_max_sessions{backend=~\"$backend\",server=~\"$server\",instance=~\"$host\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ backend }} / {{ server }}", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Maximum observed number of active sessions", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 227 + }, + "id": 101, + "panels": [], + "title": "By Server Health Weight", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 228 + }, + "hiddenSeries": false, + "id": 62, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "haproxy_server_downtime_seconds_total{backend=~\"$backend\",instance=~\"$host\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ backend }} / {{ server }}", + "metric": "haproxy_backend_current_queue", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Total downtime in seconds", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 1, + "description": "", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 228 + }, + "hiddenSeries": false, + "id": 56, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(haproxy_server_check_failures_total{backend=~\"$backend\",server=~\"$server\",instance=~\"$host\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ backend }} / {{ server }}", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Total number of failed health checks", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 0, + "description": "", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 240 + }, + "hiddenSeries": false, + "id": 57, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(haproxy_server_connection_errors_total{backend=~\"$backend\",server=~\"$server\",instance=~\"$host\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ backend }} / {{ server }}", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Total of connection errors", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 1, + "description": "", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 240 + }, + "hiddenSeries": false, + "id": 55, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "haproxy_server_check_duration_milliseconds", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ backend }} / {{ server }}", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Previously run health check duration, in milliseconds", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 252 + }, + "hiddenSeries": false, + "id": 72, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "haproxy_server_up{backend=~\"$backend\",instance=~\"$host\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ backend }} / {{ server }}", + "metric": "haproxy_backend_current_queue", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Current health status of the server", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 252 + }, + "hiddenSeries": false, + "id": 73, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "haproxy_server_weight{backend=~\"$backend\",instance=~\"$host\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ backend }} / {{ server }}", + "metric": "haproxy_backend_current_queue", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Current weight of the server", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 264 + }, + "id": 102, + "panels": [], + "title": "Scrape info", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 265 + }, + "hiddenSeries": false, + "id": 40, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "sum by (instance) (haproxy_exporter_csv_parse_failures)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Exporter csv failures {{ instance }}", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Number of errors while parsing CSV", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 265 + }, + "hiddenSeries": false, + "id": 41, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "sum by (instance) (haproxy_exporter_total_scrapes)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "HAProxy scrapes - {{ instance}}", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current total HAProxy scrapes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": "", + "schemaVersion": 36, + "style": "dark", + "tags": [ + "haproxy", + "servers" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(haproxy_up{cluster='$cluster'}, instance)", + "hide": 0, + "includeAll": true, + "label": "Host", + "multi": false, + "name": "host", + "options": [], + "query": { + "query": "label_values(haproxy_up{cluster='$cluster'}, instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "/([^:]+).*/", + "skipUrlSync": false, + "sort": 1, + "type": "query", + "useTags": false + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Backend", + "multi": true, + "name": "backend", + "options": [], + "query": { + "query": "label_values(haproxy_backend_bytes_in_total{instance=~\"$host\"}, backend)", + "refId": "Prometheus-backend-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query", + "useTags": false + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Frontend", + "multi": true, + "name": "frontend", + "options": [], + "query": { + "query": "label_values(haproxy_frontend_bytes_in_total{instance=~\"$host\"}, frontend)", + "refId": "Prometheus-frontend-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Server", + "multi": true, + "name": "server", + "options": [], + "query": { + "query": "label_values(haproxy_server_bytes_in_total{instance=~\"$host\", backend=~\"$backend\"}, server)", + "refId": "Prometheus-server-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query", + "useTags": false + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "HTTP Code", + "multi": true, + "name": "code", + "options": [], + "query": { + "query": "label_values(haproxy_server_http_responses_total{instance=~\"$host\", backend=~\"$backend\", server=~\"$server\"}, code)", + "refId": "Prometheus-code-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query", + "useTags": false + }, + { + "auto": true, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": true, + "text": "auto", + "value": "$__auto_interval_interval" + }, + "hide": 0, + "label": "Interval", + "name": "interval", + "options": [ + { + "selected": true, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + } + ], + "query": "30s,1m,5m,1h,6h,1d", + "queryValue": "", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(haproxy_up, cluster)", + "hide": 0, + "includeAll": false, + "label": "Cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(haproxy_up, cluster)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "HAProxy", + "uid": "3eATHHJWl", + "version": 2, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/libvirt.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/libvirt.json new file mode 100644 index 000000000..44ae2853f --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/libvirt.json @@ -0,0 +1,1718 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 138, + "iteration": 1616777933810, + "links": [], + "panels": [ + { + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 18, + "panels": [], + "title": "CPU", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 8, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "irate(libvirt_domain_info_cpu_time_seconds_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}[5m])", + "legendFormat": "{{uuid}} {{ instance }}/{{ domain }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total CPU Usage", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(libvirt_domain_info_cpu_time_seconds_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}[5m])/libvirt_domain_info_virtual_cpus", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ uuid }} {{ instance }}/{{ domain }}", + "refId": "A" + }, + { + "expr": "", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Per-cpu average usage", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 8, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 21, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "libvirt_domain_info_memory_usage_bytes{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}", + "legendFormat": "{{uuid}} {{ instance }}/{{ domain }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "RAM usage", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "libvirt_domain_info_memory_usage_bytes{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"} / libvirt_domain_info_maximum_memory_bytes", + "legendFormat": "{{uuid}} {{ instance }}/{{ domain }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "RAM used vs allocated", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 19, + "panels": [], + "title": "Block devices", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 8, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "irate(libvirt_domain_block_stats_read_requests_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}[5m])", + "legendFormat": "{{ target_device }} {{ uuid }} ({{ instance }}/{{ domain }}:{{ target_device }})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Read requests", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "rps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 8, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "irate(libvirt_domain_block_stats_write_requests_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}[5m])", + "legendFormat": "{{ target_device }} {{uuid}} ({{ instance }}/{{ domain }}:{{ target_device }})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Write requests", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "wps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 8, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "irate(libvirt_domain_block_stats_read_bytes_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}[5m])", + "legendFormat": "{{ target_device }} {{uuid}} {{ instance }}/{{ domain }}:{{ target_device }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Read bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 8, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 23 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "irate(libvirt_domain_block_stats_write_bytes_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}[5m])", + "legendFormat": "{{ target_device }} {{uuid}} {{ instance }}/{{ domain }}:{{ target_device }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Write bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 20, + "panels": [], + "title": "Networking", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 8, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 31 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "irate(libvirt_domain_interface_stats_receive_packets_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}[5m]) ", + "legendFormat": "{{ target_device }} {{ uuid }} {{ instance }}/{{ domain }}:{{ target_device }} ({{ source_bridge }})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Packets received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "rps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 8, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 31 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "irate(libvirt_domain_interface_stats_transmit_packets_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}[5m]) ", + "legendFormat": "{{uuid}} {{ instance }}/{{ domain }}: {{ target_device }} ({{ source_bridge }})", + "refId": "A" + }, + { + "expr": "", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Packets transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "wps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 8, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 38 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "irate(libvirt_domain_interface_stats_receive_bytes_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}[5m]) ", + "legendFormat": "{{uuid }} {{ instance }}/{{ domain }}: {{ target_device }} ({{ source_bridge }})", + "refId": "A" + }, + { + "expr": "", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Receive bandwidth", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 8, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 38 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "irate(libvirt_domain_interface_stats_transmit_bytes_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}[5m]) ", + "legendFormat": "{{uuid }} {{ instance }}/{{ domain }}: {{ target_device }} ({{ source_bridge }})", + "refId": "A" + }, + { + "expr": "", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Transmit bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 8, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 45 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "irate(libvirt_domain_interface_stats_receive_packets_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}[5m]) ", + "legendFormat": "{{uuid }} {{ instance }}/{{ domain }}: {{ target_device }} ({{ source_bridge }})", + "refId": "A" + }, + { + "expr": "", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Receive packets", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 8, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 45 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "irate(libvirt_domain_interface_stats_transmit_packets_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}[5m]) ", + "legendFormat": "{{uuid }} {{ instance }}/{{ domain }}: {{ target_device }} ({{ source_bridge }})", + "refId": "A" + }, + { + "expr": "", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Transmit bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 8, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 52 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "libvirt_domain_interface_stats_receive_errors_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}", + "legendFormat": "{{ uuid }} {{ instance }}/{{ domain }}: {{ target_device }} ({{ source_bridge }})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Receive errors", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 8, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 52 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "irate(libvirt_domain_interface_stats_receive_drops_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}[5m])", + "instant": false, + "legendFormat": "{{uuid}} on {{ instance }} {{ target_device }} ({{ source_bridge }})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Receive drops", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 8, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 52 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "libvirt_domain_interface_stats_transmit_errors_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}", + "legendFormat": "{{uuid}} {{ instance }}/{{ domain }}: {{ target_device }} ({{ source_bridge }})", + "refId": "A" + }, + { + "expr": "", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Transmit errors", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 8, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 52 + }, + "hiddenSeries": false, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "libvirt_domain_interface_stats_transmit_drops_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}", + "legendFormat": "{{ uuid }} {{ instance }}/{{ domain }}: {{ target_device }} ({{ source_bridge }})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Transmit drops", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 22, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "label_values(libvirt_up{job=\"libvirt_exporter\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "Hypervisor", + "multi": true, + "name": "instance", + "options": [], + "query": "label_values(libvirt_up{job=\"libvirt_exporter\"}, instance)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "label_values(libvirt_domain_info_virtual_cpus{job=\"libvirt_exporter\"}, domain)", + "hide": 0, + "includeAll": true, + "label": "Domain", + "multi": true, + "name": "domain", + "options": [], + "query": "label_values(libvirt_domain_info_virtual_cpus{job=\"libvirt_exporter\"}, domain)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "label_values(libvirt_domain_interface_stats_receive_bytes_total{job=\"libvirt_exporter\"}, source_bridge)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "source_bridge", + "options": [], + "query": "label_values(libvirt_domain_interface_stats_receive_bytes_total{job=\"libvirt_exporter\"}, source_bridge)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Libvirt (KVM)", + "uid": "gh5hj2qWZ", + "version": 29 +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/memcached.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/memcached.json new file mode 100644 index 000000000..afae3d8f7 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/memcached.json @@ -0,0 +1,901 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Dashboard for memcached servers monitored by Prometheus", + "editable": true, + "gnetId": 37, + "graphTooltip": 0, + "id": 118, + "iteration": 1616694161401, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 1, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (memcached_commands_total{instance=~\"$node\", status=\"hit\"}) / sum (memcached_commands_total{instance=~\"$node\"}) ", + "hide": false, + "intervalFactor": 2, + "legendFormat": "hit ratio", + "refId": "A", + "step": 2, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "% Hit ratio", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "172.17.0.1:9150": "#0A437C" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (memcached_current_connections{instance=~\"$node\"}) by (instance)", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 2, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Connections", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "ratio": "#6ED0E0" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(memcached_commands_total{instance=~\"$node\", command=\"set\"}) / sum(memcached_commands_total{instance=~\"$node\", command=\"get\"})", + "intervalFactor": 2, + "legendFormat": "ratio {{ instance }}", + "refId": "A", + "step": 2, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Set / Get ratio", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 2, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (memcached_commands_total{instance=~\"$node\"}) by (command)", + "intervalFactor": 2, + "legendFormat": "{{command}}", + "refId": "A", + "step": 2, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Commands", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "evicts": "#890F02", + "memcached_items_evicted_total{instance=\"172.17.0.1:9150\",job=\"prometheus\"}": "#890F02", + "reclaims": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "reclaims", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(memcached_items_evicted_total{instance=~\"$node\"}[5m])", + "intervalFactor": 2, + "legendFormat": "evicts {{ instance }}", + "refId": "A", + "step": 2, + "target": "" + }, + { + "expr": "irate(memcached_items_reclaimed_total{instance=~\"$node\"}[5m])", + "intervalFactor": 2, + "legendFormat": "reclaims {{ instance }}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "evicts / reclaims", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 6, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "{instance=\"172.17.0.1:9150\",job=\"prometheus\"}", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(memcached_read_bytes_total{instance=~\"$node\"}[10m])", + "intervalFactor": 2, + "legendFormat": "read from network {{ instance }}", + "refId": "A", + "step": 2, + "target": "" + }, + { + "expr": "irate(memcached_written_bytes_total{instance=~\"$node\"}[10m])", + "intervalFactor": 2, + "legendFormat": "written to network {{ instance }}", + "refId": "B", + "step": 5, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Read / written bytes", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 14 + }, + "hiddenSeries": false, + "id": 7, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "memcached_current_bytes{instance=~\"$node\"} / memcached_limit_bytes{instance=~\"$node\"}", + "intervalFactor": 2, + "legendFormat": "% used memory used {{ instance }}", + "refId": "A", + "step": 2, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total memory usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 14 + }, + "hiddenSeries": false, + "id": 5, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "memcached_current_items{instance=~\"$node\"}", + "intervalFactor": 2, + "legendFormat": "items {{ instance }}", + "refId": "A", + "step": 2, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Items in cache", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 22, + "style": "dark", + "tags": [ + "prometheus", + "memcached" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "node", + "options": [], + "query": "label_values(memcached_up, instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Memcached", + "uid": "DjolHN1Zk", + "version": 5 +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_full.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_full.json new file mode 100644 index 000000000..e55cc1ce6 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_full.json @@ -0,0 +1,13568 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 1860, + "graphTooltip": 0, + "id": 83, + "iteration": 1658139283650, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 261, + "panels": [], + "title": "Quick CPU / Mem / Disk", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Busy state of all CPU cores together", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 85 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 20, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "(((count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))) - avg(sum by (mode)(irate(node_cpu_seconds_total{mode='idle',instance=\"$node\",job=\"$job\"}[5m])))) * 100) / count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 900 + } + ], + "title": "CPU Busy", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Busy state of all CPU cores together (5 min average)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 85 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 155, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "avg(node_load5{instance=\"$node\",job=\"$job\"}) / count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)) * 100", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "refId": "A", + "step": 900 + } + ], + "title": "Sys Load (5m avg)", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Busy state of all CPU cores together (15 min average)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 85 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 19, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "avg(node_load15{instance=\"$node\",job=\"$job\"}) / count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)) * 100", + "hide": false, + "intervalFactor": 1, + "refId": "A", + "step": 900 + } + ], + "title": "Sys Load (15m avg)", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Non available RAM memory", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 1 + }, + "hideTimeOverride": false, + "id": 16, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "((node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}) / (node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} )) * 100", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "refId": "A", + "step": 900 + }, + { + "expr": "100 - ((node_memory_MemAvailable_bytes{instance=\"$node\",job=\"$job\"} * 100) / node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "refId": "B", + "step": 900 + } + ], + "title": "RAM Used", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Used Swap", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 10 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 25 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 21, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "((node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"}) / (node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} )) * 100", + "intervalFactor": 1, + "refId": "A", + "step": 900 + } + ], + "title": "SWAP Used", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Used Root FS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 154, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "100 - ((node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"} * 100) / node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 900 + } + ], + "title": "Root FS Used", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of CPU cores", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 18, + "y": 1 + }, + "id": 14, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 900 + } + ], + "title": "CPU Cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "System uptime", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 20, + "y": 1 + }, + "hideTimeOverride": true, + "id": 15, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "node_time_seconds{instance=\"$node\",job=\"$job\"} - node_boot_time_seconds{instance=\"$node\",job=\"$job\"}", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 1800 + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total RootFS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 70 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 18, + "y": 3 + }, + "id": 23, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "refId": "A", + "step": 900 + } + ], + "title": "RootFS Total", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total RAM", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 20, + "y": 3 + }, + "id": 75, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}", + "intervalFactor": 1, + "refId": "A", + "step": 900 + } + ], + "title": "RAM Total", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total SWAP", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 22, + "y": 3 + }, + "id": 18, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}", + "intervalFactor": 1, + "refId": "A", + "step": 900 + } + ], + "title": "SWAP Total", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 263, + "panels": [], + "title": "Basic CPU / Mem / Net / Disk", + "type": "row" + }, + { + "aliasColors": { + "Busy": "#EAB839", + "Busy Iowait": "#890F02", + "Busy other": "#1F78C1", + "Idle": "#052B51", + "Idle - Waiting for something to happen": "#052B51", + "guest": "#9AC48A", + "idle": "#052B51", + "iowait": "#EAB839", + "irq": "#BF1B00", + "nice": "#C15C17", + "softirq": "#E24D42", + "steal": "#FCE2DE", + "system": "#508642", + "user": "#5195CE" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 2, + "description": "Basic CPU info", + "fieldConfig": { + "defaults": { + "unit": "percent" + }, + "overrides": [] + }, + "fill": 4, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 6 + }, + "hiddenSeries": false, + "id": 77, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": 250, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": true, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Busy Iowait", + "color": "#890F02" + }, + { + "alias": "Idle", + "color": "#7EB26D" + }, + { + "alias": "Busy System", + "color": "#EAB839" + }, + { + "alias": "Busy User", + "color": "#0A437C" + }, + { + "alias": "Busy Other", + "color": "#6D1F62" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "avg by (mode)(irate(node_cpu_seconds_total{job=\"$job\",instance=\"$node\", mode!='idle'}[5m])) * 100", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ mode }}", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode='idle',instance=~\"$node\",job=\"$job\"}[5m])) * 100", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ mode }}", + "range": true, + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU Basic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3860", + "format": "percent", + "label": "", + "logBase": 1, + "max": "100", + "min": "0", + "show": true + }, + { + "$$hashKey": "object:3861", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "SWAP Used": "#BF1B00", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap Used": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 2, + "description": "Basic memory usage", + "fill": 4, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 6 + }, + "hiddenSeries": false, + "id": 78, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "RAM Total", + "color": "#E0F9D7", + "fill": 0, + "stack": false + }, + { + "alias": "RAM Cache + Buffer", + "color": "#052B51" + }, + { + "alias": "RAM Free", + "color": "#7EB26D" + }, + { + "alias": "Avaliable", + "color": "#DEDAF7", + "fill": 0, + "stack": false + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "RAM Total", + "refId": "A", + "step": 240 + }, + { + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - (node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "RAM Used", + "refId": "B", + "step": 240 + }, + { + "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RAM Cache + Buffer", + "refId": "C", + "step": 240 + }, + { + "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RAM Free", + "refId": "D", + "step": 240 + }, + { + "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "SWAP Used", + "refId": "E", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Basic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:4042", + "format": "bytes", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:4043", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Recv_bytes_eth2": "#7EB26D", + "Recv_bytes_lo": "#0A50A1", + "Recv_drop_eth2": "#6ED0E0", + "Recv_drop_lo": "#E0F9D7", + "Recv_errs_eth2": "#BF1B00", + "Recv_errs_lo": "#CCA300", + "Trans_bytes_eth2": "#7EB26D", + "Trans_bytes_lo": "#0A50A1", + "Trans_drop_eth2": "#6ED0E0", + "Trans_drop_lo": "#E0F9D7", + "Trans_errs_eth2": "#BF1B00", + "Trans_errs_lo": "#CCA300", + "recv_bytes_lo": "#0A50A1", + "recv_drop_eth0": "#99440A", + "recv_drop_lo": "#967302", + "recv_errs_eth0": "#BF1B00", + "recv_errs_lo": "#890F02", + "trans_bytes_eth0": "#7EB26D", + "trans_bytes_lo": "#0A50A1", + "trans_drop_eth0": "#99440A", + "trans_drop_lo": "#967302", + "trans_errs_eth0": "#BF1B00", + "trans_errs_lo": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Basic network info per interface", + "fill": 4, + "fillGradient": 0, + "gridPos": { + "h": 13, + "w": 12, + "x": 0, + "y": 13 + }, + "hiddenSeries": false, + "id": 74, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*trans.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[5m])*8", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "recv {{device}}", + "refId": "A", + "step": 240 + }, + { + "expr": "irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[5m])*8", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "trans {{device}} ", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network Traffic Basic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bps", + "logBase": 1, + "show": true + }, + { + "format": "pps", + "label": "", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 3, + "description": "Disk space used of all filesystems mounted", + "fill": 4, + "fillGradient": 0, + "gridPos": { + "h": 13, + "w": 12, + "x": 12, + "y": 13 + }, + "height": "", + "hiddenSeries": false, + "id": 152, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100 - ((node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} * 100) / node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{mountpoint}}", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Disk Space Used Basic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "logBase": 1, + "max": "100", + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 265, + "panels": [], + "title": "CPU / Memory / Net / Disk", + "type": "row" + }, + { + "aliasColors": { + "Idle - Waiting for something to happen": "#052B51", + "guest": "#9AC48A", + "idle": "#052B51", + "iowait": "#EAB839", + "irq": "#BF1B00", + "nice": "#C15C17", + "softirq": "#E24D42", + "steal": "#FCE2DE", + "system": "#508642", + "user": "#5195CE" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 2, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 4, + "fillGradient": 0, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 27 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 250, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": true, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode=\"system\",instance=\"$node\",job=\"$job\"}[5m])) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "System - Processes executing in kernel mode", + "range": true, + "refId": "A", + "step": 20 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode='user',instance=\"$node\",job=\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "User - Normal processes executing in user mode", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode='nice',instance=\"$node\",job=\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Nice - Niced processes executing in user mode", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode='iowait',instance=\"$node\",job=\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Iowait - Waiting for I/O to complete", + "refId": "E", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode='irq',instance=\"$node\",job=\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Irq - Servicing interrupts", + "refId": "F", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode='softirq',instance=\"$node\",job=\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Softirq - Servicing softirqs", + "refId": "G", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode='steal',instance=\"$node\",job=\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Steal - Time spent in other operating systems when running in a virtualized environment", + "refId": "H", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode='guest',instance=\"$node\",job=\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Guest - Time spent running a virtual CPU for a guest operating system", + "refId": "I", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode='idle',instance=\"$node\",job=\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Idle - Waiting for something to happen", + "refId": "D", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:4998", + "format": "percent", + "label": "percentage", + "logBase": 1, + "max": "100", + "min": "0", + "show": true + }, + { + "$$hashKey": "object:4999", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap - Swap memory usage": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839", + "Unused - Free memory unassigned": "#052B51" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 2, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 4, + "fillGradient": 0, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 27 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Hardware Corrupted - *./", + "stack": false + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"} - node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Apps - Memory used by user-space applications", + "refId": "A", + "step": 240 + }, + { + "expr": "node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "PageTables - Memory used to map between virtual and physical memory addresses", + "refId": "B", + "step": 240 + }, + { + "expr": "node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "SwapCache - Memory that keeps track of pages that have been fetched from swap but not yet been modified", + "refId": "C", + "step": 240 + }, + { + "expr": "node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Slab - Memory used by the kernel to cache data structures for its own use (caches like inode, dentry, etc)", + "refId": "D", + "step": 240 + }, + { + "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Cache - Parked file data (file content) cache", + "refId": "E", + "step": 240 + }, + { + "expr": "node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Buffers - Block device (e.g. harddisk) cache", + "refId": "F", + "step": 240 + }, + { + "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Unused - Free memory unassigned", + "refId": "G", + "step": 240 + }, + { + "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Swap - Swap space used", + "refId": "H", + "step": 240 + }, + { + "expr": "node_memory_HardwareCorrupted_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working", + "refId": "I", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Stack", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:5069", + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:5070", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "receive_packets_eth0": "#7EB26D", + "receive_packets_lo": "#E24D42", + "transmit_packets_eth0": "#7EB26D", + "transmit_packets_lo": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 4, + "fillGradient": 0, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 39 + }, + "hiddenSeries": false, + "id": 84, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Trans.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[5m])*8", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Receive", + "refId": "A", + "step": 240 + }, + { + "expr": "irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[5m])*8", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Transmit", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bps", + "label": "bits out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 3, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 4, + "fillGradient": 0, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 39 + }, + "height": "", + "hiddenSeries": false, + "id": 156, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} - node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{mountpoint}}", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Disk Space Used", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 51 + }, + "hiddenSeries": false, + "id": 229, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Read.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "alias": "/.*sda2_.*/", + "color": "#BA43A9" + }, + { + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[5m])", + "intervalFactor": 4, + "legendFormat": "{{device}} - Reads completed", + "refId": "A", + "step": 480 + }, + { + "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{device}} - Writes completed", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Disk IOps", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "iops", + "label": "IO read (-) / write (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "io time": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 3, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 4, + "fillGradient": 0, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 51 + }, + "hiddenSeries": false, + "id": 42, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*read*./", + "transform": "negative-Y" + }, + { + "alias": "/.*sda.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde.*/", + "color": "#E24D42" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{device}} - Successfully read bytes", + "refId": "A", + "step": 240 + }, + { + "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{device}} - Successfully written bytes", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "I/O Usage Read / Write", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": false, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes read (-) / write (+)", + "logBase": 1, + "show": true + }, + { + "format": "ms", + "label": "", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "io time": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 3, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 4, + "fillGradient": 0, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 63 + }, + "hiddenSeries": false, + "id": 127, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"} [5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{device}} - Time spent doing I/Os", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "I/O Usage Times", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": false, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "time", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "s", + "label": "", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 75 + }, + "id": 266, + "panels": [], + "title": "Memory Meminfo", + "type": "row" + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 2, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 76 + }, + "hiddenSeries": false, + "id": 136, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_Inactive_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Inactive - Memory which has been less recently used. It is more eligible to be reclaimed for other purposes", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_Active_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Active - Memory that has been used more recently and usually not reclaimed unless absolutely necessary", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Active / Inactive", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 2, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 76 + }, + "hiddenSeries": false, + "id": 135, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Committed_AS - *./" + }, + { + "alias": "/.*CommitLimit - *./", + "color": "#BF1B00", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_Committed_AS_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Committed_AS - Amount of memory presently allocated on the system", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_CommitLimit_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "CommitLimit - Amount of memory currently available to be allocated on the system", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Commited", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 2, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 86 + }, + "hiddenSeries": false, + "id": 191, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_Inactive_file_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Inactive_file - File-backed memory on inactive LRU list", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_Inactive_anon_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Inactive_anon - Anonymous and swap cache on inactive LRU list, including tmpfs (shmem)", + "refId": "B", + "step": 4 + }, + { + "expr": "node_memory_Active_file_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Active_file - File-backed memory on active LRU list", + "refId": "C", + "step": 4 + }, + { + "expr": "node_memory_Active_anon_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Active_anon - Anonymous and swap cache on active least-recently-used (LRU) list, including tmpfs", + "refId": "D", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Active / Inactive Detail", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "bytes", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#052B51", + "Total RAM + Swap": "#052B51", + "Total Swap": "#614D93", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 2, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 86 + }, + "hiddenSeries": false, + "id": 130, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_Writeback_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writeback - Memory which is actively being written back to disk", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_WritebackTmp_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "WritebackTmp - Memory used by FUSE for temporary writeback buffers", + "refId": "B", + "step": 4 + }, + { + "expr": "node_memory_Dirty_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Dirty - Memory which is waiting to get written back to the disk", + "refId": "C", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Writeback and Dirty", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 2, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 96 + }, + "hiddenSeries": false, + "id": 138, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages", + "fill": 0 + }, + { + "alias": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_Mapped_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Mapped - Used memory in mapped pages files which have been mmaped, such as libraries", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_Shmem_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Shmem - Used shared memory (shared between several processes, thus including RAM disks)", + "refId": "B", + "step": 4 + }, + { + "expr": "node_memory_ShmemHugePages_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages", + "refId": "C", + "step": 4 + }, + { + "expr": "node_memory_ShmemPmdMapped_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "ShmemPmdMapped - Ammount of shared (shmem/tmpfs) memory backed by huge pages", + "refId": "D", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Shared and Mapped", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#052B51", + "Total RAM + Swap": "#052B51", + "Total Swap": "#614D93", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 2, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 96 + }, + "hiddenSeries": false, + "id": 131, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_SUnreclaim_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "SUnreclaim - Part of Slab, that cannot be reclaimed on memory pressure", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "SReclaimable - Part of Slab, that might be reclaimed, such as caches", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Slab", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#052B51", + "Total RAM + Swap": "#052B51", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 2, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 106 + }, + "hiddenSeries": false, + "id": 70, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_VmallocChunk_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "VmallocChunk - Largest contigious block of vmalloc area which is free", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_VmallocTotal_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "VmallocTotal - Total size of vmalloc memory area", + "refId": "B", + "step": 4 + }, + { + "expr": "node_memory_VmallocUsed_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "VmallocUsed - Amount of vmalloc area which is used", + "refId": "C", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Vmalloc", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 2, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 106 + }, + "hiddenSeries": false, + "id": 159, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_Bounce_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Bounce - Memory used for block device bounce buffers", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Bounce", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#052B51", + "Total RAM + Swap": "#052B51", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 2, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 116 + }, + "hiddenSeries": false, + "id": 129, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Inactive *./", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_AnonHugePages_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "AnonHugePages - Memory in anonymous huge pages", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_AnonPages_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "AnonPages - Memory in user pages not backed by files", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Anonymous", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 2, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 116 + }, + "hiddenSeries": false, + "id": 160, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_KernelStack_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "KernelStack - Kernel memory stack. This is not reclaimable", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_Percpu_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "PerCPU - Per CPU memory allocated dynamically by loadable modules", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Kernel / CPU", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#806EB7", + "Total RAM + Swap": "#806EB7", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 2, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 126 + }, + "hiddenSeries": false, + "id": 140, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_HugePages_Free{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "HugePages_Free - Huge pages in the pool that are not yet allocated", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_HugePages_Rsvd{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "HugePages_Rsvd - Huge pages for which a commitment to allocate from the pool has been made, but no allocation has yet been made", + "refId": "B", + "step": 4 + }, + { + "expr": "node_memory_HugePages_Surp{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "HugePages_Surp - Huge pages in the pool above the value in /proc/sys/vm/nr_hugepages", + "refId": "C", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory HugePages Counter", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "pages", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#806EB7", + "Total RAM + Swap": "#806EB7", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 2, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 126 + }, + "hiddenSeries": false, + "id": 71, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_HugePages_Total{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "HugePages - Total size of the pool of huge pages", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Hugepagesize - Huge Page size", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory HugePages Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#052B51", + "Total RAM + Swap": "#052B51", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 2, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 136 + }, + "hiddenSeries": false, + "id": 128, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_DirectMap1G_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "DirectMap1G - Amount of pages mapped as this size", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_DirectMap2M_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "DirectMap2M - Amount of pages mapped as this size", + "refId": "B", + "step": 4 + }, + { + "expr": "node_memory_DirectMap4k_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "DirectMap4K - Amount of pages mapped as this size", + "refId": "C", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory DirectMap", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 2, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 136 + }, + "hiddenSeries": false, + "id": 137, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_Unevictable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Unevictable - Amount of unevictable memory that can't be swapped out for a variety of reasons", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_Mlocked_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "MLocked - Size of pages locked to memory using the mlock() system call", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Unevictable and MLocked", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#052B51", + "Total RAM + Swap": "#052B51", + "Total Swap": "#614D93", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 2, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 146 + }, + "hiddenSeries": false, + "id": 132, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_NFS_Unstable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "NFS Unstable - Memory in NFS pages sent to the server, but not yet commited to the storage", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory NFS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 156 + }, + "id": 267, + "panels": [], + "title": "Memory Vmstat", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 157 + }, + "hiddenSeries": false, + "id": 176, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*out/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_pgpgin{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pagesin - Page in operations", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgpgout{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pagesout - Page out operations", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Pages In / Out", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "pages out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 157 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*out/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_pswpin{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pswpin - Pages swapped in", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pswpout{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pswpout - Pages swapped out", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Pages Swap In / Out", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "pages out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 2, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 167 + }, + "hiddenSeries": false, + "id": 175, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Pgfault - Page major and minor fault operations", + "fill": 0, + "stack": false + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgfault - Page major and minor fault operations", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgmajfault - Major page fault operations", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[5m]) - irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgminfault - Minor page fault operations", + "refId": "C", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Page Faults", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "faults", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#052B51", + "Total RAM + Swap": "#052B51", + "Total Swap": "#614D93", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 2, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 167 + }, + "hiddenSeries": false, + "id": 307, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_oom_kill{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "oom killer invocations ", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "OOM Killer", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 177 + }, + "id": 293, + "panels": [], + "title": "System Timesync", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 178 + }, + "hiddenSeries": false, + "id": 260, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Variation*./", + "color": "#890F02" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_timex_estimated_error_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Estimated error in seconds", + "refId": "A", + "step": 240 + }, + { + "expr": "node_timex_offset_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Time offset in between local system and reference clock", + "refId": "B", + "step": 240 + }, + { + "expr": "node_timex_maxerror_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Maximum error in seconds", + "refId": "C", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Time Syncronized Drift", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "seconds", + "logBase": 1, + "show": true + }, + { + "format": "short", + "label": "counter", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 178 + }, + "hiddenSeries": false, + "id": 291, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_timex_loop_time_constant{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Phase-locked loop time adjust", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Time PLL Adjust", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 188 + }, + "hiddenSeries": false, + "id": 168, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Variation*./", + "color": "#890F02" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_timex_sync_status{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Is clock synchronized to a reliable server (1 = yes, 0 = no)", + "refId": "A", + "step": 240 + }, + { + "expr": "node_timex_frequency_adjustment_ratio{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Local clock frequency adjustment", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Time Syncronized Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 188 + }, + "hiddenSeries": false, + "id": 294, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_timex_tick_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Seconds between clock ticks", + "refId": "A", + "step": 240 + }, + { + "expr": "node_timex_tai_offset_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "International Atomic Time (TAI) offset", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Time Misc", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "seconds", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 198 + }, + "id": 312, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 199 + }, + "hiddenSeries": false, + "id": 62, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_procs_blocked{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Processes blocked waiting for I/O to complete", + "refId": "A", + "step": 240 + }, + { + "expr": "node_procs_running{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Processes in runnable state", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Processes Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 199 + }, + "hiddenSeries": false, + "id": 315, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_processes_state{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ state }}", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Processes State", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 209 + }, + "hiddenSeries": false, + "id": 148, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_forks_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Processes forks second", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Processes Forks", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "forks / sec", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 209 + }, + "hiddenSeries": false, + "id": 149, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Max.*/", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[5m])", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Processes virtual memory size in bytes", + "refId": "A", + "step": 240 + }, + { + "expr": "process_resident_memory_max_bytes{instance=\"$node\",job=\"$job\"}", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Maximum amount of virtual memory available in bytes", + "refId": "B", + "step": 240 + }, + { + "expr": "irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[5m])", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Processes virtual memory size in bytes", + "refId": "C", + "step": 240 + }, + { + "expr": "irate(process_virtual_memory_max_bytes{instance=\"$node\",job=\"$job\"}[5m])", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Maximum amount of virtual memory available in bytes", + "refId": "D", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Processes Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 219 + }, + "hiddenSeries": false, + "id": 313, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "PIDs limit", + "color": "#F2495C", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_processes_pids{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Number of PIDs", + "refId": "A", + "step": 240 + }, + { + "expr": "node_processes_max_processes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "PIDs limit", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "PIDs Number and Limit", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 219 + }, + "hiddenSeries": false, + "id": 305, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*waiting.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "CPU {{ cpu }} - seconds spent running a process", + "refId": "A", + "step": 240 + }, + { + "expr": "irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "CPU {{ cpu }} - seconds spent by processing waiting for this CPU", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Process schdeule stats Running / Waiting", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "seconds", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 229 + }, + "hiddenSeries": false, + "id": 314, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Threads limit", + "color": "#F2495C", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_processes_threads{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Allocated threads", + "refId": "A", + "step": 240 + }, + { + "expr": "node_processes_max_threads{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Threads limit", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Threads Number and Limit", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "System Processes", + "type": "row" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 199 + }, + "id": 269, + "panels": [], + "title": "System Misc", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 200 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_context_switches_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Context switches", + "refId": "A", + "step": 240 + }, + { + "expr": "irate(node_intr_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Interrupts", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Context Switches / Interrupts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 200 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Load 1m", + "refId": "A", + "step": 480 + }, + { + "expr": "node_load5{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Load 5m", + "refId": "B", + "step": 480 + }, + { + "expr": "node_load15{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Load 15m", + "refId": "C", + "step": 480 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "System Load", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 210 + }, + "hiddenSeries": false, + "id": 259, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Critical*./", + "color": "#E24D42", + "fill": 0 + }, + { + "alias": "/.*Max*./", + "color": "#EF843C", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_interrupts_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ type }} - {{ info }}", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Interrupts Detail", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:4591", + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:4592", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 210 + }, + "hiddenSeries": false, + "id": 306, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_schedstat_timeslices_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "CPU {{ cpu }}", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Schedule timeslices executed by each cpu", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:5847", + "format": "short", + "label": "counter", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:5848", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 220 + }, + "hiddenSeries": false, + "id": 151, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_entropy_available_bits{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Entropy available to random number generators", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Entropy", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 220 + }, + "hiddenSeries": false, + "id": 308, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(process_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Time spent", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU time spent in user and system contexts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "seconds", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 230 + }, + "hiddenSeries": false, + "id": 64, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Max*./", + "color": "#890F02", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_max_fds{instance=\"$node\",job=\"$job\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Maximum open file descriptors", + "refId": "A", + "step": 240 + }, + { + "expr": "process_open_fds{instance=\"$node\",job=\"$job\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Open file descriptors", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "File Descriptors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 240 + }, + "id": 304, + "panels": [], + "title": "Hardware Misc", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 241 + }, + "hiddenSeries": false, + "id": 158, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Critical*./", + "color": "#E24D42", + "fill": 0 + }, + { + "alias": "/.*Max*./", + "color": "#EF843C", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_hwmon_temp_celsius{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ chip }} {{ sensor }} temp", + "refId": "A", + "step": 240 + }, + { + "expr": "node_hwmon_temp_crit_alarm_celsius{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ chip }} {{ sensor }} Critical Alarm", + "refId": "B", + "step": 240 + }, + { + "expr": "node_hwmon_temp_crit_celsius{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ chip }} {{ sensor }} Critical", + "refId": "C", + "step": 240 + }, + { + "expr": "node_hwmon_temp_crit_hyst_celsius{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ chip }} {{ sensor }} Critical Historical", + "refId": "D", + "step": 240 + }, + { + "expr": "node_hwmon_temp_max_celsius{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ chip }} {{ sensor }} Max", + "refId": "E", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Hardware temperature monitor", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "celsius", + "label": "temperature", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 241 + }, + "hiddenSeries": false, + "id": 300, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Max*./", + "color": "#EF843C", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_cooling_device_cur_state{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Current {{ name }} in {{ type }}", + "refId": "A", + "step": 240 + }, + { + "expr": "node_cooling_device_max_state{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Max {{ name }} in {{ type }}", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Throttle cooling device", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 251 + }, + "hiddenSeries": false, + "id": 302, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_power_supply_online{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ power_supply }} online", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Power supply", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 261 + }, + "id": 296, + "panels": [], + "title": "Systemd", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 262 + }, + "hiddenSeries": false, + "id": 297, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_systemd_socket_accepted_connections_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ name }} Connections", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Systemd Sockets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:4646", + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:4647", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 262 + }, + "hiddenSeries": false, + "id": 298, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Failed", + "color": "#F2495C" + }, + { + "alias": "Inactive", + "color": "#FF9830" + }, + { + "alias": "Active", + "color": "#73BF69" + }, + { + "alias": "Deactivating", + "color": "#FFCB7D" + }, + { + "alias": "Activating", + "color": "#C8F2C2" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"activating\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Activating", + "refId": "A", + "step": 240 + }, + { + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"active\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Active", + "refId": "B", + "step": 240 + }, + { + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"deactivating\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Deactivating", + "refId": "C", + "step": 240 + }, + { + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"failed\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Failed", + "refId": "D", + "step": 240 + }, + { + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"inactive\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Inactive", + "refId": "E", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Systemd Units State", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 272 + }, + "id": 270, + "panels": [], + "title": "Storage Disk", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 273 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Read.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "alias": "/.*sda2_.*/", + "color": "#BA43A9" + }, + { + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[5m])", + "intervalFactor": 4, + "legendFormat": "{{device}} - Reads completed", + "refId": "A", + "step": 8 + }, + { + "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{device}} - Writes completed", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Disk IOps Completed", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "iops", + "label": "IO read (-) / write (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 273 + }, + "hiddenSeries": false, + "id": 33, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Read.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "alias": "/.*sda2_.*/", + "color": "#BA43A9" + }, + { + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "{{device}} - Read bytes", + "refId": "A", + "step": 8 + }, + { + "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Written bytes", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Disk R/W Data", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": "bytes read (-) / write (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 3, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 283 + }, + "hiddenSeries": false, + "id": 37, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Read.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "alias": "/.*sda2_.*/", + "color": "#BA43A9" + }, + { + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_read_time_seconds_total{instance=\"$node\",job=\"$job\"}[5m])", + "hide": false, + "intervalFactor": 4, + "legendFormat": "{{device}} - Read time", + "refId": "A", + "step": 8 + }, + { + "expr": "irate(node_disk_write_time_seconds_total{instance=\"$node\",job=\"$job\"}[5m])", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{device}} - Write time", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Disk R/W Time", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "time. read (-) / write (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 283 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "alias": "/.*sda2_.*/", + "color": "#BA43A9" + }, + { + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_io_time_weighted_seconds_total{instance=\"$node\",job=\"$job\"}[5m])", + "intervalFactor": 4, + "legendFormat": "{{device}} - IO time weighted", + "refId": "A", + "step": 8 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Disk IOs Weighted", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "time", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 293 + }, + "hiddenSeries": false, + "id": 133, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Read.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "alias": "/.*sda2_.*/", + "color": "#BA43A9" + }, + { + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_reads_merged_total{instance=\"$node\",job=\"$job\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{device}} - Read merged", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_disk_writes_merged_total{instance=\"$node\",job=\"$job\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{device}} - Write merged", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Disk R/W Merged", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "iops", + "label": "I/Os", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 3, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 293 + }, + "hiddenSeries": false, + "id": 36, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "alias": "/.*sda2_.*/", + "color": "#BA43A9" + }, + { + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\"}[5m])", + "intervalFactor": 4, + "legendFormat": "{{device}} - IO time", + "refId": "A", + "step": 8 + }, + { + "expr": "irate(node_disk_discard_time_seconds_total{instance=\"$node\",job=\"$job\"}[5m])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}} - discard time", + "refId": "B", + "step": 8 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Time Spent Doing I/Os", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "time", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 303 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "alias": "/.*sda2_.*/", + "color": "#BA43A9" + }, + { + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_io_now{instance=\"$node\",job=\"$job\"}[5m])", + "intervalFactor": 4, + "legendFormat": "{{device}} - IO now", + "refId": "A", + "step": 8 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Disk IOs Current in Progress", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "iops", + "label": "I/Os", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 303 + }, + "hiddenSeries": false, + "id": 301, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "alias": "/.*sda2_.*/", + "color": "#BA43A9" + }, + { + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_discards_completed_total{instance=\"$node\",job=\"$job\"}[5m])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}} - Discards completed", + "refId": "A", + "step": 8 + }, + { + "expr": "irate(node_disk_discards_merged_total{instance=\"$node\",job=\"$job\"}[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}} - Discards merged", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Disk IOps Discards completed / merged", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "iops", + "label": "IOs", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 313 + }, + "id": 271, + "panels": [], + "title": "Storage Filesystem", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 3, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 314 + }, + "hiddenSeries": false, + "id": 43, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{mountpoint}} - Available", + "metric": "", + "refId": "A", + "step": 4 + }, + { + "expr": "node_filesystem_free_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{mountpoint}} - Free", + "refId": "B", + "step": 2 + }, + { + "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{mountpoint}} - Size", + "refId": "C", + "step": 2 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Filesystem space available", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 314 + }, + "hiddenSeries": false, + "id": 41, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_filesystem_files_free{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{mountpoint}} - Free file nodes", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "File Nodes Free", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "file nodes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 324 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_filefd_maximum{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Max open files", + "refId": "A", + "step": 8 + }, + { + "expr": "node_filefd_allocated{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Open files", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "File Descriptor", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "files", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 324 + }, + "hiddenSeries": false, + "id": 219, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_filesystem_files{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{mountpoint}} - File nodes total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "File Nodes Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "file Nodes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "/ ReadOnly": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 334 + }, + "hiddenSeries": false, + "id": 44, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_filesystem_readonly{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{mountpoint}} - ReadOnly", + "refId": "A", + "step": 4 + }, + { + "expr": "node_filesystem_device_error{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{mountpoint}} - Device error", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Filesystem in ReadOnly / Error", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 344 + }, + "id": 272, + "panels": [], + "title": "Network Traffic", + "type": "row" + }, + { + "aliasColors": { + "receive_packets_eth0": "#7EB26D", + "receive_packets_lo": "#E24D42", + "transmit_packets_eth0": "#7EB26D", + "transmit_packets_lo": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 345 + }, + "hiddenSeries": false, + "id": 60, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Trans.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_packets_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}} - Receive", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_network_transmit_packets_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}} - Transmit", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network Traffic by Packets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": "packets out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 345 + }, + "hiddenSeries": false, + "id": 142, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Trans.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_errs_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Receive errors", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_network_transmit_errs_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Rransmit errors", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network Traffic Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": "packets out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 355 + }, + "hiddenSeries": false, + "id": 143, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Trans.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_drop_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Receive drop", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_network_transmit_drop_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Transmit drop", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network Traffic Drop", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": "packets out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 355 + }, + "hiddenSeries": false, + "id": 141, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Trans.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_compressed_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Receive compressed", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_network_transmit_compressed_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Transmit compressed", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network Traffic Compressed", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": "packets out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 365 + }, + "hiddenSeries": false, + "id": 146, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Trans.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_multicast_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Receive multicast", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network Traffic Multicast", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": "packets out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 365 + }, + "hiddenSeries": false, + "id": 144, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Trans.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_fifo_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Receive fifo", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_network_transmit_fifo_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Transmit fifo", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network Traffic Fifo", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": "packets out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 375 + }, + "hiddenSeries": false, + "id": 145, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Trans.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_frame_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{device}} - Receive frame", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network Traffic Frame", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": "packets out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 375 + }, + "hiddenSeries": false, + "id": 231, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_transmit_carrier_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Statistic transmit_carrier", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network Traffic Carrier", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 385 + }, + "hiddenSeries": false, + "id": 232, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "irate(node_network_transmit_colls_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}}", + "range": true, + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network Traffic Collisions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:5619", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:5620", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 385 + }, + "hiddenSeries": false, + "id": 61, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "NF conntrack limit", + "color": "#890F02", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_nf_conntrack_entries{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "NF conntrack entries", + "refId": "A", + "step": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_nf_conntrack_entries_limit{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "NF conntrack limit", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "NF Contrack", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:5564", + "format": "short", + "label": "entries", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:5565", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 395 + }, + "id": 230, + "links": [], + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "center", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "node_arp_entries{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ device }}", + "range": true, + "refId": "A", + "step": 4 + } + ], + "title": "ARP Entries", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 395 + }, + "id": 288, + "links": [], + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "titleSize": 10 + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "node_network_mtu_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{ device }}", + "range": true, + "refId": "A", + "step": 4 + } + ], + "title": "Link MTU", + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "links": [], + "mappings": [ + { + "options": { + "1250000": { + "index": 0, + "text": "10MB" + }, + "12500000": { + "index": 1, + "text": "100MB" + }, + "125000000": { + "index": 2, + "text": "1G" + }, + "1250000000": { + "index": 3, + "text": "10G" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 405 + }, + "id": 280, + "links": [], + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max by (device)(node_network_speed_bytes{instance=\"$node\",job=\"$job\", device!~\"tap.*\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{ device }}", + "range": true, + "refId": "A", + "step": 4 + } + ], + "title": "Link Speed", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 405 + }, + "id": 289, + "links": [], + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "titleSize": 10 + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "node_network_transmit_queue_length{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ device }}", + "range": true, + "refId": "A", + "step": 4 + } + ], + "title": " Interface transmit queue length", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 415 + }, + "hiddenSeries": false, + "id": 290, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Dropped.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_softnet_processed_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "CPU {{cpu}} - Processed", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_softnet_dropped_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "CPU {{cpu}} - Dropped", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Softnet Packets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:5682", + "format": "short", + "label": "packetes drop (-) / process (+)", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:5683", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 415 + }, + "hiddenSeries": false, + "id": 310, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_softnet_times_squeezed_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "CPU {{cpu}} - Squeezed", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Softnet Out of Quota", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 425 + }, + "hiddenSeries": false, + "id": 309, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_network_up{operstate=\"up\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{interface}} - Operational state UP", + "refId": "A", + "step": 4 + }, + { + "expr": "node_network_carrier{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{device}} - Physical link state", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network Operational Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 435 + }, + "id": 273, + "panels": [], + "title": "Network Sockstat", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 436 + }, + "hiddenSeries": false, + "id": 63, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_sockstat_TCP_alloc{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "TCP_alloc - Allocated sockets", + "refId": "A", + "step": 240 + }, + { + "expr": "node_sockstat_TCP_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "TCP_inuse - Tcp sockets currently in use", + "refId": "B", + "step": 240 + }, + { + "expr": "node_sockstat_TCP_mem{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "TCP_mem - Used memory for tcp", + "refId": "C", + "step": 240 + }, + { + "expr": "node_sockstat_TCP_orphan{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "TCP_orphan - Orphan sockets", + "refId": "D", + "step": 240 + }, + { + "expr": "node_sockstat_TCP_tw{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "TCP_tw - Sockets wating close", + "refId": "E", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Sockstat TCP", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 436 + }, + "hiddenSeries": false, + "id": 124, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_sockstat_UDPLITE_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "UDPLITE_inuse - Udplite sockets currently in use", + "refId": "A", + "step": 240 + }, + { + "expr": "node_sockstat_UDP_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "UDP_inuse - Udp sockets currently in use", + "refId": "B", + "step": 240 + }, + { + "expr": "node_sockstat_UDP_mem{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "UDP_mem - Used memory for udp", + "refId": "C", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Sockstat UDP", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 446 + }, + "hiddenSeries": false, + "id": 126, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_sockstat_sockets_used{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Sockets_used - Sockets currently in use", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Sockstat Used", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "sockets", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 446 + }, + "hiddenSeries": false, + "id": 220, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_sockstat_TCP_mem_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "mem_bytes - TCP sockets in that state", + "refId": "A", + "step": 240 + }, + { + "expr": "node_sockstat_UDP_mem_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "mem_bytes - UDP sockets in that state", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Sockstat Memory Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 456 + }, + "hiddenSeries": false, + "id": 125, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_sockstat_FRAG_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "FRAG_inuse - Frag sockets currently in use", + "refId": "A", + "step": 240 + }, + { + "expr": "node_sockstat_FRAG_memory{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "FRAG_memory - Used memory for frag", + "refId": "B", + "step": 240 + }, + { + "expr": "node_sockstat_RAW_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "RAW_inuse - Raw sockets currently in use", + "refId": "C", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Sockstat FRAG / RAW", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 466 + }, + "id": 274, + "panels": [], + "title": "Network Netstat", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 467 + }, + "height": "", + "hiddenSeries": false, + "id": 221, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 12, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_IpExt_InOctets{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "InOctets - Received octets", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_IpExt_OutOctets{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "OutOctets - Sent octets", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Netstat IP In / Out Octets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "octects out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 467 + }, + "height": "", + "hiddenSeries": false, + "id": 81, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "irate(node_netstat_Ip_Forwarding{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Forwarding - IP forwarding", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Netstat IP Forwarding", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "datagrams", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 477 + }, + "height": "", + "hiddenSeries": false, + "id": 115, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 12, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Icmp_InMsgs{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "InMsgs - Messages which the entity received. Note that this counter includes all those counted by icmpInErrors", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_Icmp_OutMsgs{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "OutMsgs - Messages which this entity attempted to send. Note that this counter includes all those counted by icmpOutErrors", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "ICMP In / Out", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "messages out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 477 + }, + "height": "", + "hiddenSeries": false, + "id": 50, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 12, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Icmp_InErrors{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "InErrors - Messages which the entity received but determined as having ICMP-specific errors (bad ICMP checksums, bad length, etc.)", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "ICMP Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "messages out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 487 + }, + "height": "", + "hiddenSeries": false, + "id": 55, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 12, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*Snd.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Udp_InDatagrams{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "InDatagrams - Datagrams received", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_Udp_OutDatagrams{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "OutDatagrams - Datagrams sent", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "UDP In / Out", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "datagrams out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 487 + }, + "height": "", + "hiddenSeries": false, + "id": 109, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 12, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Udp_InErrors{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "InErrors - UDP Datagrams that could not be delivered to an application", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_Udp_NoPorts{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "NoPorts - UDP Datagrams received on a port with no listener", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_netstat_UdpLite_InErrors{instance=\"$node\",job=\"$job\"}[5m])", + "interval": "", + "legendFormat": "InErrors Lite - UDPLite Datagrams that could not be delivered to an application", + "refId": "C" + }, + { + "expr": "irate(node_netstat_Udp_RcvbufErrors{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "RcvbufErrors - UDP buffer errors received", + "refId": "D", + "step": 4 + }, + { + "expr": "irate(node_netstat_Udp_SndbufErrors{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "SndbufErrors - UDP buffer errors send", + "refId": "E", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "UDP Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "datagrams", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 497 + }, + "height": "", + "hiddenSeries": false, + "id": 299, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 12, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*Snd.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Tcp_InSegs{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "InSegs - Segments received, including those received in error. This count includes segments received on currently established connections", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_Tcp_OutSegs{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "OutSegs - Segments sent, including those on current connections but excluding those containing only retransmitted octets", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "TCP In / Out", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "datagrams out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 497 + }, + "height": "", + "hiddenSeries": false, + "id": 104, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 12, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_ListenOverflows{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "ListenOverflows - Times the listen queue of a socket overflowed", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_ListenDrops{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "ListenDrops - SYNs to LISTEN sockets ignored", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPSynRetrans{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "TCPSynRetrans - SYN-SYN/ACK retransmits to break down retransmissions in SYN, fast/timeout retransmits", + "refId": "C", + "step": 4 + }, + { + "expr": "irate(node_netstat_Tcp_RetransSegs{instance=\"$node\",job=\"$job\"}[5m])", + "interval": "", + "legendFormat": "RetransSegs - Segments retransmitted - that is, the number of TCP segments transmitted containing one or more previously transmitted octets", + "refId": "D" + }, + { + "expr": "irate(node_netstat_Tcp_InErrs{instance=\"$node\",job=\"$job\"}[5m])", + "interval": "", + "legendFormat": "InErrs - Segments received in error (e.g., bad TCP checksums)", + "refId": "E" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "TCP Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 507 + }, + "height": "", + "hiddenSeries": false, + "id": 85, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 12, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*MaxConn *./", + "color": "#890F02", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_netstat_Tcp_CurrEstab{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "CurrEstab - TCP connections for which the current state is either ESTABLISHED or CLOSE- WAIT", + "refId": "A", + "step": 4 + }, + { + "expr": "node_netstat_Tcp_MaxConn{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "MaxConn - Limit on the total number of TCP connections the entity can support (Dinamic is \"-1\")", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "TCP Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "connections", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 507 + }, + "height": "", + "hiddenSeries": false, + "id": 91, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 12, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Sent.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_SyncookiesFailed{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "SyncookiesFailed - Invalid SYN cookies received", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_SyncookiesRecv{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "SyncookiesRecv - SYN cookies received", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_SyncookiesSent{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "SyncookiesSent - SYN cookies sent", + "refId": "C", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "TCP SynCookie", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 517 + }, + "height": "", + "hiddenSeries": false, + "id": 82, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 12, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Tcp_ActiveOpens{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "ActiveOpens - TCP connections that have made a direct transition to the SYN-SENT state from the CLOSED state", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_Tcp_PassiveOpens{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "PassiveOpens - TCP connections that have made a direct transition to the SYN-RCVD state from the LISTEN state", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "TCP Direct Transition", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "connections", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 527 + }, + "id": 279, + "panels": [], + "title": "Node Exporter", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 528 + }, + "hiddenSeries": false, + "id": 40, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_scrape_collector_duration_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{collector}} - Scrape duration", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Node Exporter Scrape Time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "seconds", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 528 + }, + "hiddenSeries": false, + "id": 157, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*error.*/", + "color": "#F2495C", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_scrape_collector_success{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{collector}} - Scrape success", + "refId": "A", + "step": 4 + }, + { + "expr": "node_textfile_scrape_error{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{collector}} - Scrape textfile error (1 = true)", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Node Exporter Scrape", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": false, + "schemaVersion": 36, + "style": "dark", + "tags": [ + "linux" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "Prometheus", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "node", + "value": "node" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(node_cpu_seconds_total, job)", + "hide": 0, + "includeAll": false, + "label": "Job", + "multi": false, + "name": "job", + "options": [], + "query": { + "query": "label_values(node_cpu_seconds_total, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "kef1p-phycon0001", + "value": "kef1p-phycon0001" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(node_cpu_seconds_total{job=\"$job\"}, instance)", + "hide": 0, + "includeAll": false, + "label": "Host:", + "multi": false, + "name": "node", + "options": [], + "query": { + "query": "label_values(node_cpu_seconds_total{job=\"$job\"}, instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "[a-z]+|nvme[0-9]+n[0-9]+", + "value": "[a-z]+|nvme[0-9]+n[0-9]+" + }, + "hide": 2, + "includeAll": false, + "multi": false, + "name": "diskdevices", + "options": [ + { + "selected": true, + "text": "[a-z]+|nvme[0-9]+n[0-9]+", + "value": "[a-z]+|nvme[0-9]+n[0-9]+" + } + ], + "query": "[a-z]+|nvme[0-9]+n[0-9]+", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Node Exporter Full", + "uid": "node-exporter-full", + "version": 5, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_side_by_side.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_side_by_side.json new file mode 100644 index 000000000..00da9e539 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_side_by_side.json @@ -0,0 +1,5775 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Dashboard to view multiple servers", + "editable": true, + "gnetId": 405, + "graphTooltip": 0, + "id": 10, + "iteration": 1616777402260, + "links": [], + "panels": [ + { + "content": "", + "datasource": null, + "editable": true, + "error": false, + "gridPos": { + "h": 3, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 11, + "maxPerRow": 6, + "mode": "html", + "options": {}, + "repeat": "node", + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.1:9100", + "value": "10.225.1.1:9100" + } + }, + "style": {}, + "title": "$node", + "type": "text" + }, + { + "content": "", + "datasource": null, + "editable": true, + "error": false, + "gridPos": { + "h": 3, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 162, + "maxPerRow": 6, + "mode": "html", + "options": {}, + "repeat": null, + "repeatIteration": 1616777402260, + "repeatPanelId": 11, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.2:9100", + "value": "10.225.1.2:9100" + } + }, + "style": {}, + "title": "$node", + "type": "text" + }, + { + "content": "", + "datasource": null, + "editable": true, + "error": false, + "gridPos": { + "h": 3, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 163, + "maxPerRow": 6, + "mode": "html", + "options": {}, + "repeat": null, + "repeatIteration": 1616777402260, + "repeatPanelId": 11, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.3:9100", + "value": "10.225.1.3:9100" + } + }, + "style": {}, + "title": "$node", + "type": "text" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 0, + "y": 3 + }, + "id": 20, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "maxPerRow": 6, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "repeat": "node", + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.1:9100", + "value": "10.225.1.1:9100" + } + }, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(node_cpu_seconds_total{instance=~\"$node\", mode=\"system\"}) or count(node_cpu{instance=~\"$node\", mode=\"system\"})", + "instant": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 14400, + "target": "" + } + ], + "thresholds": "", + "title": "CPU Cores", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 8, + "y": 3 + }, + "id": 164, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "maxPerRow": 6, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "repeat": null, + "repeatIteration": 1616777402260, + "repeatPanelId": 20, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.2:9100", + "value": "10.225.1.2:9100" + } + }, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(node_cpu_seconds_total{instance=~\"$node\", mode=\"system\"}) or count(node_cpu{instance=~\"$node\", mode=\"system\"})", + "instant": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 14400, + "target": "" + } + ], + "thresholds": "", + "title": "CPU Cores", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 16, + "y": 3 + }, + "id": 165, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "maxPerRow": 6, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "repeat": null, + "repeatIteration": 1616777402260, + "repeatPanelId": 20, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.3:9100", + "value": "10.225.1.3:9100" + } + }, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(node_cpu_seconds_total{instance=~\"$node\", mode=\"system\"}) or count(node_cpu{instance=~\"$node\", mode=\"system\"})", + "instant": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 14400, + "target": "" + } + ], + "thresholds": "", + "title": "CPU Cores", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 3, + "editable": true, + "error": false, + "fill": 10, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 6 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": true, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.1:9100", + "value": "10.225.1.1:9100" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"system\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"system\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "system", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"user\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"user\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "user", + "refId": "B", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"nice\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"nice\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "nice", + "refId": "C", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"iowait\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"iowait\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "iowait", + "refId": "E", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"steal\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"steal\",instance=~'$node'}[5m]))", + "intervalFactor": 2, + "legendFormat": "steal", + "refId": "H", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"idle\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"idle\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "idle", + "refId": "D", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"irq\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"irq\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "irq", + "refId": "F", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"softirq\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"softirq\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "softirq", + "refId": "G", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"guest\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"guest\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "guest", + "refId": "I", + "step": 1200 + } + ], + "thresholds": [ + { + "colorMode": "custom", + "fill": true, + "fillColor": "rgba(216, 200, 27, 0.27)", + "op": "gt", + "value": 0, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "%", + "logBase": 1, + "max": 100, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 3, + "editable": true, + "error": false, + "fill": 10, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 6 + }, + "hiddenSeries": false, + "id": 166, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": true, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatIteration": 1616777402260, + "repeatPanelId": 7, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.2:9100", + "value": "10.225.1.2:9100" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"system\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"system\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "system", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"user\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"user\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "user", + "refId": "B", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"nice\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"nice\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "nice", + "refId": "C", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"iowait\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"iowait\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "iowait", + "refId": "E", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"steal\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"steal\",instance=~'$node'}[5m]))", + "intervalFactor": 2, + "legendFormat": "steal", + "refId": "H", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"idle\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"idle\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "idle", + "refId": "D", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"irq\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"irq\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "irq", + "refId": "F", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"softirq\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"softirq\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "softirq", + "refId": "G", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"guest\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"guest\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "guest", + "refId": "I", + "step": 1200 + } + ], + "thresholds": [ + { + "colorMode": "custom", + "fill": true, + "fillColor": "rgba(216, 200, 27, 0.27)", + "op": "gt", + "value": 0, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "%", + "logBase": 1, + "max": 100, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 3, + "editable": true, + "error": false, + "fill": 10, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 6 + }, + "hiddenSeries": false, + "id": 167, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": true, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatIteration": 1616777402260, + "repeatPanelId": 7, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.3:9100", + "value": "10.225.1.3:9100" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"system\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"system\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "system", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"user\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"user\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "user", + "refId": "B", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"nice\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"nice\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "nice", + "refId": "C", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"iowait\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"iowait\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "iowait", + "refId": "E", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"steal\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"steal\",instance=~'$node'}[5m]))", + "intervalFactor": 2, + "legendFormat": "steal", + "refId": "H", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"idle\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"idle\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "idle", + "refId": "D", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"irq\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"irq\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "irq", + "refId": "F", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"softirq\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"softirq\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "softirq", + "refId": "G", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"guest\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"guest\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "guest", + "refId": "I", + "step": 1200 + } + ], + "thresholds": [ + { + "colorMode": "custom", + "fill": true, + "fillColor": "rgba(216, 200, 27, 0.27)", + "op": "gt", + "value": 0, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "%", + "logBase": 1, + "max": 100, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Slab": "#E5A8E2", + "Swap": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 13 + }, + "hiddenSeries": false, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.1:9100", + "value": "10.225.1.1:9100" + } + }, + "seriesOverrides": [ + { + "alias": "/Apps|Buffers|Cached|Free|Slab|SwapCached|PageTables|VmallocUsed/", + "fill": 5, + "stack": true + }, + { + "alias": "Swap", + "fill": 5, + "stack": true + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "( node_memory_MemTotal_bytes{instance=~'$node'} - node_memory_MemFree_bytes{instance=~'$node'} - node_memory_Buffers_bytes{instance=~'$node'} - node_memory_Cached_bytes{instance=~'$node'} - node_memory_SwapCached_bytes{instance=~'$node'} - node_memory_Slab_bytes{instance=~'$node'} - node_memory_PageTables_bytes{instance=~'$node'} - node_memory_VmallocUsed_bytes{instance=~'$node'} ) or ( node_memory_MemTotal{instance=~'$node'} - node_memory_MemFree{instance=~'$node'} - node_memory_Buffers{instance=~'$node'} - node_memory_Cached{instance=~'$node'} - node_memory_SwapCached{instance=~'$node'} - node_memory_Slab{instance=~'$node'} - node_memory_PageTables{instance=~'$node'} - node_memory_VmallocUsed{instance=~'$node'} )", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Apps", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "expr": "node_memory_Buffers_bytes{instance=~'$node'} or node_memory_Buffers{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Buffers", + "refId": "B", + "step": 1200 + }, + { + "expr": "node_memory_Cached_bytes{instance=~'$node'} or node_memory_Cached{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Cached", + "refId": "D", + "step": 1200 + }, + { + "expr": "node_memory_MemFree_bytes{instance=~'$node'} or node_memory_MemFree{instance=~'$node'}", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Free", + "refId": "E", + "step": 1200 + }, + { + "expr": "node_memory_Slab_bytes{instance=~'$node'} or node_memory_Slab{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Slab", + "refId": "F", + "step": 1200 + }, + { + "expr": "node_memory_SwapCached_bytes{instance=~'$node'} or node_memory_SwapCached{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "SwapCached", + "refId": "G", + "step": 1200 + }, + { + "expr": "node_memory_PageTables_bytes{instance=~'$node'} or node_memory_PageTables{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "PageTables", + "refId": "H", + "step": 1200 + }, + { + "expr": "node_memory_VmallocUsed_bytes{instance=~'$node'} or node_memory_VmallocUsed{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "VmallocUsed", + "metric": "", + "refId": "I", + "step": 1200 + }, + { + "expr": "(node_memory_SwapTotal_bytes{instance=~'$node'} - node_memory_SwapFree{instance=~'$node'}) or (node_memory_SwapTotal{instance=~'$node'} - node_memory_SwapFree{instance=~'$node'})", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Swap", + "metric": "", + "refId": "C", + "step": 1200 + }, + { + "expr": "node_memory_Committed_AS_bytes{instance=~'$node'} or node_memory_Committed_AS{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Committed", + "metric": "", + "refId": "J", + "step": 1200 + }, + { + "expr": "node_memory_Mapped_bytes{instance=~'$node'} or node_memory_Mapped{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Mapped", + "refId": "K", + "step": 1200 + }, + { + "expr": "node_memory_Active_bytes{instance=~'$node'} or node_memory_Active{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Active", + "metric": "", + "refId": "L", + "step": 1200 + }, + { + "expr": "node_memory_Inactive_bytes{instance=~'$node'} or node_memory_Inactive{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Inactive", + "metric": "", + "refId": "M", + "step": 1200 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "GB", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Slab": "#E5A8E2", + "Swap": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 13 + }, + "hiddenSeries": false, + "id": 168, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatIteration": 1616777402260, + "repeatPanelId": 17, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.2:9100", + "value": "10.225.1.2:9100" + } + }, + "seriesOverrides": [ + { + "alias": "/Apps|Buffers|Cached|Free|Slab|SwapCached|PageTables|VmallocUsed/", + "fill": 5, + "stack": true + }, + { + "alias": "Swap", + "fill": 5, + "stack": true + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "( node_memory_MemTotal_bytes{instance=~'$node'} - node_memory_MemFree_bytes{instance=~'$node'} - node_memory_Buffers_bytes{instance=~'$node'} - node_memory_Cached_bytes{instance=~'$node'} - node_memory_SwapCached_bytes{instance=~'$node'} - node_memory_Slab_bytes{instance=~'$node'} - node_memory_PageTables_bytes{instance=~'$node'} - node_memory_VmallocUsed_bytes{instance=~'$node'} ) or ( node_memory_MemTotal{instance=~'$node'} - node_memory_MemFree{instance=~'$node'} - node_memory_Buffers{instance=~'$node'} - node_memory_Cached{instance=~'$node'} - node_memory_SwapCached{instance=~'$node'} - node_memory_Slab{instance=~'$node'} - node_memory_PageTables{instance=~'$node'} - node_memory_VmallocUsed{instance=~'$node'} )", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Apps", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "expr": "node_memory_Buffers_bytes{instance=~'$node'} or node_memory_Buffers{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Buffers", + "refId": "B", + "step": 1200 + }, + { + "expr": "node_memory_Cached_bytes{instance=~'$node'} or node_memory_Cached{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Cached", + "refId": "D", + "step": 1200 + }, + { + "expr": "node_memory_MemFree_bytes{instance=~'$node'} or node_memory_MemFree{instance=~'$node'}", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Free", + "refId": "E", + "step": 1200 + }, + { + "expr": "node_memory_Slab_bytes{instance=~'$node'} or node_memory_Slab{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Slab", + "refId": "F", + "step": 1200 + }, + { + "expr": "node_memory_SwapCached_bytes{instance=~'$node'} or node_memory_SwapCached{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "SwapCached", + "refId": "G", + "step": 1200 + }, + { + "expr": "node_memory_PageTables_bytes{instance=~'$node'} or node_memory_PageTables{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "PageTables", + "refId": "H", + "step": 1200 + }, + { + "expr": "node_memory_VmallocUsed_bytes{instance=~'$node'} or node_memory_VmallocUsed{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "VmallocUsed", + "metric": "", + "refId": "I", + "step": 1200 + }, + { + "expr": "(node_memory_SwapTotal_bytes{instance=~'$node'} - node_memory_SwapFree{instance=~'$node'}) or (node_memory_SwapTotal{instance=~'$node'} - node_memory_SwapFree{instance=~'$node'})", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Swap", + "metric": "", + "refId": "C", + "step": 1200 + }, + { + "expr": "node_memory_Committed_AS_bytes{instance=~'$node'} or node_memory_Committed_AS{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Committed", + "metric": "", + "refId": "J", + "step": 1200 + }, + { + "expr": "node_memory_Mapped_bytes{instance=~'$node'} or node_memory_Mapped{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Mapped", + "refId": "K", + "step": 1200 + }, + { + "expr": "node_memory_Active_bytes{instance=~'$node'} or node_memory_Active{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Active", + "metric": "", + "refId": "L", + "step": 1200 + }, + { + "expr": "node_memory_Inactive_bytes{instance=~'$node'} or node_memory_Inactive{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Inactive", + "metric": "", + "refId": "M", + "step": 1200 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "GB", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Slab": "#E5A8E2", + "Swap": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 13 + }, + "hiddenSeries": false, + "id": 169, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatIteration": 1616777402260, + "repeatPanelId": 17, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.3:9100", + "value": "10.225.1.3:9100" + } + }, + "seriesOverrides": [ + { + "alias": "/Apps|Buffers|Cached|Free|Slab|SwapCached|PageTables|VmallocUsed/", + "fill": 5, + "stack": true + }, + { + "alias": "Swap", + "fill": 5, + "stack": true + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "( node_memory_MemTotal_bytes{instance=~'$node'} - node_memory_MemFree_bytes{instance=~'$node'} - node_memory_Buffers_bytes{instance=~'$node'} - node_memory_Cached_bytes{instance=~'$node'} - node_memory_SwapCached_bytes{instance=~'$node'} - node_memory_Slab_bytes{instance=~'$node'} - node_memory_PageTables_bytes{instance=~'$node'} - node_memory_VmallocUsed_bytes{instance=~'$node'} ) or ( node_memory_MemTotal{instance=~'$node'} - node_memory_MemFree{instance=~'$node'} - node_memory_Buffers{instance=~'$node'} - node_memory_Cached{instance=~'$node'} - node_memory_SwapCached{instance=~'$node'} - node_memory_Slab{instance=~'$node'} - node_memory_PageTables{instance=~'$node'} - node_memory_VmallocUsed{instance=~'$node'} )", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Apps", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "expr": "node_memory_Buffers_bytes{instance=~'$node'} or node_memory_Buffers{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Buffers", + "refId": "B", + "step": 1200 + }, + { + "expr": "node_memory_Cached_bytes{instance=~'$node'} or node_memory_Cached{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Cached", + "refId": "D", + "step": 1200 + }, + { + "expr": "node_memory_MemFree_bytes{instance=~'$node'} or node_memory_MemFree{instance=~'$node'}", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Free", + "refId": "E", + "step": 1200 + }, + { + "expr": "node_memory_Slab_bytes{instance=~'$node'} or node_memory_Slab{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Slab", + "refId": "F", + "step": 1200 + }, + { + "expr": "node_memory_SwapCached_bytes{instance=~'$node'} or node_memory_SwapCached{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "SwapCached", + "refId": "G", + "step": 1200 + }, + { + "expr": "node_memory_PageTables_bytes{instance=~'$node'} or node_memory_PageTables{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "PageTables", + "refId": "H", + "step": 1200 + }, + { + "expr": "node_memory_VmallocUsed_bytes{instance=~'$node'} or node_memory_VmallocUsed{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "VmallocUsed", + "metric": "", + "refId": "I", + "step": 1200 + }, + { + "expr": "(node_memory_SwapTotal_bytes{instance=~'$node'} - node_memory_SwapFree{instance=~'$node'}) or (node_memory_SwapTotal{instance=~'$node'} - node_memory_SwapFree{instance=~'$node'})", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Swap", + "metric": "", + "refId": "C", + "step": 1200 + }, + { + "expr": "node_memory_Committed_AS_bytes{instance=~'$node'} or node_memory_Committed_AS{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Committed", + "metric": "", + "refId": "J", + "step": 1200 + }, + { + "expr": "node_memory_Mapped_bytes{instance=~'$node'} or node_memory_Mapped{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Mapped", + "refId": "K", + "step": 1200 + }, + { + "expr": "node_memory_Active_bytes{instance=~'$node'} or node_memory_Active{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Active", + "metric": "", + "refId": "L", + "step": 1200 + }, + { + "expr": "node_memory_Inactive_bytes{instance=~'$node'} or node_memory_Inactive{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Inactive", + "metric": "", + "refId": "M", + "step": 1200 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "GB", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 20 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.1:9100", + "value": "10.225.1.1:9100" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{instance=~\"$node\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "load", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 20 + }, + "hiddenSeries": false, + "id": 170, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatIteration": 1616777402260, + "repeatPanelId": 13, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.2:9100", + "value": "10.225.1.2:9100" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{instance=~\"$node\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "load", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 20 + }, + "hiddenSeries": false, + "id": 171, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatIteration": 1616777402260, + "repeatPanelId": 13, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.3:9100", + "value": "10.225.1.3:9100" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{instance=~\"$node\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "load", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 3, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 27 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.1:9100", + "value": "10.225.1.1:9100" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100.0 - 100 * (node_filesystem_avail_bytes{instance=~'$node',device !~'tmpfs',device!~'by-uuid'} / node_filesystem_size_bytes{instance=~'$node',device !~'tmpfs',device!~'by-uuid'}) or 100.0 - 100 * (node_filesystem_avail{instance=~'$node',device !~'tmpfs',device!~'by-uuid'} / node_filesystem_size{instance=~'$node',device !~'tmpfs',device!~'by-uuid'})", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{mountpoint}}", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Space Used", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "logBase": 1, + "max": 100, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 3, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 27 + }, + "hiddenSeries": false, + "id": 172, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatIteration": 1616777402260, + "repeatPanelId": 9, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.2:9100", + "value": "10.225.1.2:9100" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100.0 - 100 * (node_filesystem_avail_bytes{instance=~'$node',device !~'tmpfs',device!~'by-uuid'} / node_filesystem_size_bytes{instance=~'$node',device !~'tmpfs',device!~'by-uuid'}) or 100.0 - 100 * (node_filesystem_avail{instance=~'$node',device !~'tmpfs',device!~'by-uuid'} / node_filesystem_size{instance=~'$node',device !~'tmpfs',device!~'by-uuid'})", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{mountpoint}}", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Space Used", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "logBase": 1, + "max": 100, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 3, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 27 + }, + "hiddenSeries": false, + "id": 173, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatIteration": 1616777402260, + "repeatPanelId": 9, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.3:9100", + "value": "10.225.1.3:9100" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100.0 - 100 * (node_filesystem_avail_bytes{instance=~'$node',device !~'tmpfs',device!~'by-uuid'} / node_filesystem_size_bytes{instance=~'$node',device !~'tmpfs',device!~'by-uuid'}) or 100.0 - 100 * (node_filesystem_avail{instance=~'$node',device !~'tmpfs',device!~'by-uuid'} / node_filesystem_size{instance=~'$node',device !~'tmpfs',device!~'by-uuid'})", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{mountpoint}}", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Space Used", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "logBase": 1, + "max": 100, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 34 + }, + "hiddenSeries": false, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "repeatDirection": "h", + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.1:9100", + "value": "10.225.1.1:9100" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_io_time_seconds_total{instance=~\"$node\"}[5m])*100 or irate(node_disk_io_time_ms{instance=~\"$node\"}[5m])/10", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}}", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization per Device", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "logBase": 1, + "max": 100, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 34 + }, + "hiddenSeries": false, + "id": 174, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1616777402260, + "repeatPanelId": 19, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.2:9100", + "value": "10.225.1.2:9100" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_io_time_seconds_total{instance=~\"$node\"}[5m])*100 or irate(node_disk_io_time_ms{instance=~\"$node\"}[5m])/10", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}}", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization per Device", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "logBase": 1, + "max": 100, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 34 + }, + "hiddenSeries": false, + "id": 175, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1616777402260, + "repeatPanelId": 19, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.3:9100", + "value": "10.225.1.3:9100" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_io_time_seconds_total{instance=~\"$node\"}[5m])*100 or irate(node_disk_io_time_ms{instance=~\"$node\"}[5m])/10", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}}", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization per Device", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "logBase": 1, + "max": 100, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 41 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "repeatDirection": "h", + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.1:9100", + "value": "10.225.1.1:9100" + } + }, + "seriesOverrides": [ + { + "alias": "/.*_read$/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_reads_completed_total{instance=~'$node'}[5m]) or irate(node_disk_reads_completed{instance=~'$node'}[5m])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}}_read", + "metric": "", + "refId": "A", + "step": 2400, + "target": "" + }, + { + "expr": "irate(node_disk_writes_completed_total{instance=~'$node'}[5m]) or irate(node_disk_writes_completed{instance=~'$node'}[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}}_write", + "metric": "", + "refId": "B", + "step": 1200 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk IOs per Device", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "IO/second read (-) / write (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 41 + }, + "hiddenSeries": false, + "id": 176, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1616777402260, + "repeatPanelId": 14, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.2:9100", + "value": "10.225.1.2:9100" + } + }, + "seriesOverrides": [ + { + "alias": "/.*_read$/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_reads_completed_total{instance=~'$node'}[5m]) or irate(node_disk_reads_completed{instance=~'$node'}[5m])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}}_read", + "metric": "", + "refId": "A", + "step": 2400, + "target": "" + }, + { + "expr": "irate(node_disk_writes_completed_total{instance=~'$node'}[5m]) or irate(node_disk_writes_completed{instance=~'$node'}[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}}_write", + "metric": "", + "refId": "B", + "step": 1200 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk IOs per Device", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "IO/second read (-) / write (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 41 + }, + "hiddenSeries": false, + "id": 177, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1616777402260, + "repeatPanelId": 14, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.3:9100", + "value": "10.225.1.3:9100" + } + }, + "seriesOverrides": [ + { + "alias": "/.*_read$/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_reads_completed_total{instance=~'$node'}[5m]) or irate(node_disk_reads_completed{instance=~'$node'}[5m])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}}_read", + "metric": "", + "refId": "A", + "step": 2400, + "target": "" + }, + { + "expr": "irate(node_disk_writes_completed_total{instance=~'$node'}[5m]) or irate(node_disk_writes_completed{instance=~'$node'}[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}}_write", + "metric": "", + "refId": "B", + "step": 1200 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk IOs per Device", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "IO/second read (-) / write (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 48 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "repeatDirection": "h", + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.1:9100", + "value": "10.225.1.1:9100" + } + }, + "seriesOverrides": [ + { + "alias": "/.*_read/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_read_bytes_total{instance=~'$node'}[5m]) or irate(node_disk_sectors_read{instance=~'$node'}[5m]) * 512", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}}_read", + "refId": "B", + "step": 2400 + }, + { + "expr": "irate(node_disk_written_bytes_total{instance=~'$node'}[5m]) or irate(node_disk_sectors_written{instance=~'$node'}[5m]) * 512", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}}_write", + "metric": "", + "refId": "A", + "step": 2400, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Throughput per Device", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes/second read (-) / write (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 48 + }, + "hiddenSeries": false, + "id": 178, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1616777402260, + "repeatPanelId": 18, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.2:9100", + "value": "10.225.1.2:9100" + } + }, + "seriesOverrides": [ + { + "alias": "/.*_read/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_read_bytes_total{instance=~'$node'}[5m]) or irate(node_disk_sectors_read{instance=~'$node'}[5m]) * 512", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}}_read", + "refId": "B", + "step": 2400 + }, + { + "expr": "irate(node_disk_written_bytes_total{instance=~'$node'}[5m]) or irate(node_disk_sectors_written{instance=~'$node'}[5m]) * 512", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}}_write", + "metric": "", + "refId": "A", + "step": 2400, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Throughput per Device", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes/second read (-) / write (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 48 + }, + "hiddenSeries": false, + "id": 179, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1616777402260, + "repeatPanelId": 18, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.3:9100", + "value": "10.225.1.3:9100" + } + }, + "seriesOverrides": [ + { + "alias": "/.*_read/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_read_bytes_total{instance=~'$node'}[5m]) or irate(node_disk_sectors_read{instance=~'$node'}[5m]) * 512", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}}_read", + "refId": "B", + "step": 2400 + }, + { + "expr": "irate(node_disk_written_bytes_total{instance=~'$node'}[5m]) or irate(node_disk_sectors_written{instance=~'$node'}[5m]) * 512", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}}_write", + "metric": "", + "refId": "A", + "step": 2400, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Throughput per Device", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes/second read (-) / write (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 55 + }, + "hiddenSeries": false, + "id": 25, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "repeatDirection": "h", + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.1:9100", + "value": "10.225.1.1:9100" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_disk_io_now{instance=~\"$node\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}}", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Queue Length", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 55 + }, + "hiddenSeries": false, + "id": 180, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1616777402260, + "repeatPanelId": 25, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.2:9100", + "value": "10.225.1.2:9100" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_disk_io_now{instance=~\"$node\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}}", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Queue Length", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 55 + }, + "hiddenSeries": false, + "id": 181, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1616777402260, + "repeatPanelId": 25, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.3:9100", + "value": "10.225.1.3:9100" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_disk_io_now{instance=~\"$node\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}}", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Queue Length", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 62 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.1:9100", + "value": "10.225.1.1:9100" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_context_switches_total{instance=~\"$node\"}[5m]) or irate(node_context_switches{instance=~\"$node\"}[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "context switches", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Context Switches", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 62 + }, + "hiddenSeries": false, + "id": 182, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatIteration": 1616777402260, + "repeatPanelId": 22, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.2:9100", + "value": "10.225.1.2:9100" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_context_switches_total{instance=~\"$node\"}[5m]) or irate(node_context_switches{instance=~\"$node\"}[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "context switches", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Context Switches", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 62 + }, + "hiddenSeries": false, + "id": 183, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatIteration": 1616777402260, + "repeatPanelId": 22, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.3:9100", + "value": "10.225.1.3:9100" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_context_switches_total{instance=~\"$node\"}[5m]) or irate(node_context_switches{instance=~\"$node\"}[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "context switches", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Context Switches", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 69 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "repeatDirection": "h", + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.1:9100", + "value": "10.225.1.1:9100" + } + }, + "seriesOverrides": [ + { + "alias": "/.*_in/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_bytes_total{instance=~'$node'}[5m])*8 or irate(node_network_receive_bytes{instance=~'$node'}[5m])*8", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}}_in", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "expr": "irate(node_network_transmit_bytes_total{instance=~'$node'}[5m])*8 or irate(node_network_transmit_bytes{instance=~'$node'}[5m])*8", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}}_out", + "refId": "B", + "step": 1200 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bits", + "label": "bits in (-) / bits out (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 69 + }, + "hiddenSeries": false, + "id": 184, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1616777402260, + "repeatPanelId": 12, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.2:9100", + "value": "10.225.1.2:9100" + } + }, + "seriesOverrides": [ + { + "alias": "/.*_in/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_bytes_total{instance=~'$node'}[5m])*8 or irate(node_network_receive_bytes{instance=~'$node'}[5m])*8", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}}_in", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "expr": "irate(node_network_transmit_bytes_total{instance=~'$node'}[5m])*8 or irate(node_network_transmit_bytes{instance=~'$node'}[5m])*8", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}}_out", + "refId": "B", + "step": 1200 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bits", + "label": "bits in (-) / bits out (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 69 + }, + "hiddenSeries": false, + "id": 185, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1616777402260, + "repeatPanelId": 12, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.3:9100", + "value": "10.225.1.3:9100" + } + }, + "seriesOverrides": [ + { + "alias": "/.*_in/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_bytes_total{instance=~'$node'}[5m])*8 or irate(node_network_receive_bytes{instance=~'$node'}[5m])*8", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}}_in", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "expr": "irate(node_network_transmit_bytes_total{instance=~'$node'}[5m])*8 or irate(node_network_transmit_bytes{instance=~'$node'}[5m])*8", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}}_out", + "refId": "B", + "step": 1200 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bits", + "label": "bits in (-) / bits out (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 97 + }, + "hiddenSeries": false, + "id": 56, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "repeatDirection": "h", + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.1:9100", + "value": "10.225.1.1:9100" + } + }, + "seriesOverrides": [ + { + "alias": "/.*_in/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_drop_total{instance=~'$node'}[5m])", + "hide": false, + "interval": "", + "legendFormat": "dropped receive {{ device }}", + "refId": "C" + }, + { + "expr": "rate(node_network_transmit_total{instance=~'$node'}[5m]) * -1", + "hide": false, + "interval": "", + "legendFormat": "dropped transmit {{ device }}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network drops", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bits", + "label": "bits in (-) / bits out (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 97 + }, + "hiddenSeries": false, + "id": 192, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1616777402260, + "repeatPanelId": 56, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.2:9100", + "value": "10.225.1.2:9100" + } + }, + "seriesOverrides": [ + { + "alias": "/.*_in/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_drop_total{instance=~'$node'}[5m])", + "hide": false, + "interval": "", + "legendFormat": "dropped receive {{ device }}", + "refId": "C" + }, + { + "expr": "rate(node_network_transmit_total{instance=~'$node'}[5m]) * -1", + "hide": false, + "interval": "", + "legendFormat": "dropped transmit {{ device }}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network drops", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bits", + "label": "bits in (-) / bits out (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 97 + }, + "hiddenSeries": false, + "id": 193, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1616777402260, + "repeatPanelId": 56, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.3:9100", + "value": "10.225.1.3:9100" + } + }, + "seriesOverrides": [ + { + "alias": "/.*_in/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_drop_total{instance=~'$node'}[5m])", + "hide": false, + "interval": "", + "legendFormat": "dropped receive {{ device }}", + "refId": "C" + }, + { + "expr": "rate(node_network_transmit_total{instance=~'$node'}[5m]) * -1", + "hide": false, + "interval": "", + "legendFormat": "dropped transmit {{ device }}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network drops", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bits", + "label": "bits in (-) / bits out (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 104 + }, + "hiddenSeries": false, + "id": 21, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.1:9100", + "value": "10.225.1.1:9100" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_netstat_Tcp_CurrEstab{instance=~'$node'}", + "intervalFactor": 2, + "legendFormat": "established", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Netstat", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 104 + }, + "hiddenSeries": false, + "id": 194, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatIteration": 1616777402260, + "repeatPanelId": 21, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.2:9100", + "value": "10.225.1.2:9100" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_netstat_Tcp_CurrEstab{instance=~'$node'}", + "intervalFactor": 2, + "legendFormat": "established", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Netstat", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 104 + }, + "hiddenSeries": false, + "id": 195, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatIteration": 1616777402260, + "repeatPanelId": 21, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.3:9100", + "value": "10.225.1.3:9100" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_netstat_Tcp_CurrEstab{instance=~'$node'}", + "intervalFactor": 2, + "legendFormat": "established", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Netstat", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 111 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.1:9100", + "value": "10.225.1.1:9100" + } + }, + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "Udp_NoPorts", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Udp_InDatagrams{instance=~\"$node\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Udp_InDatagrams", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "expr": "irate(node_netstat_Udp_InErrors{instance=~\"$node\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Udp_InErrors", + "refId": "B", + "step": 1200 + }, + { + "expr": "irate(node_netstat_Udp_OutDatagrams{instance=~\"$node\"}[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Udp_OutDatagrams", + "refId": "C", + "step": 1200 + }, + { + "expr": "irate(node_netstat_Udp_NoPorts{instance=~\"$node\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Udp_NoPorts", + "refId": "D", + "step": 1200 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "UDP Stats", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 111 + }, + "hiddenSeries": false, + "id": 196, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatIteration": 1616777402260, + "repeatPanelId": 23, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.2:9100", + "value": "10.225.1.2:9100" + } + }, + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "Udp_NoPorts", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Udp_InDatagrams{instance=~\"$node\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Udp_InDatagrams", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "expr": "irate(node_netstat_Udp_InErrors{instance=~\"$node\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Udp_InErrors", + "refId": "B", + "step": 1200 + }, + { + "expr": "irate(node_netstat_Udp_OutDatagrams{instance=~\"$node\"}[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Udp_OutDatagrams", + "refId": "C", + "step": 1200 + }, + { + "expr": "irate(node_netstat_Udp_NoPorts{instance=~\"$node\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Udp_NoPorts", + "refId": "D", + "step": 1200 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "UDP Stats", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 111 + }, + "hiddenSeries": false, + "id": 197, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatIteration": 1616777402260, + "repeatPanelId": 23, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.3:9100", + "value": "10.225.1.3:9100" + } + }, + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "Udp_NoPorts", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Udp_InDatagrams{instance=~\"$node\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Udp_InDatagrams", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "expr": "irate(node_netstat_Udp_InErrors{instance=~\"$node\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Udp_InErrors", + "refId": "B", + "step": 1200 + }, + { + "expr": "irate(node_netstat_Udp_OutDatagrams{instance=~\"$node\"}[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Udp_OutDatagrams", + "refId": "C", + "step": 1200 + }, + { + "expr": "irate(node_netstat_Udp_NoPorts{instance=~\"$node\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Udp_NoPorts", + "refId": "D", + "step": 1200 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "UDP Stats", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 118 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.1:9100", + "value": "10.225.1.1:9100" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_nf_conntrack_entries_limit{instance=~\"$node\"} - node_nf_conntrack_entries{instance=~\"$node\"}", + "intervalFactor": 2, + "legendFormat": "free", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Conntrack", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 118 + }, + "hiddenSeries": false, + "id": 198, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatIteration": 1616777402260, + "repeatPanelId": 24, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.2:9100", + "value": "10.225.1.2:9100" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_nf_conntrack_entries_limit{instance=~\"$node\"} - node_nf_conntrack_entries{instance=~\"$node\"}", + "intervalFactor": 2, + "legendFormat": "free", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Conntrack", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 118 + }, + "hiddenSeries": false, + "id": 199, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatIteration": 1616777402260, + "repeatPanelId": 24, + "scopedVars": { + "node": { + "selected": true, + "text": "10.225.1.3:9100", + "value": "10.225.1.3:9100" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_nf_conntrack_entries_limit{instance=~\"$node\"} - node_nf_conntrack_entries{instance=~\"$node\"}", + "intervalFactor": 2, + "legendFormat": "free", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Conntrack", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 22, + "style": "dark", + "tags": [ + "prometheus" + ], + "templating": { + "list": [ + { + "allFormat": "glob", + "allValue": null, + "current": { + "text": "10.225.1.1:9100 + 10.225.1.2:9100 + 10.225.1.3:9100", + "value": [ + "10.225.1.1:9100", + "10.225.1.2:9100", + "10.225.1.3:9100" + ] + }, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "", + "multi": true, + "multiFormat": "regex values", + "name": "node", + "options": [], + "query": "label_values(node_exporter_build_info, instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Node Exporter Server Metrics Side by Side", + "uid": "snCzSH1Zk", + "version": 13 +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/openstack.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/openstack.json new file mode 100644 index 000000000..30ac8f54b --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/openstack.json @@ -0,0 +1,2604 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "OpenStack cloud overall status dashboard", + "editable": true, + "gnetId": 9701, + "graphTooltip": 0, + "id": 223, + "iteration": 1619766546105, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 36, + "panels": [], + "title": "Service status", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 14, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(openstack_nova_agent_state{adminState=\"enabled\"})-sum(openstack_nova_agent_state{adminState=\"enabled\"})", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "0.1,1", + "title": "Nova agents down", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "columns": [], + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 19, + "x": 3, + "y": 1 + }, + "id": 16, + "links": [], + "pageSize": 100, + "scroll": true, + "showHeader": true, + "sort": { + "col": 3, + "desc": false + }, + "styles": [ + { + "alias": "Status", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "#890f02", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "Value", + "thresholds": [ + "0.00", + "1.00" + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "job", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "instance", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "__name__", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "status", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "Time", + "thresholds": [], + "type": "date", + "unit": "short" + } + ], + "targets": [ + { + "expr": "openstack_nova_agent_state", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "refId": "A" + }, + { + "expr": "openstack_nova_service_state", + "format": "table", + "hide": true, + "instant": true, + "refId": "B" + } + ], + "title": "Nova agent status", + "transform": "table", + "type": "table-old" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#d44a3a", + "rgba(237, 129, 40, 0.89)", + "#299c46" + ], + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 5 + }, + "id": 26, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(openstack_nova_agent_state{adminState=\"enabled\"})", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "0,1", + "title": "Nova agents up", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 9 + }, + "id": 22, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(openstack_neutron_agent_state{adminState=\"up\"})-sum(openstack_neutron_agent_state{adminState=\"up\"})", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "0.1,1.0", + "title": "Neutron agents down", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "columns": [], + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 19, + "x": 3, + "y": 9 + }, + "id": 25, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 3, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "__name__", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "instance", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "job", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "status", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Status", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "#890f02", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "Value", + "thresholds": [ + "0.00", + "1.00" + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "adminState", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "openstack_neutron_agent_state", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Neutron agent status", + "transform": "table", + "type": "table-old" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#d44a3a", + "rgba(237, 129, 40, 0.89)", + "#299c46" + ], + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 13 + }, + "id": 28, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(openstack_neutron_agent_state{adminState=\"up\"})", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "0,1.0", + "title": "Neutron agents up", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 17 + }, + "id": 18, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(count(openstack_cinder_agent_state{adminState=\"enabled\"})-sum(openstack_cinder_agent_state{adminState=\"enabled\"}))", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "1", + "title": "Cinder agents down", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "columns": [], + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 19, + "x": 3, + "y": 17 + }, + "id": 20, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 9, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "__name__", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "instance", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "job", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "status", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Status", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "#890f02", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "Value", + "thresholds": [ + "0.00", + "1.00" + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "openstack_cinder_agent_state", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Cinder agent status", + "transform": "table", + "type": "table-old" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#d44a3a", + "rgba(237, 129, 40, 0.89)", + "#299c46" + ], + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 21 + }, + "id": 27, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(openstack_cinder_agent_state{adminState=\"enabled\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "0,1", + "title": "Cinder agent up", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 12, + "panels": [], + "title": "Resource Usage", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 26 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(openstack_nova_memory_used_bytes)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "memory in use", + "refId": "A" + }, + { + "expr": "sum(openstack_nova_memory_available_bytes)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "memory available", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Overall memory usage (TiB)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Memory in TiB", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "Time", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 26 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(openstack_nova_vcpus_used)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "used vcpu cores", + "refId": "A" + }, + { + "expr": "sum(openstack_nova_vcpus_available)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "total vcpu cores", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Overall CPU cores usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "time", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 35 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(openstack_nova_local_storage_used_bytes)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "local storage used", + "refId": "A" + }, + { + "expr": "sum(openstack_nova_local_storage_available_bytes)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "local storage available", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Local Storage (TB)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 35 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(openstack_neutron_floating_ips) ", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "floating ips", + "refId": "A" + }, + { + "expr": "avg(openstack_neutron_networks)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "networks", + "refId": "B" + }, + { + "expr": "avg(openstack_neutron_security_groups)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "security groups", + "refId": "C" + }, + { + "expr": "avg(openstack_neutron_subnets)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "subnets", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Neutron Stats", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 44 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(openstack)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "projects", + "refId": "A" + }, + { + "expr": "avg(openstack_identity_users)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "users", + "refId": "B" + }, + { + "expr": "avg(openstack_identity_groups)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "groups", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Keystone stats", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 44 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": true, + "current": true, + "hideEmpty": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 4, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(openstack_nova_running_vms)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "running vms", + "refId": "A" + }, + { + "expr": "sum(openstack_nova_total_vms)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "total vms", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Virtual Machines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": true, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 53 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(openstack_cinder_volumes)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "volumes", + "refId": "A" + }, + { + "expr": "avg(openstack_cinder_snapshots)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "snapshots", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cinder volumes/snapshots", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 53 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "avg": false, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(openstack_glance_images)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "images", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Glance Images", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.width", + "value": 500 + }, + { + "id": "displayName", + "value": "Instance" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #B" + }, + "properties": [ + { + "id": "custom.displayMode", + "value": "lcd-gauge" + }, + { + "id": "max", + "value": 0.5 + }, + { + "id": "displayName", + "value": "Connect Time" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 0.2 + }, + { + "color": "red", + "value": 0.4 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #A" + }, + "properties": [ + { + "id": "decimals", + "value": 2 + }, + { + "id": "displayName", + "value": "Certificate expires in" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-red", + "value": null + }, + { + "color": "semi-dark-yellow", + "value": 12096000 + }, + { + "color": "semi-dark-green", + "value": 25920000 + } + ] + } + }, + { + "id": "custom.displayMode", + "value": "color-background" + }, + { + "id": "custom.width", + "value": 220 + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "unit", + "value": "dtdurations" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #D" + }, + "properties": [ + { + "id": "displayName", + "value": "HTTP Response" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 300 + }, + { + "color": "red", + "value": 400 + } + ] + } + }, + { + "id": "custom.displayMode", + "value": "color-background" + }, + { + "id": "custom.align", + "value": "center" + }, + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #C" + }, + "properties": [ + { + "id": "displayName", + "value": "Transfer Time" + }, + { + "id": "max", + "value": 0.5 + }, + { + "id": "custom.displayMode", + "value": "lcd-gauge" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 0.125 + }, + { + "color": "red", + "value": 0.3 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #E" + }, + "properties": [ + { + "id": "displayName", + "value": "TLS Time" + }, + { + "id": "custom.displayMode", + "value": "lcd-gauge" + }, + { + "id": "max", + "value": 1 + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 0.5 + }, + { + "color": "red", + "value": 0.9 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #F" + }, + "properties": [ + { + "id": "displayName", + "value": "Processing Time" + }, + { + "id": "max", + "value": 0.5 + }, + { + "id": "custom.displayMode", + "value": "lcd-gauge" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 0.25 + }, + { + "color": "red", + "value": 0.4 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #G" + }, + "properties": [ + { + "id": "displayName", + "value": "Resolve Time" + }, + { + "id": "custom.displayMode", + "value": "lcd-gauge" + }, + { + "id": "max", + "value": 0.01 + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 0.005 + }, + { + "color": "red", + "value": 0.009 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Instance" + }, + "properties": [ + { + "id": "custom.width", + "value": 341 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "module" + }, + "properties": [ + { + "id": "custom.width", + "value": 109 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "service" + }, + "properties": [ + { + "id": "custom.width", + "value": 102 + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 38, + "options": { + "frameIndex": 1, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Certificate expires in" + } + ] + }, + "pluginVersion": "7.4.2", + "targets": [ + { + "expr": "probe_ssl_earliest_cert_expiry-time()", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "expr": "probe_http_status_code", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "D" + }, + { + "expr": "probe_http_duration_seconds{phase=\"resolve\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "G" + }, + { + "expr": "probe_http_duration_seconds{phase=\"connect\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + }, + { + "expr": "probe_http_duration_seconds{phase=\"tls\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "E" + }, + { + "expr": "probe_http_duration_seconds{phase=\"processing\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "F" + }, + { + "expr": "probe_http_duration_seconds{phase=\"transfer\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "C" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "OpenStack endpoint monitoring", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "instance" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "Time 6": true, + "Time 7": true, + "Value #D": false, + "__name__": true, + "__name__ 1": true, + "__name__ 2": true, + "__name__ 3": true, + "__name__ 4": true, + "__name__ 5": true, + "__name__ 6": true, + "instance": false, + "job": true, + "job 1": true, + "job 2": true, + "job 3": true, + "job 4": true, + "job 5": true, + "job 6": true, + "job 7": true, + "module 1": false, + "module 2": true, + "module 3": true, + "module 4": true, + "module 5": true, + "module 6": true, + "module 7": true, + "phase": true, + "phase 1": true, + "phase 2": true, + "phase 3": true, + "phase 4": true, + "phase 5": true, + "service 2": true, + "service 3": true, + "service 4": true, + "service 5": true, + "service 6": true, + "service 7": true + }, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "table" + } + ], + "refresh": false, + "schemaVersion": 27, + "style": "dark", + "tags": [ + "openstack", + "openstack-exporter" + ], + "templating": { + "list": [ + { + "auto": true, + "auto_count": 10, + "auto_min": "1m", + "current": { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + "description": null, + "error": null, + "hide": 0, + "label": "Interval", + "name": "interval", + "options": [ + { + "selected": true, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "OpenStack Dashboard", + "uid": "YZCsB1QmZ", + "version": 1 +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus.json new file mode 100644 index 000000000..1ff5724fd --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus.json @@ -0,0 +1,3432 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": "Prometheus", + "enable": true, + "expr": "sum(changes(prometheus_config_last_reload_success_timestamp_seconds{instance=~\"$instance\"}[10m])) by (instance)", + "hide": false, + "iconColor": "rgb(0, 96, 19)", + "limit": 100, + "name": "reloads", + "showIn": 0, + "step": "5m", + "type": "alert" + }, + { + "datasource": "Prometheus", + "enable": false, + "expr": "count(sum(up{instance=\"$instance\"}) by (instance) < 1)", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "down", + "showIn": 0, + "step": "5m", + "type": "alert" + } + ] + }, + "description": "Overview of metrics from Prometheus 2.0. \r\nUseful for using prometheus to monitor your prometheus.\r\nRevisions welcome!", + "editable": true, + "gnetId": 3662, + "graphTooltip": 0, + "id": 161, + "iteration": 1616773967282, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 34, + "panels": [], + "repeat": null, + "title": "at a glance", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "decimals": 3, + "description": "Percentage of uptime during the most recent $interval period. Change the period with the 'interval' dropdown above.", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "%", + "postfixFontSize": "100%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(avg_over_time(up{instance=~\"$instance\",job=~\"$job\"}[$interval]) * 100)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 40 + } + ], + "thresholds": "90, 99", + "title": "Uptime [$interval]", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "columns": [], + "datasource": "Prometheus", + "description": "Servers which are DOWN RIGHT NOW! \nFIX THEM!!", + "fontSize": "100%", + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 1 + }, + "hideTimeOverride": true, + "id": 25, + "links": [], + "options": {}, + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/__name__|job|Value/", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": " ", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(255, 0, 0, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(255, 0, 0, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "pattern": "instance", + "thresholds": [ + "", + "", + "" + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "up{instance=~\"$instance\",job=~\"$job\"} < 1", + "format": "table", + "intervalFactor": 2, + "refId": "A", + "step": 2 + } + ], + "timeFrom": "1s", + "title": "Currently Down", + "transform": "table", + "type": "table" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "description": "Total number of time series in prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 12, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "B", + "step": 40 + } + ], + "thresholds": "1000000,2000000", + "title": "Total Series", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 14, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "B", + "step": 40 + } + ], + "thresholds": "", + "title": "Memory Chunks", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 35, + "panels": [], + "repeat": null, + "title": "quick numbers", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "description": "The total number of rule group evaluations missed due to slow rule group evaluation.", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 9 + }, + "id": 16, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "irate(prometheus_evaluator_iterations_missed_total{job=~\"$job\",instance=~\"$instance\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 40 + } + ], + "thresholds": "1,10", + "title": "Missed Iterations [$interval]", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "description": "The total number of rule group evaluations skipped due to throttled metric storage.", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 9 + }, + "id": 18, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(sum_over_time(prometheus_evaluator_iterations_skipped_total{job=~\"$job\",instance=~\"$instance\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 40 + } + ], + "thresholds": "1,10", + "title": "Skipped Iterations [$interval]", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "description": "Total number of scrapes that hit the sample limit and were rejected.", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 8, + "y": 9 + }, + "id": 19, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(sum_over_time(prometheus_target_scrapes_exceeded_sample_limit_total{job=~\"$job\",instance=~\"$instance\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 40 + } + ], + "thresholds": "1,10", + "title": "Tardy Scrapes [$interval]", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "description": "Number of times the database failed to reload block data from disk.", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 9 + }, + "id": 13, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(sum_over_time(prometheus_tsdb_reloads_failures_total{job=~\"$job\",instance=~\"$instance\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 40 + } + ], + "thresholds": "1,10", + "title": "Reload Failures [$interval]", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "description": "Sum of all skipped scrapes", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 9 + }, + "id": 20, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "irate(prometheus_target_scrapes_exceeded_sample_limit_total{job=~\"$job\",instance=~\"$instance\"}[$interval]) + \nirate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job=~\"$job\",instance=~\"$instance\"}[$interval]) + \nirate(prometheus_target_scrapes_sample_out_of_bounds_total{job=~\"$job\",instance=~\"$instance\"}[$interval]) + \nirate(prometheus_target_scrapes_sample_out_of_order_total{job=~\"$job\",instance=~\"$instance\"}[$interval]) ", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 40 + } + ], + "thresholds": "1,10", + "title": "Skipped Scrapes [$interval]", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 36, + "panels": [], + "repeat": null, + "title": "errors", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "All non-zero failures and errors", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 17 + }, + "hiddenSeries": false, + "id": 33, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(net_conntrack_dialer_conn_failed_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Failed Connections", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(increase(prometheus_evaluator_iterations_missed_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Missed Iterations", + "refId": "B", + "step": 2 + }, + { + "expr": "sum(increase(prometheus_evaluator_iterations_skipped_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Skipped Iterations", + "refId": "C", + "step": 2 + }, + { + "expr": "sum(increase(prometheus_rule_evaluation_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Evaluation", + "refId": "D", + "step": 2 + }, + { + "expr": "sum(increase(prometheus_sd_azure_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Azure Refresh", + "refId": "E", + "step": 2 + }, + { + "expr": "sum(increase(prometheus_sd_consul_rpc_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Consul RPC", + "refId": "F", + "step": 2 + }, + { + "expr": "sum(increase(prometheus_sd_dns_lookup_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "DNS Lookup", + "refId": "G", + "step": 2 + }, + { + "expr": "sum(increase(prometheus_sd_ec2_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "EC2 Refresh", + "refId": "H", + "step": 2 + }, + { + "expr": "sum(increase(prometheus_sd_gce_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "GCE Refresh", + "refId": "I", + "step": 2 + }, + { + "expr": "sum(increase(prometheus_sd_marathon_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Marathon Refresh", + "refId": "J", + "step": 2 + }, + { + "expr": "sum(increase(prometheus_sd_openstack_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Openstack Refresh", + "refId": "K", + "step": 2 + }, + { + "expr": "sum(increase(prometheus_sd_triton_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Triton Refresh", + "refId": "L", + "step": 2 + }, + { + "expr": "sum(increase(prometheus_target_scrapes_exceeded_sample_limit_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Sample Limit", + "refId": "M", + "step": 2 + }, + { + "expr": "sum(increase(prometheus_target_scrapes_sample_duplicate_timestamp_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Duplicate Timestamp", + "refId": "N", + "step": 2 + }, + { + "expr": "sum(increase(prometheus_target_scrapes_sample_out_of_bounds_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Timestamp Out of Bounds", + "refId": "O", + "step": 2 + }, + { + "expr": "sum(increase(prometheus_target_scrapes_sample_out_of_order_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Sample Out of Order", + "refId": "P", + "step": 2 + }, + { + "expr": "sum(increase(prometheus_treecache_zookeeper_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Zookeeper", + "refId": "Q", + "step": 2 + }, + { + "expr": "sum(increase(prometheus_tsdb_compactions_failed_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TSDB Compactions", + "refId": "R", + "step": 2 + }, + { + "expr": "sum(increase(prometheus_tsdb_head_series_not_found{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Series Not Found", + "refId": "S", + "step": 2 + }, + { + "expr": "sum(increase(prometheus_tsdb_reloads_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reload", + "refId": "T", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Failures and Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Errors", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 37, + "panels": [], + "repeat": null, + "title": "up", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 25 + }, + "hiddenSeries": false, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "up{instance=~\"$instance\",job=~\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upness (stacked)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "none", + "label": "Up", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 25 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Storage Memory Chunks", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Chunks", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 38, + "panels": [], + "repeat": null, + "title": "series", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 33 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Series Count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Series", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 33 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "removed", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum( increase(prometheus_tsdb_head_series_created_total{instance=~\"$instance\"}[5m]) )", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "created", + "refId": "A", + "step": 4 + }, + { + "expr": "sum( increase(prometheus_tsdb_head_series_removed_total{instance=~\"$instance\"}[5m]) )", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "removed", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Series Created / Removed", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Series Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 39, + "panels": [], + "repeat": null, + "title": "appended samples", + "type": "row" + }, + { + "aliasColors": { + "10.58.3.10:80": "#BA43A9" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "Rate of total number of appended samples", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 41 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=~\"$job\",instance=~\"$instance\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Appended Samples per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Samples / Second", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 40, + "panels": [], + "repeat": null, + "title": "sync", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "Total number of syncs that were executed on a scrape pool.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 49 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(prometheus_target_scrape_pool_sync_total{job=~\"$job\",instance=~\"$instance\"}) by (scrape_job)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{scrape_job}}", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Scrape Sync Total", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Syncs", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "Actual interval to sync the scrape pool.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 49 + }, + "hiddenSeries": false, + "id": 21, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(prometheus_target_sync_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[$interval])) by (scrape_job) * 1000", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{scrape_job}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Target Sync", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Milliseconds", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 41, + "panels": [], + "repeat": null, + "title": "scrapes", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 57 + }, + "hiddenSeries": false, + "id": 29, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "scrape_duration_seconds{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Scrape Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Seconds", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "Total number of rejected scrapes", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 57 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(prometheus_target_scrapes_exceeded_sample_limit_total{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "exceeded sample limit", + "refId": "A", + "step": 4 + }, + { + "expr": "sum(prometheus_target_scrapes_sample_duplicate_timestamp_total{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "duplicate timestamp", + "refId": "B", + "step": 4 + }, + { + "expr": "sum(prometheus_target_scrapes_sample_out_of_bounds_total{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "out of bounds", + "refId": "C", + "step": 4 + }, + { + "expr": "sum(prometheus_target_scrapes_sample_out_of_order_total{job=~\"$job\",instance=~\"$instance\"}) ", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "out of order", + "refId": "D", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Rejected Scrapes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "Scrapes", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 42, + "panels": [], + "repeat": null, + "title": "durations", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "The duration of rule group evaluations", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 65 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1000 * rate(prometheus_evaluator_duration_seconds_sum{job=~\"$job\", instance=~\"$instance\"}[$interval]) / rate(prometheus_evaluator_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "E", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Average Rule Evaluation Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Milliseconds", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 65 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(http_request_duration_microseconds_count{job=~\"$job\",instance=~\"$instance\"}[$interval])) by (handler) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "HTTP Request Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Microseconds", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 72 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(prometheus_engine_query_duration_seconds_sum{job=~\"$job\",instance=~\"$instance\"}) by (slice)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{slice}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Prometheus Engine Query Duration Seconds", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Seconds", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "Rule-group evaluations \n - total\n - missed due to slow rule group evaluation\n - skipped due to throttled metric storage", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 72 + }, + "hiddenSeries": false, + "id": 31, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(prometheus_evaluator_iterations_total{job=~\"$job\", instance=~\"$instance\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Total", + "refId": "B", + "step": 4 + }, + { + "expr": "sum(rate(prometheus_evaluator_iterations_missed_total{job=~\"$job\", instance=~\"$instance\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Missed", + "refId": "A", + "step": 4 + }, + { + "expr": "sum(rate(prometheus_evaluator_iterations_skipped_total{job=~\"$job\", instance=~\"$instance\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Skipped", + "refId": "C", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Rule Evaluator Iterations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "iterations", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 79 + }, + "id": 43, + "panels": [], + "repeat": null, + "title": "notifications", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 80 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(prometheus_notifications_sent_total[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Notifications Sent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Notifications", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 87 + }, + "id": 44, + "panels": [], + "repeat": null, + "title": "config", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 88 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(time() - prometheus_config_last_reload_success_timestamp_seconds{job=~\"$job\",instance=~\"$instance\"}) / 60", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Minutes Since Successful Config Reload", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Minutes", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 88 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_config_last_reload_successful{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Successful Config Reload", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "Success", + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 95 + }, + "id": 45, + "panels": [], + "repeat": null, + "title": "garbage collection", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "GC invocation durations", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 96 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(go_gc_duration_seconds_sum{instance=~\"$instance\",job=~\"$job\"}[2m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GC Rate / 2m", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 103 + }, + "id": 46, + "panels": [], + "repeat": null, + "title": "Broken, ignore", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "This is probably wrong! Please help.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 104 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "allocated", + "stack": false + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(go_memstats_alloc_bytes_total{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "alloc_bytes_total", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(go_memstats_alloc_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "allocated", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(go_memstats_buck_hash_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "profiling bucket hash table", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(go_memstats_gc_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "GC metadata", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(go_memstats_heap_alloc_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "heap in-use", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(go_memstats_heap_idle_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "heap idle", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(go_memstats_heap_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "heap in use", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(go_memstats_heap_released_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "heap released", + "refId": "H", + "step": 10 + }, + { + "expr": "sum(go_memstats_heap_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "heap system", + "refId": "I", + "step": 10 + }, + { + "expr": "sum(go_memstats_mcache_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mcache in use", + "refId": "J", + "step": 10 + }, + { + "expr": "sum(go_memstats_mcache_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mcache sys", + "refId": "K", + "step": 10 + }, + { + "expr": "sum(go_memstats_mspan_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mspan in use", + "refId": "L", + "step": 10 + }, + { + "expr": "sum(go_memstats_mspan_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mspan sys", + "refId": "M", + "step": 10 + }, + { + "expr": "sum(go_memstats_next_gc_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "heap next gc", + "refId": "N", + "step": 10 + }, + { + "expr": "sum(go_memstats_other_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "other sys", + "refId": "O", + "step": 10 + }, + { + "expr": "sum(go_memstats_stack_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "stack in use", + "refId": "P", + "step": 10 + }, + { + "expr": "sum(go_memstats_stack_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "stack sys", + "refId": "Q", + "step": 10 + }, + { + "expr": "sum(go_memstats_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "sys", + "refId": "R", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Go Memory Usage (FIXME)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 104 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_target_interval_length_seconds{instance=~\"$instance\", job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{quantile}} {{interval}}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Scrape Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Seconds", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 104 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(prometheus_target_interval_length_seconds_count{job=~\"$job\",instance=~\"$instance\"}[5m])) by (interval)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{interval}}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Target Scrapes / 5m", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Scrapes", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "", + "schemaVersion": 22, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "job", + "options": [], + "query": "query_result(prometheus_tsdb_head_samples_appended_total)", + "refresh": 2, + "regex": "/.*job=\"([^\"]+)/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "instance", + "options": [], + "query": "query_result(up{job=~\"$job\"})", + "refresh": 2, + "regex": "/.*instance=\"([^\"]+).*/", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "5m", + "value": "5m" + }, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "3h", + "value": "3h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "2d", + "value": "2d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + }, + { + "selected": false, + "text": "90d", + "value": "90d" + }, + { + "selected": false, + "text": "180d", + "value": "180d" + } + ], + "query": "1h, 3h, 6h, 12h, 1d, 2d, 7d, 30d, 90d, 180d", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Prometheus 2.0 Overview", + "uid": "b7DPIZ6Zz", + "version": 4 +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus_alertmanager.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus_alertmanager.json new file mode 100644 index 000000000..510af9a72 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus_alertmanager.json @@ -0,0 +1,11257 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": "$datasource", + "enable": false, + "expr": "changes(process_start_time_seconds{ instance=~\"$instance\"}[2m]) > 0", + "hide": false, + "iconColor": "#bf1b00", + "name": "Restarts", + "showIn": 0, + "step": "1m", + "tagKeys": "instance", + "titleFormat": "Restart" + } + ] + }, + "description": "Dashboard showing Prometheus Alertmanager metrics for observing status of the cluster and possible debugging.", + "editable": true, + "gnetId": 9578, + "graphTooltip": 1, + "id": 106, + "iteration": 1616777581349, + "links": [ + { + "icon": "doc", + "tags": [], + "targetBlank": true, + "title": "Docs", + "tooltip": "Official documentation of Alertmanager", + "type": "link", + "url": "https://prometheus.io/docs/alerting/alertmanager/" + }, + { + "icon": "info", + "tags": [], + "targetBlank": true, + "title": "GitHub", + "tooltip": "Alertmanager sources on GitHub", + "type": "link", + "url": "https://github.com/prometheus/alertmanager" + }, + { + "icon": "info", + "tags": [], + "targetBlank": true, + "title": "Twitter", + "tooltip": "Twitter account with prometheus related info", + "type": "link", + "url": "https://twitter.com/PrometheusIO" + }, + { + "icon": "question", + "tags": [], + "targetBlank": true, + "title": "Mailing list", + "tooltip": "Prometheus users mailing list", + "type": "link", + "url": "https://groups.google.com/forum/#!forum/prometheus-users" + }, + { + "icon": "question", + "tags": [], + "targetBlank": true, + "title": "IRC", + "tooltip": "Join IRC using Riot", + "type": "link", + "url": "https://riot.im/app/#/room/#prometheus:matrix.org" + } + ], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 36, + "panels": [], + "title": "General info", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(244, 213, 152, 0.12)", + "full": false, + "lineColor": "#f4d598", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(alertmanager_build_info{instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Number of instances", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "columns": [], + "datasource": "$datasource", + "description": "Table containing list of Alertmanager instances showing its version, uptime, last reload time and if it was successful.", + "fontSize": "90%", + "gridPos": { + "h": 5, + "w": 9, + "x": 3, + "y": 1 + }, + "id": 26, + "links": [], + "options": {}, + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 13, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Instance", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "instance", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Version", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "version", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Uptime", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "s" + }, + { + "alias": "Last reload", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "s" + }, + { + "alias": "Last reload successful", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #C", + "thresholds": [ + "0", + "1" + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "/.*/", + "thresholds": [], + "type": "hidden", + "unit": "short" + } + ], + "targets": [ + { + "expr": "time() - (alertmanager_build_info{instance=~\"$instance\"} * on (instance, cluster) group_left process_start_time_seconds{instance=~\"$instance\", job=\"alertmanager\"})", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "A" + }, + { + "expr": "time() - alertmanager_config_last_reload_success_timestamp_seconds{instance=~\"$instance\"}", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "B" + }, + { + "expr": "alertmanager_config_last_reload_successful{instance=~\"$instance\"}", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "C" + } + ], + "title": "Instance versions and uptime", + "transform": "table", + "type": "table" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "description": "Number of peers in the Alertmanager cluster.", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 207, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(191, 27, 0, 0.08)", + "full": false, + "lineColor": "#e5ac0e", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(alertmanager_cluster_members{instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Cluster size", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "description": "Current number of active alerts.", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(191, 27, 0, 0.08)", + "full": false, + "lineColor": "#bf1b00", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(alertmanager_alerts{state=\"active\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Number of active alerts", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "description": "Current number of suppressed alerts.", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 3, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(249, 226, 210, 0.18)", + "full": false, + "lineColor": "#f9e2d2", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(alertmanager_alerts{state=\"suppressed\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Number of suppressed alerts", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "description": "Current number of active silences.", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 121, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(249, 226, 210, 0.18)", + "full": false, + "lineColor": "#f9e2d2", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(alertmanager_silences{state=\"active\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Number of active silences", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 113, + "panels": [], + "title": "Notifications", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Number of sent notifications to distinct integrations such as PagerDuty, Slack and so on. On negative axis are displayed failed notifications.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 118, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "total", + "sortDesc": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "10.225.1.1:9093", + "value": "10.225.1.1:9093" + } + }, + "seriesOverrides": [ + { + "alias": "/Failed.*/", + "color": "#99440a", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(alertmanager_notifications_total{instance=~\"$instance\"}[$__interval])) by (integration)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ integration}}", + "refId": "B" + }, + { + "expr": "sum(increase(alertmanager_notifications_failed_total{instance=~\"$instance\"}[$__interval])) by (integration)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Failed {{ integration }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Notifications sent from $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Duration of notification sends in 0.99 and 0.9 quantiles per integration.", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 115, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "10.225.1.1:9093", + "value": "10.225.1.1:9093" + } + }, + "seriesOverrides": [ + { + "alias": "/0.99.*/", + "linewidth": 1 + }, + { + "alias": "/0.5 .*/", + "linewidth": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(histogram_quantile(0.9,rate(alertmanager_notification_latency_seconds_bucket{instance=~\"$instance\"}[$__interval]))) by (integration)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "0.9q {{ integration }}", + "refId": "B" + }, + { + "expr": "sum(histogram_quantile(0.99,rate(alertmanager_notification_latency_seconds_bucket{instance=~\"$instance\"}[$__interval]))) by (integration)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "0.99q {{ integration }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Notification durations per integration on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 18, + "panels": [], + "title": "Alerts", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Number of alerts by state such as `active`, `suppressed` etc.", + "fill": 4, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 19 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "10.225.1.1:9093", + "value": "10.225.1.1:9093" + } + }, + "seriesOverrides": [ + { + "alias": "active", + "color": "#bf1b00" + }, + { + "alias": "suppressed", + "color": "#2f575e" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(alertmanager_alerts{instance=~\"$instance\"}) by (state)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{state}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active alerts in $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Number of received alerts from Prometheus by status `firing` on positive axis and `resolved` on negative axis.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "10.225.1.1:9093", + "value": "10.225.1.1:9093" + } + }, + "seriesOverrides": [ + { + "alias": "resolved", + "color": "#7eb26d", + "transform": "negative-Y" + }, + { + "alias": "firing", + "color": "#99440a" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(alertmanager_alerts_received_total{instance=~\"$instance\"}[$__interval])) by (status)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ status }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Received alerts by status for $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 34, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows cluster score representing cluster health. From Hashicorps official documentation: \n> This metric describes a node's perception of its own health based on how well it is meeting the soft real-time requirements of the protocol. This metric ranges from 0 to 8, where 0 indicates \"totally healthy\".\n\nFor more info see https://www.consul.io/docs/agent/telemetry.html#cluster-health", + "fill": 1, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 20 + }, + "id": 57, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-54b9fdf9bf-gqxs4", + "value": "alertmanager-54b9fdf9bf-gqxs4" + } + }, + "seriesOverrides": [ + { + "alias": "Number of failed peers", + "color": "#bf1b00", + "fill": 7 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_cluster_health_score{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Cluster health score", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Clusterhealth score for $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Shows cluster score representing cluster health. From Hashicorps official documentation: \n> This metric describes a node's perception of its own health based on how well it is meeting the soft real-time requirements of the protocol. This metric ranges from 0 to 8, where 0 indicates \"totally healthy\".\n\nFor more info see https://www.consul.io/docs/agent/telemetry.html#cluster-health", + "fill": 1, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 20 + }, + "id": 408, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 57, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-7f5c977766-kwr2s", + "value": "alertmanager-7f5c977766-kwr2s" + } + }, + "seriesOverrides": [ + { + "alias": "Number of failed peers", + "color": "#bf1b00", + "fill": 7 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_cluster_health_score{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Cluster health score", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Clusterhealth score for $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Shows cluster score representing cluster health. From Hashicorps official documentation: \n> This metric describes a node's perception of its own health based on how well it is meeting the soft real-time requirements of the protocol. This metric ranges from 0 to 8, where 0 indicates \"totally healthy\".\n\nFor more info see https://www.consul.io/docs/agent/telemetry.html#cluster-health", + "fill": 1, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 20 + }, + "id": 409, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 57, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-5b9858bf54-r8skx", + "value": "alertmanager-sre-5b9858bf54-r8skx" + } + }, + "seriesOverrides": [ + { + "alias": "Number of failed peers", + "color": "#bf1b00", + "fill": 7 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_cluster_health_score{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Cluster health score", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Clusterhealth score for $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Shows cluster score representing cluster health. From Hashicorps official documentation: \n> This metric describes a node's perception of its own health based on how well it is meeting the soft real-time requirements of the protocol. This metric ranges from 0 to 8, where 0 indicates \"totally healthy\".\n\nFor more info see https://www.consul.io/docs/agent/telemetry.html#cluster-health", + "fill": 1, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 20 + }, + "id": 410, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 57, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-8455c65f47-mmwlq", + "value": "alertmanager-sre-8455c65f47-mmwlq" + } + }, + "seriesOverrides": [ + { + "alias": "Number of failed peers", + "color": "#bf1b00", + "fill": 7 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_cluster_health_score{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Cluster health score", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Clusterhealth score for $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows gossip cluster members count in time and failing peers in case of any in red color.", + "fill": 1, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 24 + }, + "id": 38, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-54b9fdf9bf-gqxs4", + "value": "alertmanager-54b9fdf9bf-gqxs4" + } + }, + "seriesOverrides": [ + { + "alias": "Number of failed peers", + "color": "#bf1b00", + "fill": 7 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_cluster_members{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Number of cluster members", + "refId": "A" + }, + { + "expr": "alertmanager_cluster_failed_peers{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Number of failed peers", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster members count on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Shows gossip cluster members count in time and failing peers in case of any in red color.", + "fill": 1, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 24 + }, + "id": 411, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 38, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-7f5c977766-kwr2s", + "value": "alertmanager-7f5c977766-kwr2s" + } + }, + "seriesOverrides": [ + { + "alias": "Number of failed peers", + "color": "#bf1b00", + "fill": 7 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_cluster_members{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Number of cluster members", + "refId": "A" + }, + { + "expr": "alertmanager_cluster_failed_peers{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Number of failed peers", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster members count on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Shows gossip cluster members count in time and failing peers in case of any in red color.", + "fill": 1, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 24 + }, + "id": 412, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 38, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-5b9858bf54-r8skx", + "value": "alertmanager-sre-5b9858bf54-r8skx" + } + }, + "seriesOverrides": [ + { + "alias": "Number of failed peers", + "color": "#bf1b00", + "fill": 7 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_cluster_members{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Number of cluster members", + "refId": "A" + }, + { + "expr": "alertmanager_cluster_failed_peers{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Number of failed peers", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster members count on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Shows gossip cluster members count in time and failing peers in case of any in red color.", + "fill": 1, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 24 + }, + "id": 413, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 38, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-8455c65f47-mmwlq", + "value": "alertmanager-sre-8455c65f47-mmwlq" + } + }, + "seriesOverrides": [ + { + "alias": "Number of failed peers", + "color": "#bf1b00", + "fill": 7 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_cluster_members{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Number of cluster members", + "refId": "A" + }, + { + "expr": "alertmanager_cluster_failed_peers{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Number of failed peers", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster members count on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "On positive axis shows number of peers that joined the cluster and on negative axis number of peers that left the cluster.", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 28 + }, + "id": 75, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-54b9fdf9bf-gqxs4", + "value": "alertmanager-54b9fdf9bf-gqxs4" + } + }, + "seriesOverrides": [ + { + "alias": "Cluster left peers", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(alertmanager_cluster_peers_left_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Cluster joined peers", + "refId": "A" + }, + { + "expr": "increase(alertmanager_cluster_peers_joined_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Cluster left peers", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster peers left/joined on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "On positive axis shows number of peers that joined the cluster and on negative axis number of peers that left the cluster.", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 28 + }, + "id": 414, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 75, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-7f5c977766-kwr2s", + "value": "alertmanager-7f5c977766-kwr2s" + } + }, + "seriesOverrides": [ + { + "alias": "Cluster left peers", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(alertmanager_cluster_peers_left_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Cluster joined peers", + "refId": "A" + }, + { + "expr": "increase(alertmanager_cluster_peers_joined_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Cluster left peers", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster peers left/joined on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "On positive axis shows number of peers that joined the cluster and on negative axis number of peers that left the cluster.", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 28 + }, + "id": 415, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 75, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-5b9858bf54-r8skx", + "value": "alertmanager-sre-5b9858bf54-r8skx" + } + }, + "seriesOverrides": [ + { + "alias": "Cluster left peers", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(alertmanager_cluster_peers_left_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Cluster joined peers", + "refId": "A" + }, + { + "expr": "increase(alertmanager_cluster_peers_joined_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Cluster left peers", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster peers left/joined on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "On positive axis shows number of peers that joined the cluster and on negative axis number of peers that left the cluster.", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 28 + }, + "id": 416, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 75, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-8455c65f47-mmwlq", + "value": "alertmanager-sre-8455c65f47-mmwlq" + } + }, + "seriesOverrides": [ + { + "alias": "Cluster left peers", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(alertmanager_cluster_peers_left_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Cluster joined peers", + "refId": "A" + }, + { + "expr": "increase(alertmanager_cluster_peers_joined_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Cluster left peers", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster peers left/joined on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "On positive axis is number of attempts to reconnect the cluster. On negative axis if number of failed attempts.", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 34 + }, + "id": 68, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-54b9fdf9bf-gqxs4", + "value": "alertmanager-54b9fdf9bf-gqxs4" + } + }, + "seriesOverrides": [ + { + "alias": "Failed reconnections", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(alertmanager_cluster_reconnections_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Successful reconnections", + "refId": "A" + }, + { + "expr": "increase(alertmanager_cluster_reconnections_failed_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Failed reconnections", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster reconnections on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "On positive axis is number of attempts to reconnect the cluster. On negative axis if number of failed attempts.", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 34 + }, + "id": 417, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 68, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-7f5c977766-kwr2s", + "value": "alertmanager-7f5c977766-kwr2s" + } + }, + "seriesOverrides": [ + { + "alias": "Failed reconnections", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(alertmanager_cluster_reconnections_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Successful reconnections", + "refId": "A" + }, + { + "expr": "increase(alertmanager_cluster_reconnections_failed_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Failed reconnections", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster reconnections on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "On positive axis is number of attempts to reconnect the cluster. On negative axis if number of failed attempts.", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 34 + }, + "id": 418, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 68, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-5b9858bf54-r8skx", + "value": "alertmanager-sre-5b9858bf54-r8skx" + } + }, + "seriesOverrides": [ + { + "alias": "Failed reconnections", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(alertmanager_cluster_reconnections_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Successful reconnections", + "refId": "A" + }, + { + "expr": "increase(alertmanager_cluster_reconnections_failed_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Failed reconnections", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster reconnections on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "On positive axis is number of attempts to reconnect the cluster. On negative axis if number of failed attempts.", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 34 + }, + "id": 419, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 68, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-8455c65f47-mmwlq", + "value": "alertmanager-sre-8455c65f47-mmwlq" + } + }, + "seriesOverrides": [ + { + "alias": "Failed reconnections", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(alertmanager_cluster_reconnections_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Successful reconnections", + "refId": "A" + }, + { + "expr": "increase(alertmanager_cluster_reconnections_failed_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Failed reconnections", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster reconnections on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "On positive axis is number of sent cluster messages by type `update` or `full_state` and on negative axis the same for received messages.", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 40 + }, + "id": 48, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-54b9fdf9bf-gqxs4", + "value": "alertmanager-54b9fdf9bf-gqxs4" + } + }, + "seriesOverrides": [ + { + "alias": "/received.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*full_state.*/", + "color": "#629e51" + }, + { + "alias": "/.*update.*/", + "color": "#f4d598" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(alertmanager_cluster_messages_sent_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "sent {{msg_type}}", + "refId": "A" + }, + { + "expr": "sum(increase(alertmanager_cluster_messages_received_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "received {{msg_type}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster messages count on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "On positive axis is number of sent cluster messages by type `update` or `full_state` and on negative axis the same for received messages.", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 40 + }, + "id": 420, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 48, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-7f5c977766-kwr2s", + "value": "alertmanager-7f5c977766-kwr2s" + } + }, + "seriesOverrides": [ + { + "alias": "/received.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*full_state.*/", + "color": "#629e51" + }, + { + "alias": "/.*update.*/", + "color": "#f4d598" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(alertmanager_cluster_messages_sent_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "sent {{msg_type}}", + "refId": "A" + }, + { + "expr": "sum(increase(alertmanager_cluster_messages_received_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "received {{msg_type}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster messages count on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "On positive axis is number of sent cluster messages by type `update` or `full_state` and on negative axis the same for received messages.", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 40 + }, + "id": 421, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 48, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-5b9858bf54-r8skx", + "value": "alertmanager-sre-5b9858bf54-r8skx" + } + }, + "seriesOverrides": [ + { + "alias": "/received.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*full_state.*/", + "color": "#629e51" + }, + { + "alias": "/.*update.*/", + "color": "#f4d598" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(alertmanager_cluster_messages_sent_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "sent {{msg_type}}", + "refId": "A" + }, + { + "expr": "sum(increase(alertmanager_cluster_messages_received_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "received {{msg_type}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster messages count on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "On positive axis is number of sent cluster messages by type `update` or `full_state` and on negative axis the same for received messages.", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 40 + }, + "id": 422, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 48, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-8455c65f47-mmwlq", + "value": "alertmanager-sre-8455c65f47-mmwlq" + } + }, + "seriesOverrides": [ + { + "alias": "/received.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*full_state.*/", + "color": "#629e51" + }, + { + "alias": "/.*update.*/", + "color": "#f4d598" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(alertmanager_cluster_messages_sent_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "sent {{msg_type}}", + "refId": "A" + }, + { + "expr": "sum(increase(alertmanager_cluster_messages_received_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "received {{msg_type}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster messages count on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "On positive axis is size of sent cluster messages by type `update` or `full_state` and on negative axis the same for received messages.", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 46 + }, + "id": 53, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-54b9fdf9bf-gqxs4", + "value": "alertmanager-54b9fdf9bf-gqxs4" + } + }, + "seriesOverrides": [ + { + "alias": "/received.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*full_state.*/", + "color": "#629e51" + }, + { + "alias": "/.*update.*/", + "color": "#f4d598" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(alertmanager_cluster_messages_sent_size_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "sent {{msg_type}}", + "refId": "A" + }, + { + "expr": "sum(increase(alertmanager_cluster_messages_received_size_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "received {{msg_type}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster messages size on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "On positive axis is size of sent cluster messages by type `update` or `full_state` and on negative axis the same for received messages.", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 46 + }, + "id": 423, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 53, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-7f5c977766-kwr2s", + "value": "alertmanager-7f5c977766-kwr2s" + } + }, + "seriesOverrides": [ + { + "alias": "/received.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*full_state.*/", + "color": "#629e51" + }, + { + "alias": "/.*update.*/", + "color": "#f4d598" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(alertmanager_cluster_messages_sent_size_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "sent {{msg_type}}", + "refId": "A" + }, + { + "expr": "sum(increase(alertmanager_cluster_messages_received_size_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "received {{msg_type}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster messages size on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "On positive axis is size of sent cluster messages by type `update` or `full_state` and on negative axis the same for received messages.", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 46 + }, + "id": 424, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 53, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-5b9858bf54-r8skx", + "value": "alertmanager-sre-5b9858bf54-r8skx" + } + }, + "seriesOverrides": [ + { + "alias": "/received.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*full_state.*/", + "color": "#629e51" + }, + { + "alias": "/.*update.*/", + "color": "#f4d598" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(alertmanager_cluster_messages_sent_size_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "sent {{msg_type}}", + "refId": "A" + }, + { + "expr": "sum(increase(alertmanager_cluster_messages_received_size_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "received {{msg_type}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster messages size on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "On positive axis is size of sent cluster messages by type `update` or `full_state` and on negative axis the same for received messages.", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 46 + }, + "id": 425, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 53, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-8455c65f47-mmwlq", + "value": "alertmanager-sre-8455c65f47-mmwlq" + } + }, + "seriesOverrides": [ + { + "alias": "/received.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*full_state.*/", + "color": "#629e51" + }, + { + "alias": "/.*update.*/", + "color": "#f4d598" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(alertmanager_cluster_messages_sent_size_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "sent {{msg_type}}", + "refId": "A" + }, + { + "expr": "sum(increase(alertmanager_cluster_messages_received_size_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "received {{msg_type}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster messages size on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "On positive axis is number of queued cluster messages and on negative axis number of pruned messages.", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 52 + }, + "id": 62, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-54b9fdf9bf-gqxs4", + "value": "alertmanager-54b9fdf9bf-gqxs4" + } + }, + "seriesOverrides": [ + { + "alias": "Pruned messaged", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_cluster_messages_pruned_total{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pruned messaged", + "refId": "A" + }, + { + "expr": "alertmanager_cluster_messages_queued{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Queued messages", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster messages queue on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "On positive axis is number of queued cluster messages and on negative axis number of pruned messages.", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 52 + }, + "id": 426, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 62, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-7f5c977766-kwr2s", + "value": "alertmanager-7f5c977766-kwr2s" + } + }, + "seriesOverrides": [ + { + "alias": "Pruned messaged", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_cluster_messages_pruned_total{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pruned messaged", + "refId": "A" + }, + { + "expr": "alertmanager_cluster_messages_queued{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Queued messages", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster messages queue on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "On positive axis is number of queued cluster messages and on negative axis number of pruned messages.", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 52 + }, + "id": 427, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 62, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-5b9858bf54-r8skx", + "value": "alertmanager-sre-5b9858bf54-r8skx" + } + }, + "seriesOverrides": [ + { + "alias": "Pruned messaged", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_cluster_messages_pruned_total{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pruned messaged", + "refId": "A" + }, + { + "expr": "alertmanager_cluster_messages_queued{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Queued messages", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster messages queue on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "On positive axis is number of queued cluster messages and on negative axis number of pruned messages.", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 52 + }, + "id": 428, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 62, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-8455c65f47-mmwlq", + "value": "alertmanager-sre-8455c65f47-mmwlq" + } + }, + "seriesOverrides": [ + { + "alias": "Pruned messaged", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_cluster_messages_pruned_total{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pruned messaged", + "refId": "A" + }, + { + "expr": "alertmanager_cluster_messages_queued{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Queued messages", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster messages queue on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Cluster members", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 284, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 10 + }, + "id": 314, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-54b9fdf9bf-gqxs4", + "value": "alertmanager-54b9fdf9bf-gqxs4" + } + }, + "seriesOverrides": [ + { + "alias": "/dropped/", + "color": "#cca300", + "transform": "negative-Y" + }, + { + "alias": "/failed/", + "color": "#bf1b00", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(alertmanager_oversized_gossip_message_sent_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{key}}", + "refId": "A" + }, + { + "expr": "increase(alertmanager_oversized_gossip_message_dropped_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "dropped {{key}}", + "refId": "B" + }, + { + "expr": "increase(alertmanager_oversized_gossip_message_failure_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "failed {{key}}", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Count of oversized gossip messages on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 10 + }, + "id": 441, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 314, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-7f5c977766-kwr2s", + "value": "alertmanager-7f5c977766-kwr2s" + } + }, + "seriesOverrides": [ + { + "alias": "/dropped/", + "color": "#cca300", + "transform": "negative-Y" + }, + { + "alias": "/failed/", + "color": "#bf1b00", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(alertmanager_oversized_gossip_message_sent_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{key}}", + "refId": "A" + }, + { + "expr": "increase(alertmanager_oversized_gossip_message_dropped_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "dropped {{key}}", + "refId": "B" + }, + { + "expr": "increase(alertmanager_oversized_gossip_message_failure_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "failed {{key}}", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Count of oversized gossip messages on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 10 + }, + "id": 442, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 314, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-5b9858bf54-r8skx", + "value": "alertmanager-sre-5b9858bf54-r8skx" + } + }, + "seriesOverrides": [ + { + "alias": "/dropped/", + "color": "#cca300", + "transform": "negative-Y" + }, + { + "alias": "/failed/", + "color": "#bf1b00", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(alertmanager_oversized_gossip_message_sent_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{key}}", + "refId": "A" + }, + { + "expr": "increase(alertmanager_oversized_gossip_message_dropped_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "dropped {{key}}", + "refId": "B" + }, + { + "expr": "increase(alertmanager_oversized_gossip_message_failure_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "failed {{key}}", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Count of oversized gossip messages on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 10 + }, + "id": 443, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 314, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-8455c65f47-mmwlq", + "value": "alertmanager-sre-8455c65f47-mmwlq" + } + }, + "seriesOverrides": [ + { + "alias": "/dropped/", + "color": "#cca300", + "transform": "negative-Y" + }, + { + "alias": "/failed/", + "color": "#bf1b00", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(alertmanager_oversized_gossip_message_sent_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{key}}", + "refId": "A" + }, + { + "expr": "increase(alertmanager_oversized_gossip_message_dropped_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "dropped {{key}}", + "refId": "B" + }, + { + "expr": "increase(alertmanager_oversized_gossip_message_failure_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "failed {{key}}", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Count of oversized gossip messages on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 16 + }, + "id": 307, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-54b9fdf9bf-gqxs4", + "value": "alertmanager-54b9fdf9bf-gqxs4" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(1,rate(alertmanager_oversize_gossip_message_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{key}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Duration of oversized gossip messages on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 16 + }, + "id": 444, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 307, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-7f5c977766-kwr2s", + "value": "alertmanager-7f5c977766-kwr2s" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(1,rate(alertmanager_oversize_gossip_message_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{key}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Duration of oversized gossip messages on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 16 + }, + "id": 445, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 307, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-5b9858bf54-r8skx", + "value": "alertmanager-sre-5b9858bf54-r8skx" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(1,rate(alertmanager_oversize_gossip_message_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{key}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Duration of oversized gossip messages on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 16 + }, + "id": 446, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 307, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-8455c65f47-mmwlq", + "value": "alertmanager-sre-8455c65f47-mmwlq" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(1,rate(alertmanager_oversize_gossip_message_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{key}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Duration of oversized gossip messages on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 22 + }, + "id": 303, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-54b9fdf9bf-gqxs4", + "value": "alertmanager-54b9fdf9bf-gqxs4" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(alertmanager_silences_gossip_messages_propagated_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "silences", + "refId": "A" + }, + { + "expr": "increase(alertmanager_nflog_gossip_messages_propagated_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "nf_log", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of propagated gossip messages on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 22 + }, + "id": 447, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 303, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-7f5c977766-kwr2s", + "value": "alertmanager-7f5c977766-kwr2s" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(alertmanager_silences_gossip_messages_propagated_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "silences", + "refId": "A" + }, + { + "expr": "increase(alertmanager_nflog_gossip_messages_propagated_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "nf_log", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of propagated gossip messages on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 22 + }, + "id": 448, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 303, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-5b9858bf54-r8skx", + "value": "alertmanager-sre-5b9858bf54-r8skx" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(alertmanager_silences_gossip_messages_propagated_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "silences", + "refId": "A" + }, + { + "expr": "increase(alertmanager_nflog_gossip_messages_propagated_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "nf_log", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of propagated gossip messages on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 22 + }, + "id": 449, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 303, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-8455c65f47-mmwlq", + "value": "alertmanager-sre-8455c65f47-mmwlq" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(alertmanager_silences_gossip_messages_propagated_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "silences", + "refId": "A" + }, + { + "expr": "increase(alertmanager_nflog_gossip_messages_propagated_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "nf_log", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of propagated gossip messages on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Gossip messages", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 84, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 11 + }, + "id": 94, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-54b9fdf9bf-gqxs4", + "value": "alertmanager-54b9fdf9bf-gqxs4" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(alertmanager_nflog_queries_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Nf log query count", + "refId": "A" + }, + { + "expr": "increase(alertmanager_nflog_query_errors_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Nf log query errors", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Nf log queries count for $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 11 + }, + "id": 450, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 94, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-7f5c977766-kwr2s", + "value": "alertmanager-7f5c977766-kwr2s" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(alertmanager_nflog_queries_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Nf log query count", + "refId": "A" + }, + { + "expr": "increase(alertmanager_nflog_query_errors_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Nf log query errors", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Nf log queries count for $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 11 + }, + "id": 451, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 94, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-5b9858bf54-r8skx", + "value": "alertmanager-sre-5b9858bf54-r8skx" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(alertmanager_nflog_queries_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Nf log query count", + "refId": "A" + }, + { + "expr": "increase(alertmanager_nflog_query_errors_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Nf log query errors", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Nf log queries count for $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 11 + }, + "id": 452, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 94, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-8455c65f47-mmwlq", + "value": "alertmanager-sre-8455c65f47-mmwlq" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(alertmanager_nflog_queries_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Nf log query count", + "refId": "A" + }, + { + "expr": "increase(alertmanager_nflog_query_errors_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Nf log query errors", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Nf log queries count for $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 17 + }, + "id": 106, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-54b9fdf9bf-gqxs4", + "value": "alertmanager-54b9fdf9bf-gqxs4" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(1,rate(alertmanager_nflog_query_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Nf log query duration", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Nf log query duration for $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 17 + }, + "id": 453, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 106, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-7f5c977766-kwr2s", + "value": "alertmanager-7f5c977766-kwr2s" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(1,rate(alertmanager_nflog_query_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Nf log query duration", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Nf log query duration for $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 17 + }, + "id": 454, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 106, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-5b9858bf54-r8skx", + "value": "alertmanager-sre-5b9858bf54-r8skx" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(1,rate(alertmanager_nflog_query_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Nf log query duration", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Nf log query duration for $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 17 + }, + "id": 455, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 106, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-8455c65f47-mmwlq", + "value": "alertmanager-sre-8455c65f47-mmwlq" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(1,rate(alertmanager_nflog_query_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Nf log query duration", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Nf log query duration for $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 23 + }, + "id": 97, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-54b9fdf9bf-gqxs4", + "value": "alertmanager-54b9fdf9bf-gqxs4" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_nflog_snapshot_size_bytes{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Nf log snapshot size", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Nf log snapshot size for $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 23 + }, + "id": 456, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 97, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-7f5c977766-kwr2s", + "value": "alertmanager-7f5c977766-kwr2s" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_nflog_snapshot_size_bytes{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Nf log snapshot size", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Nf log snapshot size for $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 23 + }, + "id": 457, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 97, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-5b9858bf54-r8skx", + "value": "alertmanager-sre-5b9858bf54-r8skx" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_nflog_snapshot_size_bytes{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Nf log snapshot size", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Nf log snapshot size for $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 23 + }, + "id": 458, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 97, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-8455c65f47-mmwlq", + "value": "alertmanager-sre-8455c65f47-mmwlq" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_nflog_snapshot_size_bytes{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Nf log snapshot size", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Nf log snapshot size for $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 29 + }, + "id": 101, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-54b9fdf9bf-gqxs4", + "value": "alertmanager-54b9fdf9bf-gqxs4" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(alertmanager_nflog_snapshot_duration_seconds_sum{instance=~\"$instance\"}[$__interval]) / rate(alertmanager_nflog_snapshot_duration_seconds_sum{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Nf log snapshot size", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Nf log snapshot duration for $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 29 + }, + "id": 459, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 101, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-7f5c977766-kwr2s", + "value": "alertmanager-7f5c977766-kwr2s" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(alertmanager_nflog_snapshot_duration_seconds_sum{instance=~\"$instance\"}[$__interval]) / rate(alertmanager_nflog_snapshot_duration_seconds_sum{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Nf log snapshot size", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Nf log snapshot duration for $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 29 + }, + "id": 460, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 101, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-5b9858bf54-r8skx", + "value": "alertmanager-sre-5b9858bf54-r8skx" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(alertmanager_nflog_snapshot_duration_seconds_sum{instance=~\"$instance\"}[$__interval]) / rate(alertmanager_nflog_snapshot_duration_seconds_sum{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Nf log snapshot size", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Nf log snapshot duration for $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 29 + }, + "id": 461, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 101, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-8455c65f47-mmwlq", + "value": "alertmanager-sre-8455c65f47-mmwlq" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(alertmanager_nflog_snapshot_duration_seconds_sum{instance=~\"$instance\"}[$__interval]) / rate(alertmanager_nflog_snapshot_duration_seconds_sum{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Nf log snapshot size", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Nf log snapshot duration for $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 35 + }, + "id": 92, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-54b9fdf9bf-gqxs4", + "value": "alertmanager-54b9fdf9bf-gqxs4" + } + }, + "seriesOverrides": [ + { + "alias": "Cluster left peers", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_nflog_gc_duration_seconds{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Cluster joined peers", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Nf log Go GC time for $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 35 + }, + "id": 462, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 92, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-7f5c977766-kwr2s", + "value": "alertmanager-7f5c977766-kwr2s" + } + }, + "seriesOverrides": [ + { + "alias": "Cluster left peers", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_nflog_gc_duration_seconds{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Cluster joined peers", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Nf log Go GC time for $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 35 + }, + "id": 463, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 92, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-5b9858bf54-r8skx", + "value": "alertmanager-sre-5b9858bf54-r8skx" + } + }, + "seriesOverrides": [ + { + "alias": "Cluster left peers", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_nflog_gc_duration_seconds{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Cluster joined peers", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Nf log Go GC time for $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 35 + }, + "id": 464, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 92, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-8455c65f47-mmwlq", + "value": "alertmanager-sre-8455c65f47-mmwlq" + } + }, + "seriesOverrides": [ + { + "alias": "Cluster left peers", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_nflog_gc_duration_seconds{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Cluster joined peers", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Nf log Go GC time for $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Nflog", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 123, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 12 + }, + "id": 129, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-54b9fdf9bf-gqxs4", + "value": "alertmanager-54b9fdf9bf-gqxs4" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_silences{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{state}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Silences count by state on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 12 + }, + "id": 465, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 129, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-7f5c977766-kwr2s", + "value": "alertmanager-7f5c977766-kwr2s" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_silences{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{state}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Silences count by state on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 12 + }, + "id": 466, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 129, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-5b9858bf54-r8skx", + "value": "alertmanager-sre-5b9858bf54-r8skx" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_silences{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{state}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Silences count by state on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 12 + }, + "id": 467, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 129, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-8455c65f47-mmwlq", + "value": "alertmanager-sre-8455c65f47-mmwlq" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_silences{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{state}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Silences count by state on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 18 + }, + "id": 134, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-54b9fdf9bf-gqxs4", + "value": "alertmanager-54b9fdf9bf-gqxs4" + } + }, + "seriesOverrides": [ + { + "alias": "Silecnces query fails", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(alertmanager_silences_queries_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Silecnces query count", + "refId": "A" + }, + { + "expr": "increase(alertmanager_silences_query_errors_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Silecnces query fails", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Silences query count on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 18 + }, + "id": 468, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 134, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-7f5c977766-kwr2s", + "value": "alertmanager-7f5c977766-kwr2s" + } + }, + "seriesOverrides": [ + { + "alias": "Silecnces query fails", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(alertmanager_silences_queries_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Silecnces query count", + "refId": "A" + }, + { + "expr": "increase(alertmanager_silences_query_errors_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Silecnces query fails", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Silences query count on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 18 + }, + "id": 469, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 134, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-5b9858bf54-r8skx", + "value": "alertmanager-sre-5b9858bf54-r8skx" + } + }, + "seriesOverrides": [ + { + "alias": "Silecnces query fails", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(alertmanager_silences_queries_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Silecnces query count", + "refId": "A" + }, + { + "expr": "increase(alertmanager_silences_query_errors_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Silecnces query fails", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Silences query count on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 18 + }, + "id": 470, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 134, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-8455c65f47-mmwlq", + "value": "alertmanager-sre-8455c65f47-mmwlq" + } + }, + "seriesOverrides": [ + { + "alias": "Silecnces query fails", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(alertmanager_silences_queries_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Silecnces query count", + "refId": "A" + }, + { + "expr": "increase(alertmanager_silences_query_errors_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Silecnces query fails", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Silences query count on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 24 + }, + "id": 138, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-54b9fdf9bf-gqxs4", + "value": "alertmanager-54b9fdf9bf-gqxs4" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(1,rate(alertmanager_silences_query_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Silecnces query duration", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Silences query duration on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 24 + }, + "id": 471, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 138, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-7f5c977766-kwr2s", + "value": "alertmanager-7f5c977766-kwr2s" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(1,rate(alertmanager_silences_query_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Silecnces query duration", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Silences query duration on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 24 + }, + "id": 472, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 138, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-5b9858bf54-r8skx", + "value": "alertmanager-sre-5b9858bf54-r8skx" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(1,rate(alertmanager_silences_query_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Silecnces query duration", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Silences query duration on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 24 + }, + "id": 473, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 138, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-8455c65f47-mmwlq", + "value": "alertmanager-sre-8455c65f47-mmwlq" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(1,rate(alertmanager_silences_query_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Silecnces query duration", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Silences query duration on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 30 + }, + "id": 149, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-54b9fdf9bf-gqxs4", + "value": "alertmanager-54b9fdf9bf-gqxs4" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_silences_snapshot_size_bytes{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Silecnces snapshot size", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Silences snapshot size on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 30 + }, + "id": 474, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 149, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-7f5c977766-kwr2s", + "value": "alertmanager-7f5c977766-kwr2s" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_silences_snapshot_size_bytes{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Silecnces snapshot size", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Silences snapshot size on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 30 + }, + "id": 475, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 149, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-5b9858bf54-r8skx", + "value": "alertmanager-sre-5b9858bf54-r8skx" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_silences_snapshot_size_bytes{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Silecnces snapshot size", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Silences snapshot size on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 30 + }, + "id": 476, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 149, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-8455c65f47-mmwlq", + "value": "alertmanager-sre-8455c65f47-mmwlq" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_silences_snapshot_size_bytes{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Silecnces snapshot size", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Silences snapshot size on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 36 + }, + "id": 143, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-54b9fdf9bf-gqxs4", + "value": "alertmanager-54b9fdf9bf-gqxs4" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_silences_snapshot_duration_seconds{instance=~\"$instance\", quantile=\"0.99\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Silecnces snapshot duration", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Silences snapshot duration on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 36 + }, + "id": 477, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 143, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-7f5c977766-kwr2s", + "value": "alertmanager-7f5c977766-kwr2s" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_silences_snapshot_duration_seconds{instance=~\"$instance\", quantile=\"0.99\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Silecnces snapshot duration", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Silences snapshot duration on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 36 + }, + "id": 478, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 143, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-5b9858bf54-r8skx", + "value": "alertmanager-sre-5b9858bf54-r8skx" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_silences_snapshot_duration_seconds{instance=~\"$instance\", quantile=\"0.99\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Silecnces snapshot duration", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Silences snapshot duration on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 36 + }, + "id": 479, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 143, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-8455c65f47-mmwlq", + "value": "alertmanager-sre-8455c65f47-mmwlq" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_silences_snapshot_duration_seconds{instance=~\"$instance\", quantile=\"0.99\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Silecnces snapshot duration", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Silences snapshot duration on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 42 + }, + "id": 131, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-54b9fdf9bf-gqxs4", + "value": "alertmanager-54b9fdf9bf-gqxs4" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_silences_gc_duration_seconds{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Silecnces GC duration", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Silences GC duraton on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 42 + }, + "id": 480, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 131, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-7f5c977766-kwr2s", + "value": "alertmanager-7f5c977766-kwr2s" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_silences_gc_duration_seconds{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Silecnces GC duration", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Silences GC duraton on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 42 + }, + "id": 481, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 131, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-5b9858bf54-r8skx", + "value": "alertmanager-sre-5b9858bf54-r8skx" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_silences_gc_duration_seconds{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Silecnces GC duration", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Silences GC duraton on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 42 + }, + "id": 482, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 131, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-8455c65f47-mmwlq", + "value": "alertmanager-sre-8455c65f47-mmwlq" + } + }, + "seriesOverrides": [ + { + "alias": "Nf log query errors", + "color": "#890f02", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "alertmanager_silences_gc_duration_seconds{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Silecnces GC duration", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Silences GC duraton on $instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Silences", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 173, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 13 + }, + "id": 175, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-54b9fdf9bf-gqxs4", + "value": "alertmanager-54b9fdf9bf-gqxs4" + } + }, + "seriesOverrides": [ + { + "alias": "/Limit .*/", + "color": "#C15C17", + "dashes": true, + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + }, + { + "expr": "max(kube_pod_container_resource_limits_cpu_cores{pod=~\"$instance\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Limit {{pod}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU usage/s for $instance", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 13 + }, + "id": 483, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 175, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-7f5c977766-kwr2s", + "value": "alertmanager-7f5c977766-kwr2s" + } + }, + "seriesOverrides": [ + { + "alias": "/Limit .*/", + "color": "#C15C17", + "dashes": true, + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + }, + { + "expr": "max(kube_pod_container_resource_limits_cpu_cores{pod=~\"$instance\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Limit {{pod}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU usage/s for $instance", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 13 + }, + "id": 484, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 175, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-5b9858bf54-r8skx", + "value": "alertmanager-sre-5b9858bf54-r8skx" + } + }, + "seriesOverrides": [ + { + "alias": "/Limit .*/", + "color": "#C15C17", + "dashes": true, + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + }, + { + "expr": "max(kube_pod_container_resource_limits_cpu_cores{pod=~\"$instance\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Limit {{pod}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU usage/s for $instance", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 13 + }, + "id": 485, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 175, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-8455c65f47-mmwlq", + "value": "alertmanager-sre-8455c65f47-mmwlq" + } + }, + "seriesOverrides": [ + { + "alias": "/Limit .*/", + "color": "#C15C17", + "dashes": true, + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + }, + { + "expr": "max(kube_pod_container_resource_limits_cpu_cores{pod=~\"$instance\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Limit {{pod}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU usage/s for $instance", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 20 + }, + "id": 177, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-54b9fdf9bf-gqxs4", + "value": "alertmanager-54b9fdf9bf-gqxs4" + } + }, + "seriesOverrides": [ + { + "alias": "/Limit .*/", + "color": "#C15C17", + "dashes": true, + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "E" + }, + { + "expr": "max(kube_pod_container_resource_limits_memory_bytes{pod=~\"$instance\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Limit {{ pod }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory usage for $instance", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 20 + }, + "id": 486, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 177, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-7f5c977766-kwr2s", + "value": "alertmanager-7f5c977766-kwr2s" + } + }, + "seriesOverrides": [ + { + "alias": "/Limit .*/", + "color": "#C15C17", + "dashes": true, + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "E" + }, + { + "expr": "max(kube_pod_container_resource_limits_memory_bytes{pod=~\"$instance\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Limit {{ pod }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory usage for $instance", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 20 + }, + "id": 487, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 177, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-5b9858bf54-r8skx", + "value": "alertmanager-sre-5b9858bf54-r8skx" + } + }, + "seriesOverrides": [ + { + "alias": "/Limit .*/", + "color": "#C15C17", + "dashes": true, + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "E" + }, + { + "expr": "max(kube_pod_container_resource_limits_memory_bytes{pod=~\"$instance\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Limit {{ pod }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory usage for $instance", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 20 + }, + "id": 488, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1555332813958, + "repeatPanelId": 177, + "scopedVars": { + "instance": { + "selected": false, + "text": "alertmanager-sre-8455c65f47-mmwlq", + "value": "alertmanager-sre-8455c65f47-mmwlq" + } + }, + "seriesOverrides": [ + { + "alias": "/Limit .*/", + "color": "#C15C17", + "dashes": true, + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "E" + }, + { + "expr": "max(kube_pod_container_resource_limits_memory_bytes{pod=~\"$instance\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Limit {{ pod }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory usage for $instance", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Resources", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 22, + "style": "dark", + "tags": [ + "alertmanager", + "prometheus", + "alerting" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Prometheus datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": "", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "definition": "query_result(alertmanager_build_info)", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": "query_result(alertmanager_build_info)", + "refresh": 2, + "regex": "/.*instance=\"([^\"]+)\".*/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "label_values(alertmanager_build_info, instance)", + "tags": [], + "tagsQuery": "label_values(alertmanager_build_info, namespace)", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Prometheus Alertmanager", + "uid": "eea-9_siZ", + "version": 4 +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus_benchmark.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus_benchmark.json new file mode 100644 index 000000000..836a429b0 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus_benchmark.json @@ -0,0 +1,5551 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Metrics useful for benchmarking and loadtesting Prometheus itself. Designed primarily for Prometheus 2.17.x.", + "editable": true, + "gnetId": 12054, + "graphTooltip": 1, + "id": 67, + "iteration": 1622464894215, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": "Prometheus", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 49, + "panels": [], + "title": "Basics", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 40, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_build_info{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{version}} - {{revision}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Prometheus Version", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 1 + }, + "hiddenSeries": false, + "id": 72, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "time() - process_start_time_seconds{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Age", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Uptime", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "dtdurations", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 1 + }, + "hiddenSeries": false, + "id": 107, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "time() - prometheus_config_last_reload_success_timestamp_seconds{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Age", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Last Successful Config Reload", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "dtdurations", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "Prometheus", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 46, + "panels": [], + "title": "Ingestion", + "type": "row" + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "Time series": "#70dbed" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_series{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Time series", + "metric": "prometheus_local_storage_memory_series", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Head Time series", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 9 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_active_appenders{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Head Appenders", + "metric": "prometheus_local_storage_memory_series", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Head Active Appenders", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "samples/s": "#e5a8e2" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 9 + }, + "hiddenSeries": false, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "samples/s", + "metric": "prometheus_local_storage_ingested_samples_total", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Samples Appended/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1552", + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1553", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "To persist": "#9AC48A" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/Max.*/", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_chunks{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chunks", + "metric": "prometheus_local_storage_memory_chunks", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Head Chunks", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 16 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_chunks_created_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Created", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "A", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_head_chunks_removed_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Removed", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Head Chunks/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1520", + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1521", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "Removed": "#e5ac0e" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 16 + }, + "hiddenSeries": false, + "id": 25, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_isolation_high_watermark{job=\"prometheus\",instance=\"$Prometheus\"} - prometheus_tsdb_isolation_low_watermark{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Difference", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Isolation Watermarks", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "Prometheus", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 52, + "panels": [], + "title": "Compaction", + "type": "row" + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max": "#447ebc", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "Min": "#447ebc", + "Now": "#7eb26d" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Max", + "fillBelowTo": "Min", + "lines": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_min_time{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Min", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + }, + { + "expr": "time() * 1000", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Now", + "refId": "C" + }, + { + "expr": "prometheus_tsdb_head_max_time{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Max", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Head Time Range", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "dateTimeAsIso", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 24 + }, + "hiddenSeries": false, + "id": 29, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_gc_duration_seconds_sum{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "GC Time/s", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Head GC Time/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1584", + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1585", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 24 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Queue length", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_blocks_loaded{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Blocks Loaded", + "metric": "prometheus_local_storage_indexing_batch_sizes_sum", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Blocks Loaded", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Failed Compactions": "#bf1b00", + "Failed Reloads": "#bf1b00", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 31 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_reloads_total{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Reloads", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TSDB Reloads/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Failed Compactions": "#bf1b00", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 31 + }, + "hiddenSeries": false, + "id": 31, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_wal_fsync_duration_seconds_sum{job=\"prometheus\",instance=\"$Prometheus\"}[2m]) / rate(prometheus_tsdb_wal_fsync_duration_seconds_count{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Fsync Latency", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_wal_truncate_duration_seconds_sum{job=\"prometheus\",instance=\"$Prometheus\"}[2m]) / rate(prometheus_tsdb_wal_trunacte_duration_seconds_count{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Truncate Latency", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "WAL Fsync&Truncate Latency", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1616", + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1617", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Failed Compactions": "#bf1b00", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "{instance=\"demo.robustperception.io\",job=\"prometheus\"}": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 31 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_wal_corruptions_total{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "WAL Corruptions", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_reloads_failures_total{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Reload Failures", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "B", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_head_series_not_found{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Head Series Not Found", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "C", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_compactions_failed_total{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Compaction Failures", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "D", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_retention_cutoffs_failures_total{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Retention Cutoff Failures", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "E", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_checkpoint_creations_failed_total{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "WAL Checkpoint Creation Failures", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "F", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_checkpoint_deletions_failed_total{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "WAL Checkpoint Deletion Failures", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "G", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TSDB Problems/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Failed Compactions": "#bf1b00", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 38 + }, + "hiddenSeries": false, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_compactions_total{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Compactions", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Compactions/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 38 + }, + "hiddenSeries": false, + "id": 33, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_compaction_duration_seconds_sum{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Compaction Time/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Allocated bytes": "#F9BA8F", + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "RSS": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 38 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_time_retentions_total{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Time Cutoffs", + "metric": "last", + "refId": "A", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_size_retentions_total{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Size Cutoffs", + "metric": "last", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Retention Cutoffs/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 45 + }, + "hiddenSeries": false, + "id": 27, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_compaction_chunk_range_seconds_sum{job=\"prometheus\",instance=\"$Prometheus\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_range_seconds_count{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chunk Time Range", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "First Compaction, Avg Chunk Time Range", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "dtdurationms", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 45 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_compaction_chunk_size_bytes_sum{job=\"prometheus\",instance=\"$Prometheus\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_sum{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Bytes/Sample", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "First Compaction, Avg Bytes/Sample", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 45 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_compaction_chunk_samples_sum{job=\"prometheus\",instance=\"$Prometheus\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_count{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chunk Samples", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "First Compaction, Avg Chunk Samples", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "Prometheus", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 52 + }, + "id": 55, + "panels": [], + "title": "Resource Usage", + "type": "row" + }, + { + "aliasColors": { + "Allocated bytes": "#7EB26D", + "Allocated bytes - 1m max": "#BF1B00", + "Allocated bytes - 1m min": "#BF1B00", + "Allocated bytes - 5m max": "#BF1B00", + "Allocated bytes - 5m min": "#BF1B00", + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "RSS": "#447EBC" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 53 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/-/", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"prometheus\",instance=\"$Prometheus\"}", + "intervalFactor": 2, + "legendFormat": "RSS", + "metric": "process_resident_memory_bytes", + "refId": "B", + "step": 10 + }, + { + "expr": "prometheus_local_storage_target_heap_size_bytes{job=\"prometheus\",instance=\"$Prometheus\"}", + "intervalFactor": 2, + "legendFormat": "Target heap size", + "metric": "go_memstats_alloc_bytes", + "refId": "D", + "step": 10 + }, + { + "expr": "go_memstats_next_gc_bytes{job=\"prometheus\",instance=\"$Prometheus\"}", + "intervalFactor": 2, + "legendFormat": "Next GC", + "metric": "go_memstats_next_gc_bytes", + "refId": "C", + "step": 10 + }, + { + "expr": "go_memstats_alloc_bytes{job=\"prometheus\",instance=\"$Prometheus\"}", + "intervalFactor": 2, + "legendFormat": "Allocated", + "metric": "go_memstats_alloc_bytes", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Allocated bytes": "#F9BA8F", + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "RSS": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 53 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(go_memstats_alloc_bytes_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Allocated Bytes/s", + "metric": "go_memstats_alloc_bytes", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Allocations", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1648", + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1649", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 53 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(process_cpu_seconds_total{job=\"prometheus\",instance=\"$Prometheus\"}[1m])", + "intervalFactor": 2, + "legendFormat": "Irate", + "metric": "prometheus_local_storage_ingested_samples_total", + "refId": "A", + "step": 10 + }, + { + "expr": "rate(process_cpu_seconds_total{job=\"prometheus\",instance=\"$Prometheus\"}[5m])", + "intervalFactor": 2, + "legendFormat": "5m rate", + "metric": "prometheus_local_storage_ingested_samples_total", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + "avg" + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 60 + }, + "hiddenSeries": false, + "id": 70, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_symbol_table_size_bytes{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RAM Used", + "metric": "prometheus_local_storage_ingested_samples_total", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Symbol Tables Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + "avg" + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 60 + }, + "hiddenSeries": false, + "id": 71, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_storage_blocks_bytes_total{job=\"prometheus\",instance=\"$Prometheus\"} or prometheus_tsdb_storage_blocks_bytes{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Disk Used", + "metric": "prometheus_local_storage_ingested_samples_total", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Block Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + "avg" + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Max": "#e24d42", + "Open": "#508642" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 60 + }, + "hiddenSeries": false, + "id": 41, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_max_fds{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Max", + "refId": "A" + }, + { + "expr": "process_open_fds{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Open", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "File Descriptors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "Prometheus", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 67 + }, + "id": 91, + "panels": [], + "title": "Service Discovery", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 68 + }, + "hiddenSeries": false, + "id": 42, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_sd_discovered_targets{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{name}}-{{config}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Discovered Targets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 68 + }, + "hiddenSeries": false, + "id": 96, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_sd_updates_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Sent Updates/s", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1680", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:1681", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 68 + }, + "hiddenSeries": false, + "id": 97, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_sd_received_updates_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Received Updates/s", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1712", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:1713", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "Prometheus", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 75 + }, + "id": 99, + "panels": [], + "title": "Scraping", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 76 + }, + "hiddenSeries": false, + "id": 105, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_target_interval_length_seconds_sum{job=\"prometheus\",instance=\"$Prometheus\"}[2m]) / rate(prometheus_target_interval_length_seconds_count{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{interval}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Scrape Interval", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1744", + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:1745", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 76 + }, + "hiddenSeries": false, + "id": 104, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_target_scrapes_exceeded_sample_limit_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Exceeded Sample Limit", + "refId": "A" + }, + { + "expr": "rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Duplicate Timestamp", + "refId": "C" + }, + { + "expr": "rate(prometheus_target_scrapes_sample_out_of_bounds_total{job=\"prometheus\",instance=\"$Prometheus\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Out Of Bounds ", + "refId": "D" + }, + { + "expr": "rate(prometheus_target_scrapes_sample_out_of_order_total{job=\"prometheus\",instance=\"$Prometheus\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Out of Order", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Scrape Problems/s", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1776", + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:1777", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 76 + }, + "hiddenSeries": false, + "id": 95, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_target_metadata_cache_bytes{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{scrape_job}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Metadata Cache Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "Prometheus", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 83 + }, + "id": 63, + "panels": [], + "title": "Query Engine", + "type": "row" + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "Time spent in each mode, per second", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 84 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_engine_query_duration_seconds_sum{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{slice}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Query engine timings/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1808", + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1809", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 84 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_rule_group_iterations_missed_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m]) ", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ rule_group }} missed", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "B", + "step": 10 + }, + { + "expr": "rate(prometheus_rule_evaluation_failures_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ rule_group }} failed", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Rule group evaulation problems/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1840", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1841", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 84 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_rule_group_duration_seconds_sum{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Rule evaluation duration", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Evaluation time of rule groups/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1902", + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1903", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "Prometheus", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 91 + }, + "id": 77, + "panels": [], + "title": "Notification", + "type": "row" + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 92 + }, + "hiddenSeries": false, + "id": 86, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_notifications_sent_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{alertmanager}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Notification Sent/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1934", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1935", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 92 + }, + "hiddenSeries": false, + "id": 87, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_notifications_errors_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m]) / rate(prometheus_notifications_sent_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{alertmanager}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Notification Error Ratio", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1966", + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1967", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 92 + }, + "hiddenSeries": false, + "id": 81, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_notifications_latency_seconds_sum{job=\"prometheus\",instance=\"$Prometheus\"}[2m]) / rate(prometheus_notifications_latency_seconds_count{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{alertmanager}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Notification Latency", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1998", + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1999", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 99 + }, + "hiddenSeries": false, + "id": 85, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_notifications_alertmanagers_discovered{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Alertmanagers", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Alertmanagers Discovered", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 99 + }, + "hiddenSeries": false, + "id": 89, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_notifications_dropped_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Dropped", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Notifications Dropped/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2030", + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:2031", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 99 + }, + "hiddenSeries": false, + "id": 88, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_notifications_queue_length{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Pending", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + }, + { + "expr": "prometheus_notifications_queue_capacity{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Max", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Notification Queue", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "Prometheus", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 106 + }, + "id": 58, + "panels": [], + "title": "HTTP Server", + "type": "row" + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 107 + }, + "hiddenSeries": false, + "id": 38, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_http_request_duration_seconds_count{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "HTTP requests/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2062", + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:2063", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 107 + }, + "hiddenSeries": false, + "id": 37, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_http_request_duration_seconds_sum{job=\"prometheus\",instance=\"$Prometheus\"}[2m]) / rate(prometheus_http_request_duration_seconds_count{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "HTTP request latency", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2094", + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:2095", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 107 + }, + "hiddenSeries": false, + "id": 36, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_http_request_duration_seconds_sum{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Time spent in HTTP requests/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2126", + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:2127", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Interval": "#890f02", + "Last Duration": "#f9934e", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 114 + }, + "hiddenSeries": false, + "id": 43, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_rule_group_last_duration_seconds{job=\"prometheus\",instance=\"$Prometheus\",rule_group=~\"$RuleGroup\"}\n", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ rule_group }}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Alerting: group rules evaluation time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1028", + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1029", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Interval": "#890f02", + "Last Duration": "#f9934e", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 114 + }, + "hiddenSeries": false, + "id": 66, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeatDirection": "h", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_rule_group_rules{job=\"prometheus\",instance=\"$Prometheus\",rule_group=~\"$RuleGroup\"}\n", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ rule_group }}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Alerting: number of rules in group", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1254", + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1255", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": false, + "text": "10.103.1.13", + "value": "10.103.1.13" + }, + "datasource": "Prometheus", + "definition": "", + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "Prometheus", + "options": [], + "query": { + "query": "query_result(up{job=\"prometheus\"} == 1)", + "refId": "Prometheus-Prometheus-Variable-Query" + }, + "refresh": 2, + "regex": ".*instance=\"([^\"]+)\".*", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": null, + "tags": [], + "tagsQuery": null, + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "Prometheus", + "definition": "", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": false, + "name": "RuleGroup", + "options": [], + "query": { + "query": "prometheus_rule_group_last_duration_seconds{job=\"prometheus\",instance=\"$Prometheus\"}", + "refId": "Prometheus-RuleGroup-Variable-Query" + }, + "refresh": 2, + "regex": ".*rule_group=\"(.*?)\".*", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Prometheus Benchmark", + "uid": "L0HBvojWz", + "version": 2 +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/rabbitmq.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/rabbitmq.json new file mode 100644 index 000000000..7d8779a88 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/rabbitmq.json @@ -0,0 +1,5287 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "A new RabbitMQ Management Overview", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 10991, + "graphTooltip": 1, + "id": 86, + "iteration": 1658139401222, + "links": [ + { + "icon": "doc", + "tags": [], + "targetBlank": true, + "title": "Monitoring with Prometheus & Grafana", + "tooltip": "", + "type": "link", + "url": "https://www.rabbitmq.com/prometheus.html" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#37872D", + "value": null + }, + { + "color": "#1F60C4", + "value": 10000 + }, + { + "color": "#C4162A", + "value": 100000 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "id": 64, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "sum(rabbitmq_queue_messages_ready * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Ready messages", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": -1 + }, + { + "color": "#37872D", + "value": 50 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 62, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "sum(irate(rabbitmq_channel_messages_published_total[5m]) * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Incoming messages / s", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": 0 + }, + { + "color": "#37872D", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 66, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(rabbitmq_channels * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) - sum(rabbitmq_channel_consumers * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Publishers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": 0 + }, + { + "color": "#37872D", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 37, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(rabbitmq_connections * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Connections", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": 0 + }, + { + "color": "#37872D", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 40, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(rabbitmq_queues * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Queues", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#37872D", + "value": null + }, + { + "color": "#1F60C4", + "value": 100 + }, + { + "color": "#C4162A", + "value": 500 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 3 + }, + "id": 65, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(rabbitmq_queue_messages_unacked * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Unacknowledged messages", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": -1 + }, + { + "color": "#37872D", + "value": 50 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 3 + }, + "id": 63, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "sum(rate(rabbitmq_channel_messages_redelivered_total[5m]) * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_channel_messages_delivered_total[5m]) * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_channel_messages_delivered_ack_total[5m]) * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_channel_get_total[5m]) * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_channel_get_ack_total[5m]) * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Outgoing messages / s", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": 0 + }, + { + "color": "#37872D", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 3 + }, + "id": 41, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(rabbitmq_channel_consumers * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Consumers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": 0 + }, + { + "color": "#37872D", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 3 + }, + "id": 38, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(rabbitmq_channels * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Channels", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1F60C4", + "value": null + }, + { + "color": "#37872D", + "value": 3 + }, + { + "color": "#C4162A", + "value": 8 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 3 + }, + "id": 67, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(rabbitmq_build_info * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Nodes", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 4, + "panels": [], + "title": "NODES", + "type": "row" + }, + { + "columns": [], + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fontSize": "100%", + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 69, + "links": [], + "scroll": true, + "showHeader": true, + "sort": { + "col": 8, + "desc": false + }, + "styles": [ + { + "alias": "Erlang/OTP", + "align": "auto", + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "link": false, + "mappingType": 1, + "pattern": "erlang_version", + "thresholds": [ + "" + ], + "type": "string", + "unit": "none" + }, + { + "alias": "RabbitMQ", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "mappingType": 1, + "pattern": "rabbitmq_version", + "thresholds": [ + "" + ], + "type": "string", + "unit": "none" + }, + { + "alias": "Host", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "mappingType": 1, + "pattern": "instance", + "preserveFormat": false, + "thresholds": [], + "type": "string", + "unit": "short", + "valueMaps": [] + }, + { + "alias": "Node name", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "mappingType": 1, + "pattern": "rabbitmq_node", + "thresholds": [ + "" + ], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "job", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Cluster", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "rabbitmq_node", + "thresholds": [], + "type": "hidden", + "unit": "short", + "valueMaps": [] + }, + { + "alias": "prometheus.erl", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "prometheus_client_version", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "rabbitmq_prometheus", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "prometheus_plugin_version", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "rabbitmq_build_info * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "transform": "table", + "type": "table-old" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "If the value is zero or less, the memory alarm will be triggered and all publishing connections across all cluster nodes will be blocked.\n\nThis value can temporarily go negative because the memory alarm is triggered with a slight delay.\n\nThe kernel's view of the amount of memory used by the node can differ from what the node itself can observe. This means that this value can be negative for a sustained period of time.\n\nBy default nodes use resident set size (RSS) to compute how much memory they use. This strategy can be changed (see the guides below).\n\n* [Alarms](https://www.rabbitmq.com/alarms.html)\n* [Memory Alarms](https://www.rabbitmq.com/memory.html)\n* [Reasoning About Memory Use](https://www.rabbitmq.com/memory-use.html)\n* [Blocked Connection Notifications](https://www.rabbitmq.com/connection-blocked.html)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 11 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(rabbitmq_resident_memory_limit_bytes * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) -\n(rabbitmq_process_resident_memory_bytes * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "lt", + "value": 536870912, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "lt", + "value": 0, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Memory available before publishers blocked", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "This metric is reported for the partition where the RabbitMQ data directory is stored.\n\nIf the value is zero or less, the disk alarm will be triggered and all publishing connections across all cluster nodes will be blocked.\n\nThis value can temporarily go negative because the free disk space alarm is triggered with a slight delay.\n\n* [Alarms](https://www.rabbitmq.com/alarms.html)\n* [Disk Space Alarms](https://www.rabbitmq.com/disk-alarms.html)\n* [Disk Space](https://www.rabbitmq.com/production-checklist.html#resource-limits-disk-space)\n* [Persistence Configuration](https://www.rabbitmq.com/persistence-conf.html)\n* [Blocked Connection Notifications](https://www.rabbitmq.com/connection-blocked.html)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 12, + "y": 11 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rabbitmq_disk_space_available_bytes * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "lt", + "value": 1073741824, + "yaxis": "left" + }, + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "lt", + "value": 5368709120, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Disk space available before publishers blocked", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "When this value reaches zero, new connections will not be accepted and disk write operations may fail.\n\nClient libraries, peer nodes and CLI tools will not be able to connect when the node runs out of available file descriptors.\n\n* [Open File Handles Limit](https://www.rabbitmq.com/production-checklist.html#resource-limits-file-handle-limit)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 11 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(rabbitmq_process_max_fds * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) -\n(rabbitmq_process_open_fds * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "lt", + "value": 500, + "yaxis": "left" + }, + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "lt", + "value": 1000, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "File descriptors available", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": -1, + "format": "none", + "label": "", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "When this value reaches zero, new connections will not be accepted.\n\nClient libraries, peer nodes and CLI tools will not be able to connect when the node runs out of available file descriptors.\n\n* [Networking and RabbitMQ](https://www.rabbitmq.com/networking.html)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 15 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(rabbitmq_process_max_tcp_sockets * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) -\n(rabbitmq_process_open_tcp_sockets * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "lt", + "value": 500, + "yaxis": "left" + }, + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "lt", + "value": 1000, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "TCP sockets available", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": -1, + "format": "none", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 27, + "panels": [], + "title": "QUEUED MESSAGES", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of ready messages ready to be delivered to consumers.\n\nAim to keep this value as low as possible. RabbitMQ behaves best when messages are flowing through it. It's OK for publishers to occasionally outpace consumers, but the expectation is that consumers will eventually process all ready messages.\n\nIf this metric keeps increasing, your system will eventually run out of memory and/or disk space. Consider using TTL or Queue Length Limit to prevent unbounded message growth.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Consumers](https://www.rabbitmq.com/consumers.html)\n* [Queue Length Limit](https://www.rabbitmq.com/maxlength.html)\n* [Time-To-Live and Expiration](https://www.rabbitmq.com/ttl.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 20 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rabbitmq_queue_messages_ready * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Messages ready to be delivered to consumers", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The total number of messages that are either in-flight to consumers, currently being processed by consumers or simply waiting for the consumer acknowledgements to be processed by the queue. Until the queue processes the message acknowledgement, the message will remain unacknowledged.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Confirms and Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [Consumer Prefetch](https://www.rabbitmq.com/consumer-prefetch.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 20 + }, + "hiddenSeries": false, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rabbitmq_queue_messages_unacked * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Messages pending consumer acknowledgement", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 11, + "panels": [], + "title": "INCOMING MESSAGES", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The incoming message rate before any routing rules are applied.\n\nIf this value is lower than the number of messages published to queues, it may indicate that some messages are delivered to more than one queue.\n\nIf this value is higher than the number of messages published to queues, messages cannot be routed and will either be dropped or returned to publishers.\n\n* [Publishers](https://www.rabbitmq.com/publishers.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 26 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_published_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Messages published / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The rate of messages confirmed by the broker to publishers. Publishers must opt-in to receive message confirmations.\n\nIf this metric is consistently at zero it may suggest that publisher confirms are not used by clients. The safety of published messages is likely to be at risk.\n\n* [Publisher Confirms](https://www.rabbitmq.com/confirms.html#publisher-confirms)\n* [Publisher Confirms and Data Safety](https://www.rabbitmq.com/publishers.html#data-safety)\n* [When Will Published Messages Be Confirmed by the Broker?](https://www.rabbitmq.com/confirms.html#when-publishes-are-confirmed)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 26 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_confirmed_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Messages confirmed to publishers / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The rate of messages received from publishers and successfully routed to the master queue replicas.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Publishers](https://www.rabbitmq.com/publishers.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 31 + }, + "hiddenSeries": false, + "id": 61, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_queue_messages_published_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Messages routed to queues / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The rate of messages received from publishers that have publisher confirms enabled and the broker has not confirmed yet.\n\n* [Publishers](https://www.rabbitmq.com/publishers.html)\n* [Confirms and Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [When Will Published Messages Be Confirmed by the Broker?](https://www.rabbitmq.com/confirms.html#when-publishes-are-confirmed)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 31 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_unconfirmed[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Messages unconfirmed to publishers / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The rate of messages that cannot be routed and are dropped. \n\nAny value above zero means message loss and likely suggests a routing problem on the publisher end.\n\n* [Unroutable Message Handling](https://www.rabbitmq.com/publishers.html#unroutable)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 36 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/rabbit/", + "color": "#C4162A" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_unroutable_dropped_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Unroutable messages dropped / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The rate of messages that cannot be routed and are returned back to publishers.\n\nSustained values above zero may indicate a routing problem on the publisher end.\n\n* [Unroutable Message Handling](https://www.rabbitmq.com/publishers.html#unroutable)\n* [When Will Published Messages Be Confirmed by the Broker?](https://www.rabbitmq.com/confirms.html#when-publishes-are-confirmed)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 36 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/rabbit/", + "color": "#C4162A" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_unroutable_returned_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Unroutable messages returned to publishers / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 29, + "panels": [], + "title": "OUTGOING MESSAGES", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The rate of messages delivered to consumers. It includes messages that have been redelivered.\n\nThis metric does not include messages that have been fetched by consumers using `basic.get` (consumed by polling).\n\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 42 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n (rate(rabbitmq_channel_messages_delivered_total[60s]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) +\n (rate(rabbitmq_channel_messages_delivered_ack_total[60s]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})\n) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Messages delivered / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The rate of messages that have been redelivered to consumers. It includes messages that have been requeued automatically and redelivered due to channel exceptions or connection closures.\n\nHaving some redeliveries is expected, but if this metric is consistently non-zero, it is worth investigating why.\n\n* [Negative Acknowledgement and Requeuing of Deliveries](https://www.rabbitmq.com/confirms.html#consumer-nacks-requeue)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 42 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_redelivered_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 20, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 100, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Messages redelivered / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The rate of message deliveries to consumers that use manual acknowledgement mode.\n\nWhen this mode is used, RabbitMQ waits for consumers to acknowledge messages before more messages can be delivered.\n\nThis is the safest way of consuming messages.\n\n* [Consumer Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [Consumer Prefetch](https://www.rabbitmq.com/consumer-prefetch.html)\n* [Consumer Acknowledgement Modes, Prefetch and Throughput](https://www.rabbitmq.com/confirms.html#channel-qos-prefetch-throughput)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 47 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_delivered_ack_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Messages delivered with manual ack / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The rate of message deliveries to consumers that use automatic acknowledgement mode.\n\nWhen this mode is used, RabbitMQ does not wait for consumers to acknowledge message deliveries.\n\nThis mode is fire-and-forget and does not offer any delivery safety guarantees. It tends to provide higher throughput and it may lead to consumer overload and higher consumer memory usage.\n\n* [Consumer Acknowledgement Modes, Prefetch and Throughput](https://www.rabbitmq.com/confirms.html#channel-qos-prefetch-throughput)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 47 + }, + "hiddenSeries": false, + "id": 21, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_delivered_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Messages delivered auto ack / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The rate of message acknowledgements coming from consumers that use manual acknowledgement mode.\n\n* [Consumer Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [Consumer Prefetch](https://www.rabbitmq.com/consumer-prefetch.html)\n* [Consumer Acknowledgement Modes, Prefetch and Throughput](https://www.rabbitmq.com/confirms.html#channel-qos-prefetch-throughput)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 52 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_acked_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Messages acknowledged / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The rate of messages delivered to polling consumers that use automatic acknowledgement mode.\n\nThe use of polling consumers is highly inefficient and therefore strongly discouraged.\n\n* [Fetching individual messages](https://www.rabbitmq.com/consumers.html#fetching)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 52 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/rabbit/", + "color": "#C4162A" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_get_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Polling operations with auto ack / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The rate of polling consumer operations that yield no result.\n\nAny value above zero means that RabbitMQ resources are wasted by polling consumers.\n\nCompare this metric to the other polling consumer metrics to see the inefficiency rate.\n\nThe use of polling consumers is highly inefficient and therefore strongly discouraged.\n\n* [Fetching individual messages](https://www.rabbitmq.com/consumers.html#fetching)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 57 + }, + "hiddenSeries": false, + "id": 25, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/rabbit/", + "color": "#C4162A" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_get_empty_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Polling operations that yield no result / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The rate of messages delivered to polling consumers that use manual acknowledgement mode.\n\nThe use of polling consumers is highly inefficient and therefore strongly discouraged.\n\n* [Fetching individual messages](https://www.rabbitmq.com/consumers.html#fetching)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 57 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/rabbit/", + "color": "#C4162A" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_get_ack_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Polling operations with manual ack / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 53, + "panels": [], + "title": "QUEUES", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of queue masters per node. \n\nThis metric makes it easy to see sub-optimal queue distribution in a cluster.\n\n* [Queue Masters, Data Locality](https://www.rabbitmq.com/ha.html#master-migration-data-locality)\n* [Queues](https://www.rabbitmq.com/queues.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 63 + }, + "hiddenSeries": false, + "id": 57, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rabbitmq_queues * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total queues", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": -1, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The rate of queue declarations performed by clients.\n\nLow sustained values above zero are to be expected. High rates may be indicative of queue churn or high rates of connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [Queues](https://www.rabbitmq.com/queues.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 63 + }, + "hiddenSeries": false, + "id": 58, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_queues_declared_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Queues declared / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The rate of new queues created (as opposed to redeclarations).\n\nLow sustained values above zero are to be expected. High rates may be indicative of queue churn or high rates of connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [Queues](https://www.rabbitmq.com/queues.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 63 + }, + "hiddenSeries": false, + "id": 60, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_queues_created_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Queues created / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The rate of queues deleted.\n\nLow sustained values above zero are to be expected. High rates may be indicative of queue churn or high rates of connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [Queues](https://www.rabbitmq.com/queues.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 63 + }, + "hiddenSeries": false, + "id": 59, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_queues_deleted_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Queues deleted / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 68 + }, + "id": 51, + "panels": [], + "title": "CHANNELS", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of channels on all currently opened connections.\n\nIf this metric grows monotonically it is highly likely a channel leak in one of the applications. Confirm channel leaks by using the _Channels opened_ and _Channels closed_ metrics.\n\n* [Channel Leak](https://www.rabbitmq.com/channels.html#channel-leaks)\n* [Channels](https://www.rabbitmq.com/channels.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 69 + }, + "hiddenSeries": false, + "id": 54, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rabbitmq_channels * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total channels", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": -1, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The rate of new channels opened by applications across all connections. Channels are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of channel churn or mass connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [High Channel Churn](https://www.rabbitmq.com/channels.html#high-channel-churn)\n* [Channels](https://www.rabbitmq.com/channels.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 69 + }, + "hiddenSeries": false, + "id": 55, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channels_opened_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Channels opened / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The rate of channels closed by applications across all connections. Channels are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of channel churn or mass connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [High Channel Churn](https://www.rabbitmq.com/channels.html#high-channel-churn)\n* [Channels](https://www.rabbitmq.com/channels.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 69 + }, + "hiddenSeries": false, + "id": 56, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channels_closed_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Channels closed / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 74 + }, + "id": 46, + "panels": [], + "title": "CONNECTIONS", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of client connections.\n\nIf this metric grows monotonically it is highly likely a connection leak in one of the applications. Confirm connection leaks by using the _Connections opened_ and _Connections closed_ metrics.\n\n* [Connection Leak](https://www.rabbitmq.com/connections.html#monitoring)\n* [Connections](https://www.rabbitmq.com/connections.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 75 + }, + "hiddenSeries": false, + "id": 47, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rabbitmq_connections * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total connections", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": -1, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The rate of new connections opened by clients. Connections are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of connection churn or mass connection recovery.\n\n* [Connection Leak](https://www.rabbitmq.com/connections.html#monitoring)\n* [Connections](https://www.rabbitmq.com/connections.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 75 + }, + "hiddenSeries": false, + "id": 48, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_connections_opened_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Connections opened / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The rate of connections closed. Connections are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of connection churn or mass connection recovery.\n\n* [Connections](https://www.rabbitmq.com/connections.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 75 + }, + "hiddenSeries": false, + "id": 49, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_connections_closed_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Connections closed / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": "", + "schemaVersion": 36, + "style": "dark", + "tags": [ + "rabbitmq-prometheus" + ], + "templating": { + "list": [ + { + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(rabbitmq_identity_info, namespace)", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(rabbitmq_identity_info, namespace)", + "refId": "Prometheus-namespace-Variable-Query" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "rabbit@kef1p-phycon0001", + "value": "rabbit@kef1p-phycon0001" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(rabbitmq_identity_info, rabbitmq_node)", + "hide": 0, + "includeAll": false, + "label": "RabbitMQ Cluster", + "multi": false, + "name": "rabbitmq_node", + "options": [], + "query": { + "query": "label_values(rabbitmq_identity_info, rabbitmq_node)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "15s", + "30s", + "1m", + "5m", + "10m" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "RabbitMQ-Overview - Update", + "uid": "Hz7D2_LGp", + "version": 2, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/redfish.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/redfish.json new file mode 100644 index 000000000..9cd65e141 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/redfish.json @@ -0,0 +1,3769 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": 12403, + "graphTooltip": 0, + "id": 130, + "iteration": 1616784463607, + "links": [], + "panels": [ + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": { + "align": null + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1277 + }, + { + "color": "yellow", + "value": 1287 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 53, + "options": { + "colorMode": "value", + "fieldOptions": { + "calcs": [ + "mean" + ], + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [], + "values": true + }, + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "count" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "6.6.1", + "targets": [ + { + "expr": "count(up{job=\"redfish-exporter\"} == 1)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "expr": "", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "iDRAC Up", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": { + "align": null + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + }, + { + "color": "green", + "value": 1277 + }, + { + "color": "yellow", + "value": 1287 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 54, + "options": { + "colorMode": "value", + "fieldOptions": { + "calcs": [ + "mean" + ], + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [], + "values": true + }, + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "count" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "6.6.1", + "targets": [ + { + "expr": "count(redfish_system_power_state == 1)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Powered On", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": { + "align": null + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + }, + { + "color": "green", + "value": 1277 + }, + { + "color": "yellow", + "value": 1287 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 55, + "options": { + "colorMode": "value", + "fieldOptions": { + "calcs": [ + "mean" + ], + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [], + "values": true + }, + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "count" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "6.6.1", + "targets": [ + { + "expr": "count(redfish_system_power_state != 1)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Powered Off", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": { + "align": null + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1277 + }, + { + "color": "yellow", + "value": 1287 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 56, + "options": { + "colorMode": "value", + "fieldOptions": { + "calcs": [ + "mean" + ], + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [], + "values": true + }, + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "count" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "6.6.1", + "targets": [ + { + "expr": "count(redfish_chassis_health != 1)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Unhealthy Nodes", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": { + "align": null + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1277 + }, + { + "color": "yellow", + "value": 1287 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 57, + "options": { + "colorMode": "value", + "fieldOptions": { + "calcs": [ + "mean" + ], + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [], + "values": true + }, + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "count" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "6.6.1", + "targets": [ + { + "expr": "count(redfish_logservices_entry_count{name=\"SEL Log Service\", severity!=\"OK\"} != 0)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Nodes with SEL Logs", + "type": "stat" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 31, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 15, + "w": 5, + "x": 0, + "y": 4 + }, + "hiddenSeries": false, + "id": 36, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(redfish_chassis_power_average_consumed_watts)", + "legendFormat": "Rack power consumption", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Power Consumption", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "watt", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 15, + "w": 6, + "x": 5, + "y": 4 + }, + "hiddenSeries": false, + "id": 44, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(redfish_chassis_power_average_consumed_watts) by (env)", + "interval": "", + "legendFormat": "{{ env }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Power Consumption", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "watt", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fontSize": "100%", + "gridPos": { + "h": 15, + "w": 5, + "x": 11, + "y": 4 + }, + "id": 38, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": null, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "BMC", + "align": "left", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "", + "linkUrl": "https://${__cell}", + "pattern": "instance", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Power state", + "align": "auto", + "colorMode": "row", + "colors": [ + "#73BF69", + "#73BF69", + "#C4162A" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value", + "thresholds": [ + "0", + "1.1" + ], + "type": "string", + "unit": "short", + "valueMaps": [ + { + "text": "ON", + "value": "1" + }, + { + "text": "OFF", + "value": "2" + } + ] + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "__name__", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "env", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "job", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "resource", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "system_id", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkUrl": "https://${__cell_3}", + "mappingType": 1, + "pattern": "server", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "hostname", + "thresholds": [], + "type": "hidden", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort_desc(redfish_system_power_state)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Power states", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fontSize": "100%", + "gridPos": { + "h": 15, + "w": 8, + "x": 16, + "y": 4 + }, + "id": 33, + "interval": "", + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 8, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "/__name__|chassis_id|Time|env|job|resource|instance/", + "type": "hidden" + }, + { + "alias": "Status", + "align": "auto", + "colorMode": "row", + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value", + "preserveFormat": false, + "thresholds": [ + "1.2", + "2.0" + ], + "type": "string", + "unit": "short", + "valueMaps": [ + { + "text": "WARNING", + "value": "2" + }, + { + "text": "ERROR", + "value": "3" + }, + { + "text": "HEALTHY", + "value": "1" + } + ] + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "", + "linkUrl": "https://$__cell_4", + "mappingType": 1, + "pattern": "server", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort(redfish_chassis_health)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Chassis status", + "transform": "table", + "type": "table" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 11, + "w": 5, + "x": 0, + "y": 19 + }, + "hiddenSeries": false, + "id": 39, + "interval": "5m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(redfish_chassis_temperature_celsius{sensor_id=~\".*InletTemp\"}) by (env)", + "hide": false, + "interval": "", + "legendFormat": "max inlet {{ env }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Max Inlet Temp", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "celsius", + "label": "", + "logBase": 1, + "max": null, + "min": "16", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 11, + "w": 6, + "x": 5, + "y": 19 + }, + "hiddenSeries": false, + "id": 40, + "interval": "5m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(redfish_chassis_temperature_celsius{sensor_id=~\".*CPU1Temp\"}) by (env)", + "hide": false, + "interval": "", + "legendFormat": "{{ env }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Max CPU1 Temp", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "celsius", + "label": null, + "logBase": 1, + "max": null, + "min": "40", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 11, + "w": 5, + "x": 11, + "y": 19 + }, + "hiddenSeries": false, + "id": 41, + "interval": "5m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(redfish_chassis_temperature_celsius{sensor_id=~\".*CPU2Temp\"}) by (env)", + "hide": false, + "interval": "", + "legendFormat": "{{ env }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Max CPU2 Temp", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "celsius", + "label": null, + "logBase": 1, + "max": null, + "min": "40", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": { + "align": null + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 11, + "w": 4, + "x": 16, + "y": 19 + }, + "hiddenSeries": false, + "id": 42, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "count(redfish_system_power_state == 1) by (env)", + "hide": false, + "interval": "", + "legendFormat": "Powered up {{ env }}", + "refId": "A" + }, + { + "expr": "count(redfish_system_power_state == 2) by (env) * -1", + "hide": true, + "interval": "", + "legendFormat": "Powered down {{ env }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Powered ON by Rack", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": { + "align": null + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 11, + "w": 4, + "x": 20, + "y": 19 + }, + "hiddenSeries": false, + "id": 43, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "count(redfish_system_power_state == 1) by (env)", + "hide": true, + "interval": "", + "legendFormat": "Powered up {{ env }}", + "refId": "A" + }, + { + "expr": "count(redfish_system_power_state == 2) by (env)", + "hide": false, + "interval": "", + "legendFormat": "Powered down {{ env }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Powered OFF by Rack", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 7, + "x": 0, + "y": 30 + }, + "heatmap": {}, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 46, + "interval": "1m", + "legend": { + "show": false + }, + "options": {}, + "pluginVersion": "7.1.5", + "reverseYBuckets": false, + "targets": [ + { + "expr": "redfish_chassis_temperature_celsius{sensor_id=~\".*InletTemp\"}", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Max Inlet Temp", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 5, + "x": 7, + "y": 30 + }, + "heatmap": {}, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 51, + "interval": "1m", + "legend": { + "show": false + }, + "options": {}, + "pluginVersion": "7.1.5", + "reverseYBuckets": false, + "targets": [ + { + "expr": "max(redfish_chassis_fan_rpm_percentage) by (server) > 0", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Max server fan speed", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 12, + "y": 30 + }, + "heatmap": {}, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 48, + "interval": "1m", + "legend": { + "show": false + }, + "options": {}, + "pluginVersion": "7.1.5", + "reverseYBuckets": false, + "targets": [ + { + "expr": "redfish_chassis_temperature_celsius{sensor_id=~\".*CPU1Temp\"} != 0", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ env }}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU1 Temp", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "short", + "logBase": 1, + "max": "95", + "min": "25", + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 18, + "y": 30 + }, + "heatmap": {}, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 50, + "interval": "1m", + "legend": { + "show": false + }, + "options": {}, + "pluginVersion": "7.1.5", + "reverseYBuckets": false, + "targets": [ + { + "expr": "redfish_chassis_temperature_celsius{sensor_id=~\".*CPU2Temp\"} != 0", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ env }}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU2 Temp", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "short", + "logBase": 1, + "max": "95", + "min": "25", + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": { + "align": null + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 13, + "x": 0, + "y": 40 + }, + "hiddenSeries": false, + "id": 47, + "interval": "5m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "redfish_chassis_temperature_celsius{sensor_id=~\".*InletTemp\"}", + "hide": false, + "interval": "", + "legendFormat": "{{ env }} {{ server }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Max Inlet Temp", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 11, + "x": 13, + "y": 40 + }, + "hiddenSeries": false, + "id": 49, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "redfish_chassis_temperature_celsius{sensor_id=~\".*CPU1Temp\"} != 0", + "hide": false, + "interval": "", + "legendFormat": "{{ env }} {{ server }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Max CPU1 Temp", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fontSize": "100%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 34, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 8, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Count", + "align": "auto", + "colorMode": "row", + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "Value", + "preserveFormat": false, + "thresholds": [ + "0", + "1" + ], + "type": "number", + "unit": "short", + "valueMaps": [ + { + "text": "WARNING", + "value": "2" + }, + { + "text": "ERROR", + "value": "3" + }, + { + "text": "HEALTHY", + "value": "1" + } + ] + }, + { + "alias": "BMC", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "", + "linkUrl": "https://${__cell:raw}", + "mappingType": 1, + "pattern": "instance", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "__name__", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "job", + "thresholds": [], + "type": "hidden", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort(redfish_logservices_entry_count{name=\"SEL Log Service\", severity!=\"OK\"})", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Errors in event log", + "transform": "table", + "type": "table" + }, + { + "collapsed": false, + "datasource": "Prometheus", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 29, + "panels": [], + "repeat": "server", + "scopedVars": { + "server": { + "selected": true, + "text": "cpu-e-1041", + "value": "cpu-e-1041" + } + }, + "title": "$server", + "type": "row" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "max": 500, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 250 + }, + { + "color": "red", + "value": 400 + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 58 + }, + "id": 19, + "options": { + "fieldOptions": { + "calcs": [ + "mean" + ], + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [], + "values": false + }, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "6.6.1", + "scopedVars": { + "server": { + "selected": true, + "text": "cpu-e-1041", + "value": "cpu-e-1041" + } + }, + "targets": [ + { + "expr": "redfish_chassis_power_average_consumed_watts{server=~\"$server\"}", + "hide": false, + "interval": "", + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Power comsumption", + "type": "gauge" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "displayName": "Disks", + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 8, + "y": 58 + }, + "id": 24, + "options": { + "fieldOptions": { + "calcs": [ + "mean" + ], + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [], + "values": false + }, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "min" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "6.6.1", + "scopedVars": { + "server": { + "selected": true, + "text": "cpu-e-1041", + "value": "cpu-e-1041" + } + }, + "targets": [ + { + "expr": "count(redfish_system_storage_drive_state{server=~\"$server\"} != 1) or vector(0)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk with errors", + "type": "gauge" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "displayName": "Controllers", + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 10, + "y": 58 + }, + "id": 25, + "options": { + "fieldOptions": { + "calcs": [ + "mean" + ], + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [], + "values": false + }, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "min" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "6.6.1", + "scopedVars": { + "server": { + "selected": true, + "text": "cpu-e-1041", + "value": "cpu-e-1041" + } + }, + "targets": [ + { + "expr": "count(redfish_system_pcie_device_health_state{server=~\"$server\"} != 1) or vector(0)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "PCI-E with errors", + "type": "gauge" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "displayName": "Fans", + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 12, + "y": 58 + }, + "id": 26, + "options": { + "fieldOptions": { + "calcs": [ + "mean" + ], + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [], + "values": false + }, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "min" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "6.6.1", + "scopedVars": { + "server": { + "selected": true, + "text": "cpu-e-1041", + "value": "cpu-e-1041" + } + }, + "targets": [ + { + "expr": "count(redfish_chassis_temperature_sensor_state{server=~\"$server\"} != 1) or vector(0)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Sensors with errors", + "type": "gauge" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "displayName": "Power Supply", + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 14, + "y": 58 + }, + "id": 27, + "options": { + "fieldOptions": { + "calcs": [ + "mean" + ], + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [], + "values": false + }, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "min" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "6.6.1", + "scopedVars": { + "server": { + "selected": true, + "text": "cpu-e-1041", + "value": "cpu-e-1041" + } + }, + "targets": [ + { + "expr": "count(redfish_chassis_power_powersupply_health{server=~\"$server\"} > 1) or vector(0)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "PS with errors", + "type": "gauge" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "max": 80, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 50 + }, + { + "color": "red", + "value": 70 + } + ] + }, + "unit": "degree" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 58 + }, + "id": 17, + "options": { + "fieldOptions": { + "calcs": [ + "mean" + ], + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [], + "values": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "6.6.1", + "scopedVars": { + "server": { + "selected": true, + "text": "cpu-e-1041", + "value": "cpu-e-1041" + } + }, + "targets": [ + { + "expr": "redfish_chassis_temperature_celsius{server=~\"$server\"}", + "interval": "", + "legendFormat": "{{sensor}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Temperatures", + "type": "gauge" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorPostfix": false, + "colorPrefix": false, + "colorValue": false, + "colors": [ + "#d44a3a", + "#299c46", + "#299c46" + ], + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 0, + "y": 62 + }, + "id": 6, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "scopedVars": { + "server": { + "selected": true, + "text": "cpu-e-1041", + "value": "cpu-e-1041" + } + }, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false, + "ymax": null, + "ymin": null + }, + "tableColumn": "", + "targets": [ + { + "expr": "redfish_system_power_state{server=~\"$server\"}", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "0,2", + "timeFrom": null, + "timeShift": null, + "title": "Power Status", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "Up", + "value": "1" + }, + { + "op": "=", + "text": "Down", + "value": "2" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorPostfix": false, + "colorPrefix": false, + "colorValue": false, + "colors": [ + "#299c46", + "#299c46", + "#d44a3a" + ], + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 2, + "y": 62 + }, + "id": 7, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "scopedVars": { + "server": { + "selected": true, + "text": "cpu-e-1041", + "value": "cpu-e-1041" + } + }, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false, + "ymax": null, + "ymin": null + }, + "tableColumn": "", + "targets": [ + { + "expr": "redfish_chassis_health{server=~\"$server\"}", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "1,2", + "timeFrom": null, + "timeShift": null, + "title": "General Health", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "OK", + "value": "1" + }, + { + "op": "=", + "text": "WARNING", + "value": "2" + }, + { + "op": "=", + "text": "CRITICAL", + "value": "3" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorPostfix": false, + "colorPrefix": false, + "colorValue": false, + "colors": [ + "#d44a3a", + "#299c46", + "#299c46" + ], + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 6, + "y": 62 + }, + "id": 8, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "scopedVars": { + "server": { + "selected": true, + "text": "cpu-e-1041", + "value": "cpu-e-1041" + } + }, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false, + "ymax": null, + "ymin": null + }, + "tableColumn": "", + "targets": [ + { + "expr": "redfish_manager_health_state{server=~\"$server\"}", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "1,2", + "timeFrom": null, + "timeShift": null, + "title": "BMC Health", + "type": "singlestat", + "valueFontSize": "70%", + "valueMaps": [ + { + "op": "=", + "text": "Healthy", + "value": "1" + }, + { + "op": "=", + "text": "Warning", + "value": "2" + }, + { + "op": "=", + "text": "Critical", + "value": "3" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 1, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 14, + "w": 8, + "x": 8, + "y": 62 + }, + "hiddenSeries": false, + "id": 4, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 3, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "scopedVars": { + "server": { + "selected": true, + "text": "cpu-e-1041", + "value": "cpu-e-1041" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "redfish_chassis_fan_rpm_percentage{server=~\"$server\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{fan}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "lt", + "value": 10000, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Fans", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "rpm", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "max": 80, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 50 + }, + { + "color": "red", + "value": 70 + } + ] + }, + "unit": "degree" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 62 + }, + "hiddenSeries": false, + "id": 45, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "scopedVars": { + "server": { + "selected": true, + "text": "cpu-e-1041", + "value": "cpu-e-1041" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "redfish_chassis_temperature_celsius{server=~\"$server\"}", + "interval": "", + "legendFormat": "{{sensor}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Temperatures", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "degree", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "columns": [], + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fontSize": "100%", + "gridPos": { + "h": 12, + "w": 8, + "x": 0, + "y": 64 + }, + "id": 2, + "interval": "", + "links": [], + "options": {}, + "pageSize": null, + "pluginVersion": "6.7.3", + "repeat": null, + "scopedVars": { + "server": { + "selected": true, + "text": "cpu-e-1041", + "value": "cpu-e-1041" + } + }, + "showHeader": true, + "sort": { + "col": 7, + "desc": false + }, + "styles": [ + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "__name__", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "instance", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "job", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "env", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Status", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value", + "preserveFormat": false, + "sanitize": false, + "thresholds": [ + "0", + "1" + ], + "type": "string", + "unit": "short", + "valueMaps": [ + { + "text": "Enabled", + "value": "1" + }, + { + "text": "Disabled", + "value": "2" + }, + { + "text": "StandbyOffinline", + "value": "3" + }, + { + "text": "StandbySpare", + "value": "4" + }, + { + "text": "InTest", + "value": "5" + }, + { + "text": "Starting", + "value": "6" + }, + { + "text": "Absent", + "value": "7" + }, + { + "text": "UnavailableOffline", + "value": "8" + }, + { + "text": "Deferring", + "value": "9" + }, + { + "text": "Quiesced", + "value": "10" + }, + { + "text": "Updating", + "value": "11" + } + ] + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "resource", + "thresholds": [], + "type": "hidden", + "unit": "short" + } + ], + "targets": [ + { + "expr": "redfish_system_storage_drive_state{server=~\"$server\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk states / health", + "transform": "table", + "type": "table" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 0, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 69 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "scopedVars": { + "server": { + "selected": true, + "text": "cpu-e-1041", + "value": "cpu-e-1041" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "redfish_chassis_power_average_consumed_watts{server=~\"$server\"}", + "interval": "", + "legendFormat": "{{resource}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Power comsumption", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 22, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "cpu-e-1041", + "value": "cpu-e-1041" + }, + "datasource": "Prometheus", + "definition": "label_values(redfish_exporter_collector_duration_seconds, server)", + "hide": 0, + "includeAll": false, + "label": "server", + "multi": false, + "name": "server", + "options": [], + "query": "label_values(redfish_exporter_collector_duration_seconds, server)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-30m", + "to": "now-5m" + }, + "timepicker": { + "nowDelay": "1m", + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Redfish exporter", + "uid": "b02mElQGX", + "version": 7 +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/grafana_home_dashboard.json b/etc/kayobe/kolla/config/grafana/grafana_home_dashboard.json new file mode 100644 index 000000000..6c6236b24 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/grafana_home_dashboard.json @@ -0,0 +1,91 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 120, + "links": [], + "panels": [ + { + "datasource": null, + "gridPos": { + "h": 18, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "alertName": "", + "dashboardAlerts": false, + "folder": null, + "maxItems": 20, + "showInstances": false, + "sortOrder": 1, + "stateFilter": { + "firing": true, + "inactive": false, + "pending": true + } + }, + "title": "Alerts", + "type": "alertlist" + }, + { + "datasource": null, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 4, + "options": { + "folderId": null, + "maxItems": 10, + "query": "", + "showHeadings": true, + "showRecentlyViewed": false, + "showSearch": false, + "showStarred": true, + "tags": [] + }, + "pluginVersion": "8.1.6", + "title": "Dashboards", + "type": "dashlist" + } + ], + "schemaVersion": 30, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Home Dashboard", + "uid": "RC7ADbDnk", + "version": 4 +} diff --git a/etc/kayobe/kolla/config/grafana/provisioning.yaml b/etc/kayobe/kolla/config/grafana/provisioning.yaml new file mode 100644 index 000000000..07dfa91dd --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/provisioning.yaml @@ -0,0 +1,15 @@ +--- +# # config file version +apiVersion: 1 + +providers: + - name: Ceph + folder: Ceph + type: file + options: + path: /var/lib/grafana/dashboards/ceph + - name: OpenStack + folder: OpenStack + type: file + options: + path: /var/lib/grafana/dashboards/openstack diff --git a/etc/kayobe/kolla/config/prometheus/blackbox.rules b/etc/kayobe/kolla/config/prometheus/blackbox.rules new file mode 100644 index 000000000..5f91d07e7 --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/blackbox.rules @@ -0,0 +1,77 @@ +{% raw %} + +groups: +- name: Blackbox + rules: + - alert: BlackboxProbeFailed + expr: probe_success == 0 + for: 0m + labels: + severity: critical + annotations: + summary: Blackbox probe failed (instance {{ $labels.instance }}) + description: "Probe failed" + + - alert: BlackboxSlowProbe + expr: avg_over_time(probe_duration_seconds[1m]) > 1 + for: 1m + labels: + severity: warning + annotations: + summary: Blackbox slow probe (instance {{ $labels.instance }}) + description: "Blackbox probe took more than 1s to complete" + + - alert: BlackboxProbeHttpFailure + expr: probe_http_status_code <= 199 OR probe_http_status_code >= 400 + for: 0m + labels: + severity: critical + annotations: + summary: Blackbox probe HTTP failure (instance {{ $labels.instance }}) + description: "HTTP status code is not 200-399" + + - alert: BlackboxSslCertificateWillExpireSoon + expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 30 + for: 0m + labels: + severity: warning + annotations: + summary: Blackbox SSL certificate will expire soon (instance {{ $labels.instance }}) + description: "SSL certificate expires in 30 days" + + - alert: BlackboxSslCertificateWillExpireSoon + expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 3 + for: 0m + labels: + severity: critical + annotations: + summary: Blackbox SSL certificate will expire soon (instance {{ $labels.instance }}) + description: "SSL certificate expires in 3 days" + + - alert: BlackboxSslCertificateExpired + expr: probe_ssl_earliest_cert_expiry - time() <= 0 + for: 0m + labels: + severity: critical + annotations: + summary: Blackbox SSL certificate expired (instance {{ $labels.instance }}) + description: "SSL certificate has expired already" + + - alert: BlackboxProbeSlowHttp + expr: avg_over_time(probe_http_duration_seconds[1m]) > 1 + for: 1m + labels: + severity: warning + annotations: + summary: Blackbox probe slow HTTP (instance {{ $labels.instance }}) + description: "HTTP request took more than 1s" + + - alert: BlackboxProbeSlowPing + expr: avg_over_time(probe_icmp_duration_seconds[1m]) > 1 + for: 1m + labels: + severity: warning + annotations: + summary: Blackbox probe slow ping (instance {{ $labels.instance }}) + description: "Blackbox ping took more than 1s" +{% endraw %} diff --git a/etc/kayobe/kolla/config/prometheus/cadvisor.rules b/etc/kayobe/kolla/config/prometheus/cadvisor.rules new file mode 100644 index 000000000..d97a3a545 --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/cadvisor.rules @@ -0,0 +1,54 @@ +# Taken from https://awesome-prometheus-alerts.grep.to/rules + +{% raw %} + +groups: +- name: Cadvisor + rules: + + - alert: ContainerKilled + expr: time() - container_last_seen > 61 + for: 5m + labels: + severity: warning + annotations: + summary: "Container killed (instance {{ $labels.instance }})" + description: "A container has disappeared" + + - alert: ContainerCpuUsage + expr: (sum(rate(container_cpu_usage_seconds_total{name!=""}[3m])) BY (instance, name) * 100) > 3000 + for: 2m + labels: + severity: warning + annotations: + summary: "Container CPU usage (instance {{ $labels.instance }})" + description: "Container CPU usage is above 300%" + + - alert: ContainerVolumeUsage + expr: (1 - (sum(container_fs_inodes_free{name!=""}) BY (instance) / sum(container_fs_inodes_total{name!=""}) BY (instance)) * 100) > 80 + for: 1m + labels: + severity: warning + annotations: + summary: "Container Volume usage (instance {{ $labels.instance }})" + description: "Container Volume usage is above 80%" + + - alert: ContainerVolumeIoUsage + expr: (sum(container_fs_io_current{name!=""}) BY (instance, name) * 100) > 80 + for: 5m + labels: + severity: warning + annotations: + summary: "Container Volume IO usage (instance {{ $labels.instance }})" + description: "Container Volume IO usage is above 80%" + + - alert: ContainerHighThrottleRate + expr: rate(container_cpu_cfs_throttled_seconds_total[3m]) > 1 + for: 1m + labels: + severity: warning + annotations: + summary: "Container high throttle rate (instance {{ $labels.instance }})" + description: "Container is being throttled" + +{% endraw %} diff --git a/etc/kayobe/kolla/config/prometheus/ceph.rules b/etc/kayobe/kolla/config/prometheus/ceph.rules new file mode 100644 index 000000000..ccbc95da1 --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/ceph.rules @@ -0,0 +1,222 @@ +# Modified official set of upstream alerts https://github.com/ceph/ceph/blob/octopus/monitoring/prometheus/alerts/ceph_default_alerts.yml + +{% raw %} + +groups: +- name: health + rules: + - alert: HealthError + expr: ceph_health_status == 2 + for: 5m + labels: + severity: critical + annotations: + description: > + Ceph in HEALTH_ERROR state for more than 5 minutes. + Please check "ceph health detail" for more information. + + - alert: HealthWarn + expr: ceph_health_status == 1 + for: 15m + labels: + severity: warning + annotations: + description: > + Ceph has been in HEALTH_WARN for more than 15 minutes. + Please check "ceph health detail" for more information. + +- name: mon + rules: + - alert: LowMonitorQuorumCount + expr: sum(ceph_mon_quorum_status) < 3 + labels: + severity: critical + annotations: + description: | + Monitor count in quorum is below three. + + Only {{ $value }} of {{ with query "count(ceph_mon_quorum_status)" }}{{ . | first | value }}{{ end }} monitors are active. + + The following monitors are down: + {{- range query "(ceph_mon_quorum_status == 0) + on(ceph_daemon) group_left(hostname) (ceph_mon_metadata * 0)" }} + - {{ .Labels.ceph_daemon }} on {{ .Labels.hostname }} + {{- end }} + +- name: osd + rules: + - alert: 10percentOSDsDown + expr: count(ceph_osd_up == 0) / count(ceph_osd_up) * 100 >= 10 + labels: + severity: critical + annotations: + description: | + {{ $value | humanize }}% or {{ with query "count(ceph_osd_up == 0)" }}{{ . | first | value }}{{ end }} of {{ with query "count(ceph_osd_up)" }}{{ . | first | value }}{{ end }} OSDs are down (≥ 10%). + + The following OSDs are down: + {{- range query "(ceph_osd_up * on(ceph_daemon) group_left(hostname) ceph_osd_metadata) == 0" }} + - {{ .Labels.ceph_daemon }} on {{ .Labels.hostname }} + {{- end }} + + - alert: OSDDdown + expr: count(ceph_osd_up == 0) > 0 + for: 15m + labels: + severity: warning + annotations: + description: | + {{ $s := "" }}{{ if gt $value 1.0 }}{{ $s = "s" }}{{ end }} + {{ $value }} OSD{{ $s }} down for more than 15 minutes. + + {{ $value }} of {{ query "count(ceph_osd_up)" | first | value }} OSDs are down. + + The following OSD{{ $s }} {{ if eq $s "" }}is{{ else }}are{{ end }} down: + {{- range query "(ceph_osd_up * on(ceph_daemon) group_left(hostname) ceph_osd_metadata) == 0"}} + - {{ .Labels.ceph_daemon }} on {{ .Labels.hostname }} + {{- end }} + + - alert: OSDsNearFull + expr: | + ( + ((ceph_osd_stat_bytes_used / ceph_osd_stat_bytes) and on(ceph_daemon) ceph_osd_up == 1) + * on(ceph_daemon) group_left(hostname) ceph_osd_metadata + ) * 100 > 90 + for: 5m + labels: + severity: critical + annotations: + description: > + OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} is + dangerously full: {{ $value | humanize }}% + + - alert: FlappingOSD + expr: | + ( + rate(ceph_osd_up[5m]) + * on(ceph_daemon) group_left(hostname) ceph_osd_metadata + ) * 60 > 1 + labels: + severity: warning + annotations: + description: > + OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was + marked down and back up at {{ $value | humanize }} times once a + minute for 5 minutes. + + # alert on high deviation from average PG count + - alert: HighPgCountDeviation + expr: | + abs( + ( + (ceph_osd_numpg > 0) - on (job) group_left avg(ceph_osd_numpg > 0) by (job) + ) / on (job) group_left avg(ceph_osd_numpg > 0) by (job) + ) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata > 0.35 + for: 5m + labels: + severity: warning + annotations: + description: > + OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates + by more than 30% from average PG count. + +- name: pgs + rules: + - alert: PgsInactive + expr: ceph_pool_metadata * on(pool_id,instance) group_left() (ceph_pg_total - ceph_pg_active) > 0 + for: 5m + labels: + severity: critical + annotations: + description: > + {{ $value }} PGs have been inactive for more than 5 minutes in pool {{ $labels.name }}. + Inactive placement groups aren't able to serve read/write + requests. + + - alert: PgsUnclean + expr: ceph_pool_metadata * on(pool_id,instance) group_left() (ceph_pg_total - ceph_pg_clean) > 0 + for: 15m + labels: + severity: warning + annotations: + description: > + {{ $value }} PGs haven't been clean for more than 15 minutes in pool {{ $labels.name }}. + Unclean PGs haven't been able to completely recover from a + previous failure. + +- name: nodes + rules: + - alert: RootVolumeFull + expr: node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"} * 100 < 5 + for: 5m + labels: + severity: critical + annotations: + description: "Root volume (OSD and MON store) is dangerously full: {{ $value | humanize }}% free." + + # alert on nic packet errors and drops rates > 1 packet/s + - alert: NetworkPacketsDropped + expr: irate(node_network_receive_drop_total{device=~"eno.*|ens.*|eth.*|enp.*"}[5m]) + irate(node_network_transmit_drop_total{device=~"eno.*|ens.*|eth.*|enp.*"}[5m]) > 1 + labels: + severity: warning + annotations: + description: > + Node {{ $labels.instance }} experiences packet drop > 1 + packet/s on interface {{ $labels.device }}. + + - alert: NetworkPacketErrors + expr: | + irate(node_network_receive_errs_total{device!="lo"}[5m]) + + irate(node_network_transmit_errs_total{device!="lo"}[5m]) > 1 + labels: + severity: warning + annotations: + description: > + Node {{ $labels.instance }} experiences packet errors > 1 + packet/s on interface {{ $labels.device }}. + + - alert: StorageFillingUp + expr: | + predict_linear(node_filesystem_avail_bytes[2d], 3600 * 24 * 5) * + on(instance) group_left(nodename) node_uname_info < 0 + labels: + severity: warning + annotations: + description: > + Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} + will be full in less than 5 days assuming the average fill-up + rate of the past 48 hours. + +- name: pools + rules: + - alert: PoolFull + expr: | + ceph_pool_stored / (ceph_pool_stored + ceph_pool_max_avail) + * on(pool_id) group_right ceph_pool_metadata * 100 > 90 + labels: + severity: critical + annotations: + description: "Pool {{ $labels.name }} at {{ $value | humanize }}% capacity." + + - alert: PoolFillingUp + expr: | + ( + predict_linear(ceph_pool_stored[2d], 3600 * 24 * 5) + >= ceph_pool_stored + ceph_pool_max_avail + ) * on(pool_id) group_left(name) ceph_pool_metadata + labels: + severity: warning + annotations: + description: > + Pool {{ $labels.name }} will be full in less than 5 days + assuming the average fill-up rate of the past 48 hours. + +- name: healthchecks + rules: + - alert: SlowOSDOps + expr: ceph_healthcheck_slow_ops > 0 + for: 30s + labels: + severity: warning + annotations: + description: "{{ $value }} OSD requests are taking too long to process (osd_op_complaint_time exceeded)" + +{% endraw %} diff --git a/etc/kayobe/kolla/config/prometheus/elasticsearch.rules b/etc/kayobe/kolla/config/prometheus/elasticsearch.rules new file mode 100644 index 000000000..974bf4e99 --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/elasticsearch.rules @@ -0,0 +1,99 @@ +# Taken from https://awesome-prometheus-alerts.grep.to/rules + +{% raw %} + +groups: +- name: Elasticsearch + rules: + + - alert: ElasticsearchHeapUsageTooHigh + expr: (elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 90 + for: 2m + labels: + severity: critical + annotations: + summary: "Elasticsearch Heap Usage Too High (instance {{ $labels.instance }})" + description: "The heap usage is over 90%" + + - alert: ElasticsearchHeapUsageWarning + expr: (elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 80 + for: 2m + labels: + severity: warning + annotations: + summary: "Elasticsearch Heap Usage warning (instance {{ $labels.instance }})" + description: "The heap usage is over 80%" + + - alert: ElasticsearchDiskOutOfSpace + expr: elasticsearch_filesystem_data_available_bytes / elasticsearch_filesystem_data_size_bytes * 100 < 10 + for: 0m + labels: + severity: critical + annotations: + summary: "Elasticsearch disk out of space (instance {{ $labels.instance }})" + description: "The disk usage is over 90%" + + - alert: ElasticsearchDiskSpaceLow + expr: elasticsearch_filesystem_data_available_bytes / elasticsearch_filesystem_data_size_bytes * 100 < 20 + for: 2m + labels: + severity: warning + annotations: + summary: "Elasticsearch disk space low (instance {{ $labels.instance }})" + description: "The disk usage is over 80%" + + - alert: ElasticsearchClusterRed + expr: elasticsearch_cluster_health_status{color="red"} == 1 + for: 0m + labels: + severity: critical + annotations: + summary: "Elasticsearch Cluster Red (instance {{ $labels.instance }})" + description: "Elastic Cluster Red status" + + - alert: ElasticsearchClusterYellow + expr: elasticsearch_cluster_health_status{color="yellow"} == 1 + for: 5m + labels: + severity: warning + annotations: + summary: "Elasticsearch Cluster Yellow (instance {{ $labels.instance }})" + description: "Elastic Cluster Yellow status" + + - alert: ElasticsearchRelocatingShardsTooLong + expr: elasticsearch_cluster_health_relocating_shards > 0 + for: 15m + labels: + severity: warning + annotations: + summary: "Elasticsearch relocating shards too long (instance {{ $labels.instance }})" + description: "Elasticsearch has been relocating shards for 15min" + + - alert: ElasticsearchInitializingShardsTooLong + expr: elasticsearch_cluster_health_initializing_shards > 0 + for: 15m + labels: + severity: warning + annotations: + summary: "Elasticsearch initializing shards too long (instance {{ $labels.instance }})" + description: "Elasticsearch has been initializing shards for 15 min" + + - alert: ElasticsearchUnassignedShards + expr: elasticsearch_cluster_health_unassigned_shards > 0 + for: 0m + labels: + severity: critical + annotations: + summary: "Elasticsearch unassigned shards (instance {{ $labels.instance }})" + description: "Elasticsearch has unassigned shards" + + - alert: ElasticsearchPendingTasks + expr: elasticsearch_cluster_health_number_of_pending_tasks > 0 + for: 15m + labels: + severity: warning + annotations: + summary: "Elasticsearch pending tasks (instance {{ $labels.instance }})" + description: "Elasticsearch has pending tasks. Cluster works slowly." + +{% endraw %} diff --git a/etc/kayobe/kolla/config/prometheus/haproxy.rules b/etc/kayobe/kolla/config/prometheus/haproxy.rules new file mode 100644 index 000000000..090f6e37a --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/haproxy.rules @@ -0,0 +1,36 @@ +# Taken from https://awesome-prometheus-alerts.grep.to/rules + +{% raw %} + +groups: +- name: HAProxy + rules: + + - alert: HaproxyDown + expr: haproxy_up == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "HAProxy down (instance {{ $labels.instance }})" + description: "HAProxy down" + + - alert: HaproxyBackendDown + expr: haproxy_backend_up == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "HAProxy backend down (instance {{ $labels.instance }})" + description: "HAProxy backend is down" + + - alert: HaproxyServerDown + expr: haproxy_server_up == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "HAProxy server down (instance {{ $labels.instance }})" + description: "HAProxy server is down" + +{% endraw %} diff --git a/etc/kayobe/kolla/config/prometheus/mysql.rules b/etc/kayobe/kolla/config/prometheus/mysql.rules new file mode 100644 index 000000000..26220ae8b --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/mysql.rules @@ -0,0 +1,81 @@ +# Taken from https://awesome-prometheus-alerts.grep.to/rules + +{% raw %} + +groups: +- name: Mysql + rules: + + - alert: MysqlDown + expr: mysql_up == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "MySQL down (instance {{ $labels.instance }})" + description: "MySQL instance is down on {{ $labels.instance }}" + + - alert: MysqlTooManyConnections + expr: avg by (instance) (max_over_time(mysql_global_status_threads_connected[5m])) / avg by (instance) (mysql_global_variables_max_connections) * 100 > 80 + for: 1m + labels: + severity: warning + annotations: + summary: "MySQL too many connections (instance {{ $labels.instance }})" + description: "More than 80% of MySQL connections are in use on {{ $labels.instance }}" + + - alert: MysqlHighThreadsRunning + expr: avg by (instance) (max_over_time(mysql_global_status_threads_running[5m])) / avg by (instance) (mysql_global_variables_max_connections) * 100 > 60 + for: 1m + labels: + severity: warning + annotations: + summary: "MySQL high threads running (instance {{ $labels.instance }})" + description: "More than 60% of MySQL connections are in running state on {{ $labels.instance }}" + + - alert: MysqlSlaveIoThreadNotRunning + expr: mysql_slave_status_master_server_id > 0 and ON (instance) mysql_slave_status_slave_io_running == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "MySQL Slave IO thread not running (instance {{ $labels.instance }})" + description: "MySQL Slave IO thread not running on {{ $labels.instance }}" + + - alert: MysqlSlaveSqlThreadNotRunning + expr: mysql_slave_status_master_server_id > 0 and ON (instance) mysql_slave_status_slave_sql_running == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "MySQL Slave SQL thread not running (instance {{ $labels.instance }})" + description: "MySQL Slave SQL thread not running on {{ $labels.instance }}" + + - alert: MysqlSlaveReplicationLag + expr: mysql_slave_status_master_server_id > 0 and ON (instance) (mysql_slave_status_seconds_behind_master - mysql_slave_status_sql_delay) > 300 + for: 1m + labels: + severity: warning + annotations: + summary: "MySQL Slave replication lag (instance {{ $labels.instance }})" + description: "MysqL replication lag on {{ $labels.instance }}" + + - alert: MysqlSlowQueries + expr: increase(mysql_global_status_slow_queries[1m]) > 0 + for: 2m + labels: + severity: warning + annotations: + summary: "MySQL slow queries (instance {{ $labels.instance }})" + description: "MySQL server is having some slow queries." + + - alert: MysqlRestarted + expr: mysql_global_status_uptime < 60 + for: 1m + labels: + severity: warning + annotations: + summary: "MySQL restarted (instance {{ $labels.instance }})" + description: "MySQL has just been restarted, less than one minute ago on {{ $labels.instance }}." + +{% endraw %} diff --git a/etc/kayobe/kolla/config/prometheus/notifications.tmpl b/etc/kayobe/kolla/config/prometheus/notifications.tmpl new file mode 100644 index 000000000..b1abadc99 --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/notifications.tmpl @@ -0,0 +1,32 @@ +{% raw %} + +{{ define "slack.default.title" -}} + [{{ .Status | toUpper -}} + {{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{- end -}} + ] {{ .CommonLabels.alertname }} +{{- end }} + +{{ define "slack.default.text" -}} + +{{ range .Alerts }} +*Alert:* {{ .Annotations.summary }} - `{{ .Labels.severity }}` +*Description:* {{ .Annotations.description }} +*Details:* + {{ range .Labels.SortedPairs -}} + • *{{ .Name }}:* `{{ .Value }}` + {{ end }} +{{ end }} + +{{ end }} + +{{ define "__alert_silence_link" -}} + {{ .ExternalURL }}/#/silences/new?filter=%7B + {{- range .CommonLabels.SortedPairs -}} + {{- if ne .Name "alertname" -}} + {{- .Name }}%3D"{{- .Value -}}"%2C%20 + {{- end -}} + {{- end -}} + alertname%3D"{{ .CommonLabels.alertname }}"%7D +{{- end }} + +{% endraw %} diff --git a/etc/kayobe/kolla/config/prometheus/openstack.rules b/etc/kayobe/kolla/config/prometheus/openstack.rules new file mode 100644 index 000000000..78a7f8282 --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/openstack.rules @@ -0,0 +1,15 @@ +{% raw %} + +groups: +- name: OpenStack + rules: + - alert: OpenStackServiceDown + expr: (sum({__name__=~"openstack.+_state", job="openstack_exporter"} == 0) by (hostname, service)) + for: 1m + labels: + severity: alert + annotations: + summary: "{{ $labels.service }} at {{ $labels.hostname }} is down" + description: "OpenStack service {{ $labels.service }} at {{ $labels.hostname }} is down" + +{% endraw %} diff --git a/etc/kayobe/kolla/config/prometheus/prometheus.rules b/etc/kayobe/kolla/config/prometheus/prometheus.rules new file mode 100644 index 000000000..c9803946a --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/prometheus.rules @@ -0,0 +1,225 @@ +# Taken from https://awesome-prometheus-alerts.grep.to/rules + +{% raw %} + +groups: +- name: Prometheus + rules: + + - alert: PrometheusTargetMissing + expr: up == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Prometheus target missing (instance {{ $labels.instance }})" + description: "A Prometheus target has disappeared. An exporter might have crashed." + + - alert: PrometheusAllTargetsMissing + expr: count by (job) (up) == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Prometheus all targets missing (instance {{ $labels.instance }})" + description: "A Prometheus job does not have living target anymore." + + - alert: PrometheusConfigurationReloadFailure + expr: prometheus_config_last_reload_successful != 1 + for: 1m + labels: + severity: warning + annotations: + summary: "Prometheus configuration reload failure (instance {{ $labels.instance }})" + description: "Prometheus configuration reload error" + + - alert: PrometheusTooManyRestarts + expr: changes(process_start_time_seconds{job=~"prometheus|pushgateway|alertmanager"}[15m]) > 2 + for: 1m + labels: + severity: warning + annotations: + summary: "Prometheus too many restarts (instance {{ $labels.instance }})" + description: "Prometheus has restarted more than twice in the last 15 minutes. It might be crashlooping." + + - alert: PrometheusAlertmanagerConfigurationReloadFailure + expr: alertmanager_config_last_reload_successful != 1 + for: 1m + labels: + severity: warning + annotations: + summary: "Prometheus AlertManager configuration reload failure (instance {{ $labels.instance }})" + description: "AlertManager configuration reload error" + + - alert: PrometheusAlertmanagerConfigNotSynced + expr: count(count_values("config_hash", alertmanager_config_hash)) > 1 + for: 1m + labels: + severity: warning + annotations: + summary: "Prometheus AlertManager config not synced (instance {{ $labels.instance }})" + description: "Configurations of AlertManager cluster instances are out of sync" + + # - alert: PrometheusAlertmanagerE2eDeadManSwitch + # expr: vector(1) + # for: 1m + # labels: + # severity: critical + # annotations: + # summary: "Prometheus AlertManager E2E dead man switch (instance {{ $labels.instance }})" + # description: "Prometheus DeadManSwitch is an always-firing alert. It's used as an end-to-end test of Prometheus through the Alertmanager." + + - alert: PrometheusNotConnectedToAlertmanager + expr: prometheus_notifications_alertmanagers_discovered < 1 + for: 1m + labels: + severity: critical + annotations: + summary: "Prometheus not connected to alertmanager (instance {{ $labels.instance }})" + description: "Prometheus cannot connect the alertmanager" + + - alert: PrometheusRuleEvaluationFailures + expr: increase(prometheus_rule_evaluation_failures_total[3m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Prometheus rule evaluation failures (instance {{ $labels.instance }})" + description: "Prometheus encountered {{ $value }} rule evaluation failures, leading to potentially ignored alerts." + + - alert: PrometheusTemplateTextExpansionFailures + expr: increase(prometheus_template_text_expansion_failures_total[3m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Prometheus template text expansion failures (instance {{ $labels.instance }})" + description: "Prometheus encountered {{ $value }} template text expansion failures" + + - alert: PrometheusRuleEvaluationSlow + expr: prometheus_rule_group_last_duration_seconds > prometheus_rule_group_interval_seconds + for: 1m + labels: + severity: warning + annotations: + summary: "Prometheus rule evaluation slow (instance {{ $labels.instance }})" + description: "Prometheus rule evaluation took more time than the scheduled interval. I indicates a slower storage backend access or too complex query." + + - alert: PrometheusNotificationsBacklog + expr: min_over_time(prometheus_notifications_queue_length[10m]) > 0 + for: 1m + labels: + severity: warning + annotations: + summary: "Prometheus notifications backlog (instance {{ $labels.instance }})" + description: "The Prometheus notification queue has not been empty for 10 minutes" + + - alert: PrometheusAlertmanagerNotificationFailing + expr: rate(alertmanager_notifications_failed_total[1m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Prometheus AlertManager notification failing (instance {{ $labels.instance }})" + description: "Alertmanager is failing sending notifications" + + - alert: PrometheusTargetEmpty + expr: prometheus_sd_discovered_targets == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Prometheus target empty (instance {{ $labels.instance }})" + description: "Prometheus has no target in service discovery" + + - alert: PrometheusTargetScrapingSlow + expr: prometheus_target_interval_length_seconds{quantile="0.9",interval="1m0s"} > 61 + for: 1m + labels: + severity: warning + annotations: + summary: "Prometheus target scraping slow (instance {{ $labels.instance }})" + description: "Prometheus is scraping exporters slowly" + + - alert: PrometheusLargeScrape + expr: increase(prometheus_target_scrapes_exceeded_sample_limit_total[10m]) > 10 + for: 1m + labels: + severity: warning + annotations: + summary: "Prometheus large scrape (instance {{ $labels.instance }})" + description: "Prometheus has many scrapes that exceed the sample limit" + + - alert: PrometheusTargetScrapeDuplicate + expr: increase(prometheus_target_scrapes_sample_duplicate_timestamp_total[5m]) > 0 + for: 1m + labels: + severity: warning + annotations: + summary: "Prometheus target scrape duplicate (instance {{ $labels.instance }})" + description: "Prometheus has many samples rejected due to duplicate timestamps but different values" + + - alert: PrometheusTsdbCheckpointCreationFailures + expr: increase(prometheus_tsdb_checkpoint_creations_failed_total[3m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Prometheus TSDB checkpoint creation failures (instance {{ $labels.instance }})" + description: "Prometheus encountered {{ $value }} checkpoint creation failures" + + - alert: PrometheusTsdbCheckpointDeletionFailures + expr: increase(prometheus_tsdb_checkpoint_deletions_failed_total[3m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Prometheus TSDB checkpoint deletion failures (instance {{ $labels.instance }})" + description: "Prometheus encountered {{ $value }} checkpoint deletion failures" + + - alert: PrometheusTsdbCompactionsFailed + expr: increase(prometheus_tsdb_compactions_failed_total[3m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Prometheus TSDB compactions failed (instance {{ $labels.instance }})" + description: "Prometheus encountered {{ $value }} TSDB compactions failures" + + - alert: PrometheusTsdbHeadTruncationsFailed + expr: increase(prometheus_tsdb_head_truncations_failed_total[3m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Prometheus TSDB head truncations failed (instance {{ $labels.instance }})" + description: "Prometheus encountered {{ $value }} TSDB head truncation failures" + + - alert: PrometheusTsdbReloadFailures + expr: increase(prometheus_tsdb_reloads_failures_total[3m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Prometheus TSDB reload failures (instance {{ $labels.instance }})" + description: "Prometheus encountered {{ $value }} TSDB reload failures" + + - alert: PrometheusTsdbWalCorruptions + expr: increase(prometheus_tsdb_wal_corruptions_total[3m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Prometheus TSDB WAL corruptions (instance {{ $labels.instance }})" + description: "Prometheus encountered {{ $value }} TSDB WAL corruptions" + + - alert: PrometheusTsdbWalTruncationsFailed + expr: increase(prometheus_tsdb_wal_truncations_failed_total[3m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Prometheus TSDB WAL truncations failed (instance {{ $labels.instance }})" + description: "Prometheus encountered {{ $value }} TSDB WAL truncation failures" + +{% endraw %} diff --git a/etc/kayobe/kolla/config/prometheus/rabbitmq.rules b/etc/kayobe/kolla/config/prometheus/rabbitmq.rules new file mode 100644 index 000000000..30c18eb97 --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/rabbitmq.rules @@ -0,0 +1,71 @@ +{% raw %} + +groups: +- name: rabbitmq.rules + rules: + - alert: RabbitmqNodeDown + expr: sum(rabbitmq_build_info{host_name!=""}) < 3 + for: 30m + labels: + severity: critical + annotations: + description: Rabbitmq node down on {{ $labels.host_name }} + - alert: RabbitmqConsumersLowUtilization + expr: rabbitmq_queue_consumer_utilisation < 0.4 + for: 5m + labels: + severity: warning + annotations: + description: RabbitMQ consumers message consumption speed is low on {{ $labels.host_name }} + - alert: RabbitmqNodeNotDistributed + expr: erlang_vm_dist_node_state < 3 + for: 5m + labels: + severity: critical + annotations: + description: Rabbitmq node not distributed on node {{ $labels.host_name }} + - alert: RabbitmqMemoryHigh + expr: rabbitmq_process_resident_memory_bytes / rabbitmq_resident_memory_limit_bytes + * 100 > 90 + for: 2m + labels: + severity: warning + annotations: + description: Rabbitmq memory too high on {{ $labels.host_name }} + - alert: RabbitmqFileDescriptorsUsage + expr: rabbitmq_process_open_fds / rabbitmq_process_max_fds * 100 > 90 + for: 2m + labels: + severity: warning + annotations: + description: Rabbitmq file descriptors usage on {{ $labels.host_name }} + - alert: RabbitmqTooMuchUnack + expr: sum(rabbitmq_queue_messages_unacked) BY (queue) > 1000 + for: 1m + labels: + severity: warning + annotations: + description: Rabbitmq too much unack on {{ $labels.host_name }} + - alert: RabbitmqTooMuchConnections + expr: rabbitmq_connections > 1000 + for: 2m + labels: + severity: warning + annotations: + description: Rabbitmq too much connections on {{ $labels.host_name }} + - alert: RabbitmqNoQueueConsumer + expr: rabbitmq_queue_consumers < 1 + for: 1m + labels: + severity: warning + annotations: + description: Rabbitmq no queue consumer on {{ $labels.host_name }} + - alert: RabbitmqUnroutableMessages + expr: increase(rabbitmq_channel_messages_unroutable_returned_total[1m]) > 0 or increase(rabbitmq_channel_messages_unroutable_dropped_total[1m]) > 0 + for: 2m + labels: + severity: warning + annotations: + description: Rabbitmq unroutable messages on {{ $labels.host_name }} + +{% endraw %} diff --git a/etc/kayobe/kolla/config/prometheus/system.rules b/etc/kayobe/kolla/config/prometheus/system.rules new file mode 100644 index 000000000..22d090aae --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/system.rules @@ -0,0 +1,79 @@ +{% raw %} + +groups: +- name: Node + rules: + + - alert: LowDiskSpace + expr: ( ( node_filesystem_free_bytes - node_filesystem_avail_bytes ) / node_filesystem_free_bytes ) * 100 >= 80 + for: 1m + labels: + severity: alert + annotations: + summary: "Prometheus exporter at {{ $labels.instance }} reports low disk space" + description: "{{ $labels.device }} is {{ $value }}% full." + + - alert: LowMemory + expr: ( ( node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes - (node_memory_Hugepagesize_bytes * node_memory_HugePages_Total)) / (node_memory_MemTotal_bytes - (node_memory_Hugepagesize_bytes * node_memory_HugePages_Total))) * 100 >= 80 + for: 1m + labels: + severity: alert + annotations: + summary: "Prometheus exporter at {{ $labels.instance }} reports low memory" + description: "Memory is {{ $value }}% full." + + - alert: HostOomKillDetected + expr: increase(node_vmstat_oom_kill[5m]) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Host OOM kill detected (instance {{ $labels.instance }})" + description: "OOM kill detected" + + - alert: Overheating + expr: node_hwmon_temp_celsius >= 85 + for: 1m + labels: + severity: warning + annotations: + summary: "Prometheus exporter at {{ $labels.instance }} reports overheating" + description: "Sensor {{ $labels.chip }} reports {{ $value }} degrees celcius." + + - alert: HostNodeOvertemperatureAlarm + expr: node_hwmon_temp_crit_alarm_celsius == 1 + for: 5m + labels: + severity: critical + annotations: + summary: "Host node overtemperature alarm (instance {{ $labels.instance }})" + description: "Physical node temperature alarm triggered" + + - alert: InstanceDown + expr: up{job="node"} == 0 + for: 1m + labels: + severity: alert + annotations: + summary: "Instance {{$labels.instance}} down" + description: "{{$labels.instance}} has been down for more than 5 minutes." + + - alert: HostEdacCorrectableErrorsDetected + expr: increase(node_edac_correctable_errors_total[5m]) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Host EDAC Correctable Errors detected (instance {{ $labels.instance }})" + description: "{{ $labels.instance }} has had {{ printf \"%.0f\" $value }} correctable memory errors reported by EDAC in the last 5 minutes." + + - alert: HostEdacUncorrectableErrorsDetected + expr: node_edac_uncorrectable_errors_total > 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Host EDAC Uncorrectable Errors detected (instance {{ $labels.instance }})" + description: "{{ $labels.instance }} has had {{ printf \"%.0f\" $value }} uncorrectable memory errors reported by EDAC in the last 5 minutes." + +{% endraw %} From 5b7330fb2a5f80bb215f624ea2dfb12381a0e176 Mon Sep 17 00:00:00 2001 From: Piotr Parczewski Date: Wed, 5 Oct 2022 11:54:21 +0200 Subject: [PATCH 071/292] Use sane defaults for basic monitoring stack --- etc/kayobe/kolla.yml | 2 +- etc/kayobe/kolla/globals.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index cc60d1fe3..806ecfed8 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -524,7 +524,7 @@ kolla_enable_grafana: true #kolla_enable_keystone: #kolla_enable_keystone_federation: #kolla_enable_keystone_horizon_policy_file: -kolla_enable_kibana: true +#kolla_enable_kibana: #kolla_enable_kuryr: #kolla_enable_loadbalancer: #kolla_enable_magnum: diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 8adcb33a7..c7dc8bdeb 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -8,5 +8,6 @@ barbican_tag: xena-20220728T143658 prometheus_tag: xena-20220728T143658 {% endif %} -# Elasticsearch memory tuning +# Monitoring and alerting related settings es_heap_size: 8g +prometheus_cmdline_extras: "--storage.tsdb.retention.time=30d" From 9a9efd2369c94ad33063c79b951a31f8dd25b333 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Fri, 30 Sep 2022 10:14:57 +0100 Subject: [PATCH 072/292] Consider agents that are auto-downed The count of agents down failed to include nova-compute which was disabled by nova automatically because the connection to libvirt was lost. This still represents an error condition. --- .../kolla/config/grafana/dashboards/openstack/openstack.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/openstack.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/openstack.json index 30ac8f54b..6dc3cb865 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/openstack/openstack.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/openstack.json @@ -102,7 +102,7 @@ "tableColumn": "", "targets": [ { - "expr": "count(openstack_nova_agent_state{adminState=\"enabled\"})-sum(openstack_nova_agent_state{adminState=\"enabled\"})", + "expr": "count(openstack_nova_agent_state{adminState=\"enabled\"} or openstack_nova_agent_state{adminState=\"disabled\", disabledReason=~\"^AUTO:.*\"})-sum(openstack_nova_agent_state{adminState=\"enabled\"})", "format": "time_series", "hide": false, "instant": true, From 4aa8aed69c4bcf0ef469156a15eb2dbb6d106c47 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Mon, 22 Aug 2022 14:44:27 +0000 Subject: [PATCH 073/292] feat: create `ci-multinode` environment The `ci-multinode` environment is a clone of the `ci-aio` environment and as a result will need adapting. --- .../environments/ci-multinode/globals.yml | 63 ++++++++++ .../group_vars/controllers/network-interfaces | 12 ++ .../ci-multinode/inventory/groups | 5 + .../environments/ci-multinode/inventory/hosts | 6 + .../ci-multinode/inventory/kayobe-automation | 2 + .../ci-multinode/kolla/config/ironic.conf | 2 + .../ci-multinode/kolla/globals.yml | 10 ++ .../ci-multinode/network-allocation.yml | 0 .../environments/ci-multinode/networks.yml | 119 ++++++++++++++++++ .../environments/ci-multinode/stackhpc-ci.yml | 58 +++++++++ 10 files changed, 277 insertions(+) create mode 100644 etc/kayobe/environments/ci-multinode/globals.yml create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces create mode 100644 etc/kayobe/environments/ci-multinode/inventory/groups create mode 100644 etc/kayobe/environments/ci-multinode/inventory/hosts create mode 100644 etc/kayobe/environments/ci-multinode/inventory/kayobe-automation create mode 100644 etc/kayobe/environments/ci-multinode/kolla/config/ironic.conf create mode 100644 etc/kayobe/environments/ci-multinode/kolla/globals.yml create mode 100644 etc/kayobe/environments/ci-multinode/network-allocation.yml create mode 100644 etc/kayobe/environments/ci-multinode/networks.yml create mode 100644 etc/kayobe/environments/ci-multinode/stackhpc-ci.yml diff --git a/etc/kayobe/environments/ci-multinode/globals.yml b/etc/kayobe/environments/ci-multinode/globals.yml new file mode 100644 index 000000000..3a6674de3 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/globals.yml @@ -0,0 +1,63 @@ +--- +# Kayobe global configuration. + +############################################################################### +# Local path configuration (Ansible control host). + +# Path to Kayobe configuration directory on Ansible control host, with an +# environment path appended if kayobe_environment is set. +#kayobe_config_path: + +# Name of Kayobe environment to use. Default is $KAYOBE_ENVIRONMENT, or an +# empty string if $KAYOBE_ENVIRONMENT is not set. Can also be set via the +# --environment argument when invoking kayobe. +#kayobe_environment: + +# Path to Kayobe configuration directory on Ansible control host with an +# environment path appended if kayobe_environment is set. +#kayobe_env_config_path: + +############################################################################### +# Remote path configuration (seed, seed-hypervisor and overcloud hosts). + +# Base path for kayobe state on remote hosts. +#base_path: + +# Path in which to store configuration on remote hosts. +#config_path: + +# Path in which to cache downloaded images on remote hosts. +#image_cache_path: + +# Path on which to checkout source code repositories on remote hosts. +#source_checkout_path: + +# Path on which to create python virtualenvs on remote hosts. +#virtualenv_path: + +############################################################################### +# User configuration. + +# User with which to access remote hosts. This user will be created if it does +# not exist. +#kayobe_ansible_user: + +############################################################################### +# OS distribution. + +# OS distribution name. Valid options are "centos", "ubuntu". Default is +# "centos". +os_distribution: "{{ lookup('pipe', '. /etc/os-release && echo $ID') | trim }}" + +# OS release. Valid options are "8-stream" when os_distribution is "centos", or +# "focal" when os_distribution is "ubuntu". +#os_release: + +############################################################################### + +# Avoid a reboot. +disable_selinux_do_reboot: false + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces new file mode 100644 index 000000000..cae8e6a7d --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces @@ -0,0 +1,12 @@ +--- +############################################################################### +# Network interface definitions for the controller group. + +# Controller interface on all-in-one network. +aio_interface: breth1 +# Use dummy1 if it exists, otherwise the bridge will have no ports. +aio_bridge_ports: "{{ ['dummy1'] if 'ansible_dummy1' in hostvars[inventory_hostname] else [] }}" + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/environments/ci-multinode/inventory/groups b/etc/kayobe/environments/ci-multinode/inventory/groups new file mode 100644 index 000000000..43967dac1 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/groups @@ -0,0 +1,5 @@ +[controllers] + +[container-image-builders:children] +# Build container images on the all-in-one controller. +controllers diff --git a/etc/kayobe/environments/ci-multinode/inventory/hosts b/etc/kayobe/environments/ci-multinode/inventory/hosts new file mode 100644 index 000000000..b041573df --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/hosts @@ -0,0 +1,6 @@ +# A single all-in-one controller/compute host. +[controllers] +controller0 + +[compute:children] +controllers diff --git a/etc/kayobe/environments/ci-multinode/inventory/kayobe-automation b/etc/kayobe/environments/ci-multinode/inventory/kayobe-automation new file mode 100644 index 000000000..5ab47462a --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/kayobe-automation @@ -0,0 +1,2 @@ +[tempest_runner] +controller0 diff --git a/etc/kayobe/environments/ci-multinode/kolla/config/ironic.conf b/etc/kayobe/environments/ci-multinode/kolla/config/ironic.conf new file mode 100644 index 000000000..45f848825 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/kolla/config/ironic.conf @@ -0,0 +1,2 @@ +[DEFAULT] +minimum_required_memory = 256 diff --git a/etc/kayobe/environments/ci-multinode/kolla/globals.yml b/etc/kayobe/environments/ci-multinode/kolla/globals.yml new file mode 100644 index 000000000..9c769d3ea --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/kolla/globals.yml @@ -0,0 +1,10 @@ +--- +# Most development environments will use nested virtualisation, and we can't +# guarantee that nested KVM support is available. Use QEMU as a lowest common +# denominator. +nova_compute_virt_type: qemu + +# Reduce the control plane's memory footprint by limiting the number of worker +# processes to one per-service. +openstack_service_workers: "1" +openstack_service_rpc_workers: "1" diff --git a/etc/kayobe/environments/ci-multinode/network-allocation.yml b/etc/kayobe/environments/ci-multinode/network-allocation.yml new file mode 100644 index 000000000..e69de29bb diff --git a/etc/kayobe/environments/ci-multinode/networks.yml b/etc/kayobe/environments/ci-multinode/networks.yml new file mode 100644 index 000000000..a42719638 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/networks.yml @@ -0,0 +1,119 @@ +--- +# Kayobe network configuration. + +############################################################################### +# Network role to network mappings. + +# Map all networks to the all-in-one network. + +# Name of the network used for admin access to the overcloud +#admin_oc_net_name: +admin_oc_net_name: admin + +# Name of the network used by the seed to manage the bare metal overcloud +# hosts via their out-of-band management controllers. +#oob_oc_net_name: +oob_oc_net_name: aio + +# Name of the network used by the seed to provision the bare metal overcloud +# hosts. +#provision_oc_net_name: +provision_oc_net_name: aio + +# Name of the network used by the overcloud hosts to manage the bare metal +# compute hosts via their out-of-band management controllers. +#oob_wl_net_name: +oob_wl_net_name: aio + +# Name of the network used by the overcloud hosts to provision the bare metal +# workload hosts. +#provision_wl_net_name: +provision_wl_net_name: aio + +# Name of the network used to expose the internal OpenStack API endpoints. +#internal_net_name: +internal_net_name: aio + +# List of names of networks used to provide external network access via +# Neutron. +# Deprecated name: external_net_name +# If external_net_name is defined, external_net_names will default to a list +# containing one item, external_net_name. +#external_net_names: +external_net_names: + - aio + +# Name of the network used to expose the public OpenStack API endpoints. +#public_net_name: +public_net_name: aio + +# Name of the network used by Neutron to carry tenant overlay network traffic. +#tunnel_net_name: +tunnel_net_name: aio + +# Name of the network used to carry storage data traffic. +#storage_net_name: +storage_net_name: aio + +# Name of the network used to carry storage management traffic. +#storage_mgmt_net_name: +storage_mgmt_net_name: aio + +# Name of the network used to carry swift storage data traffic. +#swift_storage_net_name: +swift_storage_net_name: aio + +# Name of the network used to carry swift storage replication traffic. +#swift_storage_replication_net_name: +swift_storage_replication_net_name: aio + +# Name of the network used to perform hardware introspection on the bare metal +# workload hosts. +#inspection_net_name: +inspection_net_name: aio + +# Name of the network used to perform cleaning on the bare metal workload +# hosts +#cleaning_net_name: +cleaning_net_name: aio + +############################################################################### +# Network definitions. + +# All-in-one network. +aio_cidr: 192.168.33.0/24 +aio_allocation_pool_start: 192.168.33.3 +aio_allocation_pool_end: 192.168.33.30 +aio_neutron_allocation_pool_start: 192.168.33.31 +aio_neutron_allocation_pool_end: 192.168.33.127 +aio_inspection_allocation_pool_start: 192.168.33.128 +aio_inspection_allocation_pool_end: 192.168.33.254 +aio_vip_address: 192.168.33.2 + +############################################################################### +# Network virtual patch link configuration. + +# Suffix for Open vSwitch bridge names. +#network_bridge_suffix_ovs: + +# Prefix for virtual patch interface names. +#network_patch_prefix: + +# Suffix for virtual patch link interface names when connected towards the +# physical interface. +#network_patch_suffix_phy: + +# Suffix for virtual patch link interface names when connected towards the +# OVS bridge. +#network_patch_suffix_ovs: + +############################################################################### +# Network routing table configuration. + +# List of IP routing tables. Each item should be a dict containing 'id' and +# 'name' items. These tables will be added to /etc/iproute2/rt_tables. +#network_route_tables: + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml new file mode 100644 index 000000000..b78339cdb --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml @@ -0,0 +1,58 @@ +--- +############################################################################### +# Kolla configuration. + +# Docker namespace to use for Kolla images. Default is 'kolla'. +kolla_docker_namespace: stackhpc-dev + +############################################################################### +# Network configuration. + +# Don't touch resolv.conf: use Neutron DNS for accessing Pulp server via +# hostname. +resolv_is_managed: false + +############################################################################### +# StackHPC configuration. + +# Host and port of a package repository mirror. +# Build and deploy the development Pulp service repositories. +stackhpc_repo_mirror_url: "http://pulp-server.internal.sms-cloud:8080" + +# Build and deploy released Pulp repository versions. +stackhpc_repo_centos_stream_baseos_version: "{{ stackhpc_pulp_repo_centos_stream_8_baseos_version }}" +stackhpc_repo_centos_stream_appstream_version: "{{ stackhpc_pulp_repo_centos_stream_8_appstream_version }}" +stackhpc_repo_centos_stream_extras_version: "{{ stackhpc_pulp_repo_centos_stream_8_extras_version }}" +stackhpc_repo_epel_version: "{{ stackhpc_pulp_repo_epel_version }}" +stackhpc_repo_epel_modular_version: "{{ stackhpc_pulp_repo_epel_modular_version }}" +stackhpc_repo_docker_version: "{{ stackhpc_pulp_repo_docker_version }}" +stackhpc_repo_centos_stream_advanced_virtualization_version: "{{ stackhpc_pulp_repo_centos_stream_8_advanced_virtualization_version }}" +stackhpc_repo_centos_stream_nfv_openvswitch_version: "{{ stackhpc_pulp_repo_centos_stream_8_nfv_openvswitch_version }}" +stackhpc_repo_centos_stream_openstack_wallaby_version: "{{ stackhpc_pulp_repo_centos_stream_8_openstack_wallaby_version }}" +stackhpc_repo_centos_stream_powertools_version: "{{ stackhpc_pulp_repo_centos_stream_8_powertools_version }}" +stackhpc_repo_centos_opstools_version: "{{ stackhpc_pulp_repo_centos_8_opstools_version }}" +stackhpc_repo_centos_stream_storage_ceph_pacific_version: "{{ stackhpc_pulp_repo_centos_stream_8_storage_ceph_pacific_version }}" +stackhpc_repo_elasticsearch_logstash_kibana_7_x_version: "{{ stackhpc_pulp_repo_elasticsearch_logstash_kibana_7_x_version }}" +stackhpc_repo_grafana_version: "{{ stackhpc_pulp_repo_grafana_version }}" +stackhpc_repo_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_rabbitmq_erlang_version }}" +stackhpc_repo_rabbitmq_server_version: "{{ stackhpc_pulp_repo_rabbitmq_server_version }}" +stackhpc_repo_treasuredata_4_version: "{{ stackhpc_pulp_repo_treasuredata_4_version }}" +stackhpc_repo_ubuntu_cloud_archive_version: "{{ stackhpc_pulp_repo_ubuntu_cloud_archive_version }}" +stackhpc_repo_ubuntu_focal_version: "{{ stackhpc_pulp_repo_ubuntu_focal_version }}" +stackhpc_repo_ubuntu_focal_security_version: "{{ stackhpc_pulp_repo_ubuntu_focal_security_version }}" +stackhpc_repo_docker_ce_ubuntu_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_version }}" + + +# Host and port of container registry. +# Push built images to the development Pulp service registry. +stackhpc_docker_registry: "{{ stackhpc_repo_mirror_url | regex_replace('^https?://', '') }}" + +# Username and password of container registry. +stackhpc_docker_registry_username: "admin" +stackhpc_docker_registry_password: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 38653432663133653430323937626239323836653537633633326265643930653034306532646464 + 3766353637326534643136326533633034666535323364660a633639393433376564363666663233 + 38333133393730633666613965653364316162353337313330346164303631313731646461363461 + 3963323635373866630a633533376339363734626664333765313665623662613764363038383735 + 38646138376438643533376161376634653439386230353365316239613430363338 From 6f49c3f09f2c04ab8a3fbe94804ad4e0a6bef389 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 24 Aug 2022 16:57:54 +0000 Subject: [PATCH 074/292] feat: successfully configure two controllers --- .../group_vars/compute/network-interfaces | 14 ++++++++++++++ .../group_vars/controllers/network-interfaces | 2 ++ .../inventory/group_vars/seed/network-interfaces | 14 ++++++++++++++ .../environments/ci-multinode/inventory/hosts | 6 ------ .../ci-multinode/network-allocation.yml | 0 .../environments/ci-multinode/stackhpc-ci.yml | 10 ++-------- 6 files changed, 32 insertions(+), 14 deletions(-) create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces delete mode 100644 etc/kayobe/environments/ci-multinode/inventory/hosts delete mode 100644 etc/kayobe/environments/ci-multinode/network-allocation.yml diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces new file mode 100644 index 000000000..016f1e72b --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces @@ -0,0 +1,14 @@ +--- +############################################################################### +# Network interface definitions for the controller group. + +admin_interface: eno1 + +# Controller interface on all-in-one network. +aio_interface: breth1 +# Use dummy1 if it exists, otherwise the bridge will have no ports. +aio_bridge_ports: "{{ ['dummy1'] if 'ansible_dummy1' in hostvars[inventory_hostname] else [] }}" + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces index cae8e6a7d..016f1e72b 100644 --- a/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces @@ -2,6 +2,8 @@ ############################################################################### # Network interface definitions for the controller group. +admin_interface: eno1 + # Controller interface on all-in-one network. aio_interface: breth1 # Use dummy1 if it exists, otherwise the bridge will have no ports. diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces new file mode 100644 index 000000000..092b82f09 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces @@ -0,0 +1,14 @@ +--- +############################################################################### +# Network interface definitions for the controller group. + +admin_interface: eth0 + +# Controller interface on all-in-one network. +aio_interface: breth1 +# Use dummy1 if it exists, otherwise the bridge will have no ports. +aio_bridge_ports: "{{ ['dummy1'] if 'ansible_dummy1' in hostvars[inventory_hostname] else [] }}" + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/environments/ci-multinode/inventory/hosts b/etc/kayobe/environments/ci-multinode/inventory/hosts deleted file mode 100644 index b041573df..000000000 --- a/etc/kayobe/environments/ci-multinode/inventory/hosts +++ /dev/null @@ -1,6 +0,0 @@ -# A single all-in-one controller/compute host. -[controllers] -controller0 - -[compute:children] -controllers diff --git a/etc/kayobe/environments/ci-multinode/network-allocation.yml b/etc/kayobe/environments/ci-multinode/network-allocation.yml deleted file mode 100644 index e69de29bb..000000000 diff --git a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml index b78339cdb..6f2eb4599 100644 --- a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml @@ -48,11 +48,5 @@ stackhpc_repo_docker_ce_ubuntu_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_ stackhpc_docker_registry: "{{ stackhpc_repo_mirror_url | regex_replace('^https?://', '') }}" # Username and password of container registry. -stackhpc_docker_registry_username: "admin" -stackhpc_docker_registry_password: !vault | - $ANSIBLE_VAULT;1.1;AES256 - 38653432663133653430323937626239323836653537633633326265643930653034306532646464 - 3766353637326534643136326533633034666535323364660a633639393433376564363666663233 - 38333133393730633666613965653364316162353337313330346164303631313731646461363461 - 3963323635373866630a633533376339363734626664333765313665623662613764363038383735 - 38646138376438643533376161376634653439386230353365316239613430363338 +stackhpc_docker_registry_username: "" +stackhpc_docker_registry_password: "" From d057d07b34e1fe343da2be8b1bcbc82fbea127f5 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 25 Aug 2022 10:58:17 +0000 Subject: [PATCH 075/292] feat: succesfully configure three controllers with VXLAN --- etc/kayobe/ansible/configure-vxlan.yml | 20 +++++++++++++++++++ etc/kayobe/ansible/requirements.yml | 4 ++++ .../group_vars/controllers/network-interfaces | 2 +- .../post.d/50-configure-vxlan.yml | 1 + 4 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 etc/kayobe/ansible/configure-vxlan.yml create mode 120000 etc/kayobe/hooks/overcloud-host-configure/post.d/50-configure-vxlan.yml diff --git a/etc/kayobe/ansible/configure-vxlan.yml b/etc/kayobe/ansible/configure-vxlan.yml new file mode 100644 index 000000000..fed1b05a5 --- /dev/null +++ b/etc/kayobe/ansible/configure-vxlan.yml @@ -0,0 +1,20 @@ +--- +- name: Configure VXLAN + hosts: controllers + gather_facts: false + vars: + ansible_user: "{{ bootstrap_user }}" + # We can't assume that a virtualenv exists at this point, so use the system + # python interpreter. + ansible_python_interpreter: /usr/bin/python3 + # Work around no known_hosts entry on first boot. + ansible_ssh_common_args: "-o StrictHostKeyChecking=no" + roles: + - role: vxlan + vars: + vxlan_phys_dev: "{{ admin_oc_net_name | net_interface }}" + vxlan_interfaces: + - device: vxlan10 + group: 224.0.0.10 + bridge: breth1 + diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index 99a0c83f7..d3e99dfb0 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -4,3 +4,7 @@ collections: version: 1.10.0 - name: stackhpc.pulp version: 0.3.0 +roles: + - name: vxlan + src: https://github.com/stackhpc/ansible-role-vxlan.git + version: b506e9817e1eb0a81ea66e0cddc40ab2ff66bc79 diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces index 016f1e72b..2e94b6c2d 100644 --- a/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces @@ -7,7 +7,7 @@ admin_interface: eno1 # Controller interface on all-in-one network. aio_interface: breth1 # Use dummy1 if it exists, otherwise the bridge will have no ports. -aio_bridge_ports: "{{ ['dummy1'] if 'ansible_dummy1' in hostvars[inventory_hostname] else [] }}" +aio_bridge_ports: [] ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/etc/kayobe/hooks/overcloud-host-configure/post.d/50-configure-vxlan.yml b/etc/kayobe/hooks/overcloud-host-configure/post.d/50-configure-vxlan.yml new file mode 120000 index 000000000..d8d2ce4bb --- /dev/null +++ b/etc/kayobe/hooks/overcloud-host-configure/post.d/50-configure-vxlan.yml @@ -0,0 +1 @@ +../../../ansible/configure-vxlan.yml \ No newline at end of file From 27e6c619aa4db153159280565e2b7ec3d0b067a5 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 31 Aug 2022 12:01:07 +0000 Subject: [PATCH 076/292] feat: add `kayobe-compute-01` --- etc/kayobe/ansible/configure-vxlan.yml | 2 +- .../inventory/group_vars/compute/network-interfaces | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/ansible/configure-vxlan.yml b/etc/kayobe/ansible/configure-vxlan.yml index fed1b05a5..a226927f1 100644 --- a/etc/kayobe/ansible/configure-vxlan.yml +++ b/etc/kayobe/ansible/configure-vxlan.yml @@ -1,6 +1,6 @@ --- - name: Configure VXLAN - hosts: controllers + hosts: controllers,compute gather_facts: false vars: ansible_user: "{{ bootstrap_user }}" diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces index 016f1e72b..2e94b6c2d 100644 --- a/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces @@ -7,7 +7,7 @@ admin_interface: eno1 # Controller interface on all-in-one network. aio_interface: breth1 # Use dummy1 if it exists, otherwise the bridge will have no ports. -aio_bridge_ports: "{{ ['dummy1'] if 'ansible_dummy1' in hostvars[inventory_hostname] else [] }}" +aio_bridge_ports: [] ############################################################################### # Dummy variable to allow Ansible to accept this file. From b99dfa65c88c8458c5a9dd7ff58797883907db29 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 31 Aug 2022 12:02:16 +0000 Subject: [PATCH 077/292] feat: add `growroot` playbook for successful deployment --- etc/kayobe/environments/ci-multinode/compute.yml | 2 ++ etc/kayobe/environments/ci-multinode/controllers.yml | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 etc/kayobe/environments/ci-multinode/compute.yml create mode 100644 etc/kayobe/environments/ci-multinode/controllers.yml diff --git a/etc/kayobe/environments/ci-multinode/compute.yml b/etc/kayobe/environments/ci-multinode/compute.yml new file mode 100644 index 000000000..254afc8e4 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/compute.yml @@ -0,0 +1,2 @@ +compute_lvm_groups: + - "{{ stackhpc_lvm_group_rootvg }}" diff --git a/etc/kayobe/environments/ci-multinode/controllers.yml b/etc/kayobe/environments/ci-multinode/controllers.yml new file mode 100644 index 000000000..9198a4d4c --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/controllers.yml @@ -0,0 +1,2 @@ +controller_lvm_groups: + - "{{ stackhpc_lvm_group_rootvg }}" From 5b55f69713b099708b994595ddf06a6917e85dd0 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Fri, 2 Sep 2022 15:05:04 +0000 Subject: [PATCH 078/292] feat: successfully deploy environment This configuration is capable of successfully deploying a multinode cluster across three controllers and one compute node. --- etc/kayobe/ansible/configure-vxlan.yml | 3 ++- .../inventory/group_vars/seed/network-interfaces | 3 +-- .../ci-multinode/inventory/kayobe-automation | 2 +- etc/kayobe/environments/ci-multinode/stackhpc-ci.yml | 10 ++++++++-- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/etc/kayobe/ansible/configure-vxlan.yml b/etc/kayobe/ansible/configure-vxlan.yml index a226927f1..92b699615 100644 --- a/etc/kayobe/ansible/configure-vxlan.yml +++ b/etc/kayobe/ansible/configure-vxlan.yml @@ -1,6 +1,6 @@ --- - name: Configure VXLAN - hosts: controllers,compute + hosts: seed,controllers,compute gather_facts: false vars: ansible_user: "{{ bootstrap_user }}" @@ -13,6 +13,7 @@ - role: vxlan vars: vxlan_phys_dev: "{{ admin_oc_net_name | net_interface }}" + vxlan_dstport: 4790 vxlan_interfaces: - device: vxlan10 group: 224.0.0.10 diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces index 092b82f09..13f7aa8ea 100644 --- a/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces @@ -7,8 +7,7 @@ admin_interface: eth0 # Controller interface on all-in-one network. aio_interface: breth1 # Use dummy1 if it exists, otherwise the bridge will have no ports. -aio_bridge_ports: "{{ ['dummy1'] if 'ansible_dummy1' in hostvars[inventory_hostname] else [] }}" - +aio_bridge_ports: [] ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/environments/ci-multinode/inventory/kayobe-automation b/etc/kayobe/environments/ci-multinode/inventory/kayobe-automation index 5ab47462a..5ac95a0d2 100644 --- a/etc/kayobe/environments/ci-multinode/inventory/kayobe-automation +++ b/etc/kayobe/environments/ci-multinode/inventory/kayobe-automation @@ -1,2 +1,2 @@ [tempest_runner] -controller0 +kayobe-controller-01 diff --git a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml index 6f2eb4599..191b9a806 100644 --- a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml @@ -48,5 +48,11 @@ stackhpc_repo_docker_ce_ubuntu_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_ stackhpc_docker_registry: "{{ stackhpc_repo_mirror_url | regex_replace('^https?://', '') }}" # Username and password of container registry. -stackhpc_docker_registry_username: "" -stackhpc_docker_registry_password: "" +stackhpc_docker_registry_username: "stackhpc-kayobe-ci" +stackhpc_docker_registry_password: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 30333932623139383562343036366566363230656536376164633463366235623463323937303135 + 6533333638613764383136613730346436643831646530640a363262373964323461616638636135 + 65623338333236323030346564363035313462333335333032643661363834356432363833303838 + 3936616462303062350a316165396265313766316338303338336132316264316631376163643133 + 33616239633435636630373039616630323163343962336330303864626561303238 From c35932403b4ced0f558ae78245659038b05674a4 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Mon, 12 Sep 2022 15:22:43 +0100 Subject: [PATCH 079/292] fix: syntax issues raised by `yamllint` --- etc/kayobe/ansible/configure-vxlan.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/etc/kayobe/ansible/configure-vxlan.yml b/etc/kayobe/ansible/configure-vxlan.yml index 92b699615..12deba374 100644 --- a/etc/kayobe/ansible/configure-vxlan.yml +++ b/etc/kayobe/ansible/configure-vxlan.yml @@ -18,4 +18,3 @@ - device: vxlan10 group: 224.0.0.10 bridge: breth1 - From 8eb2f85f64c67be84951741dab19309412475357 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 3 Oct 2022 15:56:18 +0100 Subject: [PATCH 080/292] added notes --- .../environments/ci-multinode/README.md | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 etc/kayobe/environments/ci-multinode/README.md diff --git a/etc/kayobe/environments/ci-multinode/README.md b/etc/kayobe/environments/ci-multinode/README.md new file mode 100644 index 000000000..668e8895d --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/README.md @@ -0,0 +1,52 @@ +# Multinode Test Environment + +## Set up hosts +1. Create four baremetal instances with a centos 8 stream LVM image, and a +Centos 8 stream vm +2. SSH into each baremetal and run `sudo chown -R centos:.` in the home directory, + then add the lines + >`10.0.0.34 pelican pelican.service.compute.sms-lab.cloud` + >`10.205.3.187 pulp-server pulp-server.internal.sms-cloud` + + to /etc/hosts (if you're waiting on them starting up, you can progress until + `kayobe overcloud host configure` without this step) + +## Basic Kayobe Setup +1. SSH into the VM +1. `sudo dnf install -y python3-virtualenv` +1. `mkdir src` and `cd src` +1. Clone https://github.com/stackhpc/stackhpc-kayobe-config.git, then checkout + commit f31df6256f1b1fea99c84547d44f06c4cb74b161 +1. `cd ..` and `mkdir venvs` +1. `virtualenv venvs/kayobe` and source `venvs/kayobe/bin/activate` +1. `pip install -U pip` +1. `pip install ./src/kayobe` +1. Acquire the Ansible Vault password for this repository, and store a copy at +``~/vault-pw`` +1. `export KAYOBE_VAULT_PASSWORD=$(cat ~/vault-pw)` + +## Config changes +1. In etc/kayobe/ansible/requirements.yml remove version from vxlan +4. In etc/kayobe/ansible/configure-vxlan.yml, change the group of +vxlan_interfaces so that the last octet is different e.g. 224.0.0.15 +5. Also under vxlan_interfaces, add vni:x where x is between 500 and 1000 +5. Also under vxlan_interfaces, check vxlan_dstport is not 4789 (this causes +conflicts, change to 4790) +6. In /etc/kayobe/environments/ci-multinode/tf-networks.yml, + edit admin_ips so that the compute and controller IPs line up with the + instances that were created earlier, remove the other IPs for seed and + cephOSD +7. In /etc/kayobe/environments/ci-multinode/network-allocation.yml, remove all +the entries and just assign `aio_ips:` an empty set `[]` +8. In etc/kayobe/environments/ci-multinode/inventory/hosts, remove the seed +9. run stackhpc-kayobe-config/etc/kayobe/ansible/growroot.yml (if this fails, +manually increase the partition size on each host) + +## Final steps +1. `source kayobe-env --environment ci-aio` +10. Run `kayobe overcloud host configure` +11. Run `kayobe overcloud service deploy` + +TODO: +- openstack will use the port that as been reserved for vslan so 4790 was chosen because it's one away +-m From ed3f2869d7ed9d3b53d4182a08a6ed48caf82a47 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 3 Oct 2022 16:00:36 +0100 Subject: [PATCH 081/292] removed TODO --- etc/kayobe/environments/ci-multinode/README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/etc/kayobe/environments/ci-multinode/README.md b/etc/kayobe/environments/ci-multinode/README.md index 668e8895d..c2511150c 100644 --- a/etc/kayobe/environments/ci-multinode/README.md +++ b/etc/kayobe/environments/ci-multinode/README.md @@ -46,7 +46,3 @@ manually increase the partition size on each host) 1. `source kayobe-env --environment ci-aio` 10. Run `kayobe overcloud host configure` 11. Run `kayobe overcloud service deploy` - -TODO: -- openstack will use the port that as been reserved for vslan so 4790 was chosen because it's one away --m From 08df28022e2dea99c4d1f2b75257b12796731686 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 28 Sep 2022 15:01:28 +0000 Subject: [PATCH 082/292] feat: update vxlan role version and expand hosts --- etc/kayobe/ansible/configure-vxlan.yml | 2 +- etc/kayobe/ansible/requirements.yml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/etc/kayobe/ansible/configure-vxlan.yml b/etc/kayobe/ansible/configure-vxlan.yml index 12deba374..924fdd7ad 100644 --- a/etc/kayobe/ansible/configure-vxlan.yml +++ b/etc/kayobe/ansible/configure-vxlan.yml @@ -1,6 +1,6 @@ --- - name: Configure VXLAN - hosts: seed,controllers,compute + hosts: storage,seed,controllers,compute gather_facts: false vars: ansible_user: "{{ bootstrap_user }}" diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index d3e99dfb0..4edd6b905 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -7,4 +7,3 @@ collections: roles: - name: vxlan src: https://github.com/stackhpc/ansible-role-vxlan.git - version: b506e9817e1eb0a81ea66e0cddc40ab2ff66bc79 From a4eccccd93c49c935a6b71186ac99084e8552713 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Sun, 2 Oct 2022 14:33:40 +0000 Subject: [PATCH 083/292] feat: enable ceph and ceph related services --- etc/kayobe/environments/ci-multinode/kolla.yml | 3 +++ etc/kayobe/environments/ci-multinode/kolla/globals.yml | 9 +++++++++ 2 files changed, 12 insertions(+) create mode 100644 etc/kayobe/environments/ci-multinode/kolla.yml diff --git a/etc/kayobe/environments/ci-multinode/kolla.yml b/etc/kayobe/environments/ci-multinode/kolla.yml new file mode 100644 index 000000000..168500c8b --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/kolla.yml @@ -0,0 +1,3 @@ +--- +kolla_enable_cinder: true +kolla_enable_cinder_backup: true diff --git a/etc/kayobe/environments/ci-multinode/kolla/globals.yml b/etc/kayobe/environments/ci-multinode/kolla/globals.yml index 9c769d3ea..ebfe64897 100644 --- a/etc/kayobe/environments/ci-multinode/kolla/globals.yml +++ b/etc/kayobe/environments/ci-multinode/kolla/globals.yml @@ -8,3 +8,12 @@ nova_compute_virt_type: qemu # processes to one per-service. openstack_service_workers: "1" openstack_service_rpc_workers: "1" + +# Glance Ceph configuration +glance_backend_ceph: "yes" + +# Cinder Ceph configuration +cinder_backend_ceph: "yes" + +# Nova Ceph configuration +nova_backend_ceph: "yes" From 483b887a5b53948341823d81aecf12c69c3544a7 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Mon, 3 Oct 2022 15:30:50 +0100 Subject: [PATCH 084/292] feat: pull vxlan from ansible galaxy --- etc/kayobe/ansible/configure-vxlan.yml | 2 +- etc/kayobe/ansible/requirements.yml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/etc/kayobe/ansible/configure-vxlan.yml b/etc/kayobe/ansible/configure-vxlan.yml index 924fdd7ad..f07933c1b 100644 --- a/etc/kayobe/ansible/configure-vxlan.yml +++ b/etc/kayobe/ansible/configure-vxlan.yml @@ -10,7 +10,7 @@ # Work around no known_hosts entry on first boot. ansible_ssh_common_args: "-o StrictHostKeyChecking=no" roles: - - role: vxlan + - role: stackhpc.vxlan vars: vxlan_phys_dev: "{{ admin_oc_net_name | net_interface }}" vxlan_dstport: 4790 diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index 4edd6b905..bd4d27e3f 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -5,5 +5,4 @@ collections: - name: stackhpc.pulp version: 0.3.0 roles: - - name: vxlan - src: https://github.com/stackhpc/ansible-role-vxlan.git + - src: stackhpc.vxlan From cbffc6e899816742fe9d7a5e458f70db5dd5684c Mon Sep 17 00:00:00 2001 From: Grzegorz Koper Date: Wed, 5 Oct 2022 15:16:29 +0200 Subject: [PATCH 085/292] Removing vxlan hook --- .../hooks/overcloud-host-configure/post.d/50-configure-vxlan.yml | 1 - 1 file changed, 1 deletion(-) delete mode 120000 etc/kayobe/hooks/overcloud-host-configure/post.d/50-configure-vxlan.yml diff --git a/etc/kayobe/hooks/overcloud-host-configure/post.d/50-configure-vxlan.yml b/etc/kayobe/hooks/overcloud-host-configure/post.d/50-configure-vxlan.yml deleted file mode 120000 index d8d2ce4bb..000000000 --- a/etc/kayobe/hooks/overcloud-host-configure/post.d/50-configure-vxlan.yml +++ /dev/null @@ -1 +0,0 @@ -../../../ansible/configure-vxlan.yml \ No newline at end of file From 700b4dbb7cacdf20ea215885e28198e8567d4359 Mon Sep 17 00:00:00 2001 From: Grzegorz Koper Date: Wed, 5 Oct 2022 15:21:59 +0200 Subject: [PATCH 086/292] added cephadm.yml to multinode environment. --- .../environments/ci-multinode/cephadm.yml | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 etc/kayobe/environments/ci-multinode/cephadm.yml diff --git a/etc/kayobe/environments/ci-multinode/cephadm.yml b/etc/kayobe/environments/ci-multinode/cephadm.yml new file mode 100644 index 000000000..812f89a06 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/cephadm.yml @@ -0,0 +1,62 @@ +--- +############################################################################### +# Cephadm deployment configuration. + +# Ceph container image. +cephadm_image: "quay.io/ceph/ceph:v16.2.5" + +# Ceph OSD specification. +cephadm_osd_spec: + service_type: osd + service_id: osd_spec_default + placement: + host_pattern: "*" + data_devices: + all: true + +############################################################################### +# Ceph post-deployment configuration. + +# List of Ceph erasure coding profiles. See stackhpc.cephadm.ec_profiles role +# for format. +cephadm_ec_profiles: [] + +# List of Ceph CRUSH rules. See stackhpc.cephadm.crush_rules role for format. +cephadm_crush_rules: [] + +# List of Ceph pools. See stackhpc.cephadm.pools role for format. +cephadm_pools: + - name: backups + application: rbd + state: present + - name: images + application: rbd + state: present + - name: volumes + application: rbd + state: present + - name: vms + application: rbd + state: present + +# List of Cephx keys. See stackhpc.cephadm.keys role for format. +cephadm_keys: + - name: client.cinder + caps: + mon: "profile rbd" + osd: "profile rbd pool=volumes, profile rbd pool=vms, profile rbd-read-only pool=images" + mgr: "profile rbd pool=volumes, profile rbd pool=vms" + state: present + - name: client.cinder-backup + caps: + mon: "profile rbd" + osd: "profile rbd pool=volumes, profile rbd pool=backups" + mgr: "profile rbd pool=volumes, profile rbd pool=backups" + state: present + - name: client.glance + caps: + mon: "profile rbd" + osd: "profile rbd pool=images" + mgr: "profile rbd pool=images" + state: present + From 21ffab9b8f8de585af9ec6cbe52acbfd82d7ecda Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 5 Oct 2022 15:20:03 +0100 Subject: [PATCH 087/292] feat: add storage group vars --- .../group_vars/storage/network-interfaces | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/network-interfaces diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/network-interfaces new file mode 100644 index 000000000..35000577f --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/network-interfaces @@ -0,0 +1,14 @@ +--- +############################################################################### +# Network interface definitions for the stroage group. + +admin_interface: ens3 + +# Controller interface on all-in-one network. +aio_interface: breth1 +# Use dummy1 if it exists, otherwise the bridge will have no ports. +aio_bridge_ports: [] + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes From 6c805de8c84627abf516fce0f3ab48998edee836 Mon Sep 17 00:00:00 2001 From: Grzegorz Koper Date: Mon, 10 Oct 2022 12:55:41 +0200 Subject: [PATCH 088/292] Adding LVM configuration for storage nodes --- .../inventory/group_vars/storage/lvm | 23 +++++++++++++++++++ .../environments/ci-multinode/storage.yml | 5 ++++ 2 files changed, 28 insertions(+) create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/lvm create mode 100644 etc/kayobe/environments/ci-multinode/storage.yml diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/lvm b/etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/lvm new file mode 100644 index 000000000..03b356a9e --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/lvm @@ -0,0 +1,23 @@ +# StackHPC LVM lv_swap LV size. +stackhpc_lvm_lv_swap_size: 1g + +# StackHPC LVM lv_root LV size. +stackhpc_lvm_lv_root_size: 8g + +# StackHPC LVM lv_tmp LV size. +stackhpc_lvm_lv_tmp_size: 2g + +# StackHPC LVM lv_var LV size. +stackhpc_lvm_lv_var_size: 12g + +# StackHPC LVM lv_var_tmp LV size. +stackhpc_lvm_lv_var_tmp_size: 1g + +# StackHPC LVM lv_log LV size. +stackhpc_lvm_lv_log_size: 2g + +# StackHPC LVM lv_audit LV size. +stackhpc_lvm_lv_audit_size: 1g + +# StackHPC LVM lv_home LV size. +stackhpc_lvm_lv_home_size: 2g diff --git a/etc/kayobe/environments/ci-multinode/storage.yml b/etc/kayobe/environments/ci-multinode/storage.yml new file mode 100644 index 000000000..44b655ce7 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/storage.yml @@ -0,0 +1,5 @@ +--- +# List of storage volume groups. See mrlesmithjr.manage-lvm role for +# format. +storage_lvm_groups: + - "{{ stackhpc_lvm_group_rootvg }}" From 54dd5be1f6d3b02f37b0c69cdf265e9f593da169 Mon Sep 17 00:00:00 2001 From: Grzegorz Koper Date: Mon, 10 Oct 2022 17:49:20 +0200 Subject: [PATCH 089/292] Reencrypting docker_registry_password --- etc/kayobe/environments/ci-multinode/stackhpc-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml index 191b9a806..60c5eef5c 100644 --- a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml @@ -51,8 +51,8 @@ stackhpc_docker_registry: "{{ stackhpc_repo_mirror_url | regex_replace('^https?: stackhpc_docker_registry_username: "stackhpc-kayobe-ci" stackhpc_docker_registry_password: !vault | $ANSIBLE_VAULT;1.1;AES256 - 30333932623139383562343036366566363230656536376164633463366235623463323937303135 - 6533333638613764383136613730346436643831646530640a363262373964323461616638636135 - 65623338333236323030346564363035313462333335333032643661363834356432363833303838 - 3936616462303062350a316165396265313766316338303338336132316264316631376163643133 - 33616239633435636630373039616630323163343962336330303864626561303238 + 33356166343730633865363431306535613736663764373034396132356131343066636530393534 + 3262646436663034633131316438633230383330633533350a386365313239303464383636376338 + 61656662333939333063343131633963636431663136643137636664633233633133396339613861 + 3038613063626138610a333566393937643630366564653163613364323965396130613433316537 + 39653335393831633362343934363866346262613166393561666336623062393935 From 2ecf6f175b8dd633cfd172cac645d58bc052198f Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 13 Oct 2022 10:49:51 +0100 Subject: [PATCH 090/292] ci-multinode: Use insecure Docker registry Default changed in Xena --- etc/kayobe/environments/ci-multinode/stackhpc-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml index 60c5eef5c..8d0991f88 100644 --- a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml @@ -5,6 +5,11 @@ # Docker namespace to use for Kolla images. Default is 'kolla'. kolla_docker_namespace: stackhpc-dev +# Whether docker should be configured to use an insecure registry for Kolla +# # images. Default is false, unless docker_registry_enabled is true and +# # docker_registry_enable_tls is false. +kolla_docker_registry_insecure: "{{ 'https' not in stackhpc_repo_mirror_url }}" + ############################################################################### # Network configuration. From 89e5f87dca4d630ec61ee7cdbc4f975840d485cd Mon Sep 17 00:00:00 2001 From: Grzegorz Koper Date: Wed, 12 Oct 2022 11:13:54 +0200 Subject: [PATCH 091/292] Removing unnecessary config parts. Compute hosts are BM's --- etc/kayobe/environments/ci-multinode/kolla/globals.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/etc/kayobe/environments/ci-multinode/kolla/globals.yml b/etc/kayobe/environments/ci-multinode/kolla/globals.yml index ebfe64897..e7afedf68 100644 --- a/etc/kayobe/environments/ci-multinode/kolla/globals.yml +++ b/etc/kayobe/environments/ci-multinode/kolla/globals.yml @@ -1,14 +1,4 @@ --- -# Most development environments will use nested virtualisation, and we can't -# guarantee that nested KVM support is available. Use QEMU as a lowest common -# denominator. -nova_compute_virt_type: qemu - -# Reduce the control plane's memory footprint by limiting the number of worker -# processes to one per-service. -openstack_service_workers: "1" -openstack_service_rpc_workers: "1" - # Glance Ceph configuration glance_backend_ceph: "yes" From e0a8888e91d9ca6f86b8c49a77d6fa5e2cac77e2 Mon Sep 17 00:00:00 2001 From: Grzegorz Koper Date: Wed, 12 Oct 2022 11:17:53 +0200 Subject: [PATCH 092/292] Matching with ci-builder env --- etc/kayobe/environments/ci-multinode/stackhpc-ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml index 8d0991f88..9cfccab43 100644 --- a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml @@ -32,13 +32,15 @@ stackhpc_repo_epel_version: "{{ stackhpc_pulp_repo_epel_version }}" stackhpc_repo_epel_modular_version: "{{ stackhpc_pulp_repo_epel_modular_version }}" stackhpc_repo_docker_version: "{{ stackhpc_pulp_repo_docker_version }}" stackhpc_repo_centos_stream_advanced_virtualization_version: "{{ stackhpc_pulp_repo_centos_stream_8_advanced_virtualization_version }}" +stackhpc_repo_centos_stream_nfv_extras_version: "{{ stackhpc_pulp_repo_centos_stream_8_nfv_extras_version }}" stackhpc_repo_centos_stream_nfv_openvswitch_version: "{{ stackhpc_pulp_repo_centos_stream_8_nfv_openvswitch_version }}" -stackhpc_repo_centos_stream_openstack_wallaby_version: "{{ stackhpc_pulp_repo_centos_stream_8_openstack_wallaby_version }}" +stackhpc_repo_centos_stream_openstack_xena_version: "{{ stackhpc_pulp_repo_centos_stream_8_openstack_xena_version }}" +stackhpc_repo_centos_stream_opstools_version: "{{ stackhpc_pulp_repo_centos_stream_8_opstools_version }}" stackhpc_repo_centos_stream_powertools_version: "{{ stackhpc_pulp_repo_centos_stream_8_powertools_version }}" -stackhpc_repo_centos_opstools_version: "{{ stackhpc_pulp_repo_centos_8_opstools_version }}" stackhpc_repo_centos_stream_storage_ceph_pacific_version: "{{ stackhpc_pulp_repo_centos_stream_8_storage_ceph_pacific_version }}" stackhpc_repo_elasticsearch_logstash_kibana_7_x_version: "{{ stackhpc_pulp_repo_elasticsearch_logstash_kibana_7_x_version }}" stackhpc_repo_grafana_version: "{{ stackhpc_pulp_repo_grafana_version }}" +stackhpc_repo_mariadb_10_5_centos8_version: "{{ stackhpc_pulp_repo_mariadb_10_5_centos8_version }}" stackhpc_repo_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_rabbitmq_erlang_version }}" stackhpc_repo_rabbitmq_server_version: "{{ stackhpc_pulp_repo_rabbitmq_server_version }}" stackhpc_repo_treasuredata_4_version: "{{ stackhpc_pulp_repo_treasuredata_4_version }}" @@ -47,7 +49,6 @@ stackhpc_repo_ubuntu_focal_version: "{{ stackhpc_pulp_repo_ubuntu_focal_version stackhpc_repo_ubuntu_focal_security_version: "{{ stackhpc_pulp_repo_ubuntu_focal_security_version }}" stackhpc_repo_docker_ce_ubuntu_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_version }}" - # Host and port of container registry. # Push built images to the development Pulp service registry. stackhpc_docker_registry: "{{ stackhpc_repo_mirror_url | regex_replace('^https?://', '') }}" From 4ab17622a7ed1399040102326232f810843822e5 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 13 Oct 2022 11:01:44 +0100 Subject: [PATCH 093/292] ci-multinode: fix pep8 issues --- etc/kayobe/environments/ci-multinode/cephadm.yml | 1 - etc/kayobe/environments/ci-multinode/compute.yml | 3 +++ etc/kayobe/environments/ci-multinode/controllers.yml | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/environments/ci-multinode/cephadm.yml b/etc/kayobe/environments/ci-multinode/cephadm.yml index 812f89a06..b4cb2ca0d 100644 --- a/etc/kayobe/environments/ci-multinode/cephadm.yml +++ b/etc/kayobe/environments/ci-multinode/cephadm.yml @@ -59,4 +59,3 @@ cephadm_keys: osd: "profile rbd pool=images" mgr: "profile rbd pool=images" state: present - diff --git a/etc/kayobe/environments/ci-multinode/compute.yml b/etc/kayobe/environments/ci-multinode/compute.yml index 254afc8e4..74e7ef86c 100644 --- a/etc/kayobe/environments/ci-multinode/compute.yml +++ b/etc/kayobe/environments/ci-multinode/compute.yml @@ -1,2 +1,5 @@ +--- +# List of storage volume groups. See mrlesmithjr.manage-lvm role for +# format. compute_lvm_groups: - "{{ stackhpc_lvm_group_rootvg }}" diff --git a/etc/kayobe/environments/ci-multinode/controllers.yml b/etc/kayobe/environments/ci-multinode/controllers.yml index 9198a4d4c..3f612ecd8 100644 --- a/etc/kayobe/environments/ci-multinode/controllers.yml +++ b/etc/kayobe/environments/ci-multinode/controllers.yml @@ -1,2 +1,5 @@ +--- +# List of storage volume groups. See mrlesmithjr.manage-lvm role for +# format. controller_lvm_groups: - "{{ stackhpc_lvm_group_rootvg }}" From 47cf99af42592d5a8f42e258d862fa97426f35c2 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Fri, 14 Oct 2022 13:21:22 +0000 Subject: [PATCH 094/292] Adds Rocky 8 pulp repositories to dnf.yml (#152) * Adds Rocky 8 pulp repositories to dnf.yml This configures the hosts to use the mirrors in the local pulp. Prior to this change kayobe would attempt to configure CentOS stream mirrors on a Rocky host. * Use facts to pick dnf repositories --- etc/kayobe/dnf.yml | 53 +++++++++++++++++++++++++++++++---------- etc/kayobe/stackhpc.yml | 12 ++++++++++ 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/etc/kayobe/dnf.yml b/etc/kayobe/dnf.yml index 359c9cefc..c63276a28 100644 --- a/etc/kayobe/dnf.yml +++ b/etc/kayobe/dnf.yml @@ -41,7 +41,27 @@ # file: myrepo # gpgkey: http://gpgkey # gpgcheck: yes -dnf_custom_repos: +dnf_custom_repos: "{{ dnf_custom_repos_el8 | combine(lookup('vars', 'dnf_custom_repos_' ~ ansible_facts.distribution | lower )) }}" + +# Custom repositories shared between all RHEL derivatives. +dnf_custom_repos_el8: + epel: + baseurl: "{{ stackhpc_repo_epel_url }}" + description: "Extra Packages for Enterprise Linux $releasever - $basearch" + enabled: "{{ dnf_enable_epel | bool }}" + file: epel + gpgkey: "{{ dnf_epel_gpg_key_url }}" + gpgcheck: yes + epel-modular: + baseurl: "{{ stackhpc_repo_epel_modular_url }}" + description: "Extra Packages for Enterprise Linux Modular $releasever - $basearch" + enabled: "{{ dnf_enable_epel | bool }}" + file: epel-modular + gpgkey: "{{ dnf_epel_gpg_key_url }}" + gpgcheck: yes + +# CentOS Stream 8 specific repositories +dnf_custom_repos_centos: appstream: baseurl: "{{ stackhpc_repo_centos_stream_appstream_url }}" description: "CentOS Stream $releasever - AppStream" @@ -60,19 +80,26 @@ dnf_custom_repos: file: CentOS-Stream-Extras gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial gpgcheck: yes - epel: - baseurl: "{{ stackhpc_repo_epel_url }}" - description: "Extra Packages for Enterprise Linux $releasever - $basearch" - enabled: "{{ dnf_enable_epel | bool }}" - file: epel - gpgkey: "{{ dnf_epel_gpg_key_url }}" + +# Rocky 8 specific repositories +dnf_custom_repos_rocky: + appstream: + baseurl: "{{ stackhpc_repo_rocky_appstream_url }}" + description: "Rocky Linux $releasever - AppStream" + file: Rocky-AppStream + gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rockyofficial gpgcheck: yes - epel-modular: - baseurl: "{{ stackhpc_repo_epel_modular_url }}" - description: "Extra Packages for Enterprise Linux Modular $releasever - $basearch" - enabled: "{{ dnf_enable_epel | bool }}" - file: epel-modular - gpgkey: "{{ dnf_epel_gpg_key_url }}" + baseos: + baseurl: "{{ stackhpc_repo_rocky_baseos_url }}" + description: "Rocky Linux $releasever - BaseOS" + file: Rocky-BaseOS + gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rockyofficial + gpgcheck: yes + extras: + baseurl: "{{ stackhpc_repo_rocky_extras_url }}" + description: "Rocky Linux $releasever - Extras" + file: Rocky-Extras + gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rockyofficial gpgcheck: yes # Whether to enable EPEL repositories. This affects RedHat-based systems only. diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml index e0d41155c..b0d1b0d41 100644 --- a/etc/kayobe/stackhpc.yml +++ b/etc/kayobe/stackhpc.yml @@ -103,6 +103,18 @@ stackhpc_repo_rabbitmq_server_version: "{{ stackhpc_repo_distribution }}" stackhpc_repo_treasuredata_4_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/treasuredata/4/redhat/8/x86_64/{{ stackhpc_repo_treasuredata_4_version }}" stackhpc_repo_treasuredata_4_version: "{{ stackhpc_repo_distribution }}" +# Rocky 8 BaseOS +stackhpc_repo_rocky_baseos_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8/BaseOS/x86_64/os/{{ stackhpc_repo_rocky_baseos_version }}" +stackhpc_repo_rocky_baseos_version: "{{ stackhpc_repo_distribution }}" + +# Rocky 8 AppStream +stackhpc_repo_rocky_appstream_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8/AppStream/x86_64/os/{{ stackhpc_repo_rocky_appstream_version }}" +stackhpc_repo_rocky_appstream_version: "{{ stackhpc_repo_distribution }}" + +# Rocky 8 extras +stackhpc_repo_rocky_extras_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8/extras/x86_64/os/{{ stackhpc_repo_rocky_extras_version }}" +stackhpc_repo_rocky_extras_version: "{{ stackhpc_repo_distribution }}" + ############################################################################### # Sources From 7272f84c00c81d93df6fffa89584eb445ed7931e Mon Sep 17 00:00:00 2001 From: Grzegorz Koper Date: Wed, 19 Oct 2022 16:24:49 +0200 Subject: [PATCH 095/292] Removing the conditional. Forcing pvresize Growroot is being executed by cloud-init. Bug in growroot implementation doesnt extend the pv. --- etc/kayobe/ansible/growroot.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/etc/kayobe/ansible/growroot.yml b/etc/kayobe/ansible/growroot.yml index 6bfe63e41..f0134d00d 100644 --- a/etc/kayobe/ansible/growroot.yml +++ b/etc/kayobe/ansible/growroot.yml @@ -76,4 +76,3 @@ pv: "{{ pvs.stdout | from_json }}" disk: "{{ pv.report[0].pv[0].pv_name }}" become: True - when: "'NOCHANGE' not in growpart.stdout" From 8810a8899c9426deaa48ac353b168f3d9c289637 Mon Sep 17 00:00:00 2001 From: Grzegorz Koper Date: Wed, 19 Oct 2022 17:03:18 +0200 Subject: [PATCH 096/292] Hashing out previously used conditional. Adding bug explanation --- etc/kayobe/ansible/growroot.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/etc/kayobe/ansible/growroot.yml b/etc/kayobe/ansible/growroot.yml index f0134d00d..f1367df4c 100644 --- a/etc/kayobe/ansible/growroot.yml +++ b/etc/kayobe/ansible/growroot.yml @@ -76,3 +76,7 @@ pv: "{{ pvs.stdout | from_json }}" disk: "{{ pv.report[0].pv[0].pv_name }}" become: True +# when: "'NOCHANGE' not in growpart.stdout" +# Commenting out the conditional because growpart is already triggered by cloud-init - hence it emits NOCHANGE +# Cloud-Inits growpart implementation has a bug https://bugzilla.redhat.com/show_bug.cgi?id=2122575 +# PVresize is not being triggered From 670aca0458360ceb2b218a7caeb754382e0889b7 Mon Sep 17 00:00:00 2001 From: Piotr Parczewski Date: Thu, 20 Oct 2022 10:59:20 +0200 Subject: [PATCH 097/292] Indicate source of the alert rules --- etc/kayobe/kolla/config/prometheus/blackbox.rules | 2 ++ etc/kayobe/kolla/config/prometheus/rabbitmq.rules | 2 ++ etc/kayobe/kolla/config/prometheus/system.rules | 2 ++ 3 files changed, 6 insertions(+) diff --git a/etc/kayobe/kolla/config/prometheus/blackbox.rules b/etc/kayobe/kolla/config/prometheus/blackbox.rules index 5f91d07e7..59e144669 100644 --- a/etc/kayobe/kolla/config/prometheus/blackbox.rules +++ b/etc/kayobe/kolla/config/prometheus/blackbox.rules @@ -1,3 +1,5 @@ +# https://awesome-prometheus-alerts.grep.to/rules#blackbox-1 + {% raw %} groups: diff --git a/etc/kayobe/kolla/config/prometheus/rabbitmq.rules b/etc/kayobe/kolla/config/prometheus/rabbitmq.rules index 30c18eb97..33f27cc9e 100644 --- a/etc/kayobe/kolla/config/prometheus/rabbitmq.rules +++ b/etc/kayobe/kolla/config/prometheus/rabbitmq.rules @@ -1,3 +1,5 @@ +# https://awesome-prometheus-alerts.grep.to/rules#rabbitmq-1 + {% raw %} groups: diff --git a/etc/kayobe/kolla/config/prometheus/system.rules b/etc/kayobe/kolla/config/prometheus/system.rules index 22d090aae..76d301fbf 100644 --- a/etc/kayobe/kolla/config/prometheus/system.rules +++ b/etc/kayobe/kolla/config/prometheus/system.rules @@ -1,3 +1,5 @@ +# https://awesome-prometheus-alerts.grep.to/rules#host-and-hardware-1 + {% raw %} groups: From 916a9363086615f9fb54f51c6269a51dca23559f Mon Sep 17 00:00:00 2001 From: Piotr Parczewski Date: Thu, 20 Oct 2022 12:11:07 +0200 Subject: [PATCH 098/292] Add monitoring group --- etc/kayobe/environments/ci-aio/inventory/hosts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/environments/ci-aio/inventory/hosts b/etc/kayobe/environments/ci-aio/inventory/hosts index b041573df..a16582738 100644 --- a/etc/kayobe/environments/ci-aio/inventory/hosts +++ b/etc/kayobe/environments/ci-aio/inventory/hosts @@ -1,6 +1,9 @@ -# A single all-in-one controller/compute host. +# A single all-in-one controller/compute/monitoring host. [controllers] controller0 [compute:children] controllers + +[monitoring:children] +controllers From 56911ceb738d7bb77aac05fb9209cf799d957524 Mon Sep 17 00:00:00 2001 From: Piotr Parczewski Date: Thu, 20 Oct 2022 15:57:41 +0200 Subject: [PATCH 099/292] Use shorter regexp for catching physical network cards --- etc/kayobe/kolla/config/prometheus/ceph.rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/kolla/config/prometheus/ceph.rules b/etc/kayobe/kolla/config/prometheus/ceph.rules index ccbc95da1..7df171501 100644 --- a/etc/kayobe/kolla/config/prometheus/ceph.rules +++ b/etc/kayobe/kolla/config/prometheus/ceph.rules @@ -154,7 +154,7 @@ groups: # alert on nic packet errors and drops rates > 1 packet/s - alert: NetworkPacketsDropped - expr: irate(node_network_receive_drop_total{device=~"eno.*|ens.*|eth.*|enp.*"}[5m]) + irate(node_network_transmit_drop_total{device=~"eno.*|ens.*|eth.*|enp.*"}[5m]) > 1 + expr: irate(node_network_receive_drop_total{device=~"en.*|eth.*"}[5m]) + irate(node_network_transmit_drop_total{device=~"en.*|eth.*"}[5m]) > 1 labels: severity: warning annotations: From 6b5d1ca8c2d5ff276d148cf3d8ccce8a770f9d80 Mon Sep 17 00:00:00 2001 From: Piotr Parczewski Date: Thu, 20 Oct 2022 18:20:29 +0200 Subject: [PATCH 100/292] Build Grafana with additional panel plugin The old version is still required by some (namely Ceph) dashboards. --- etc/kayobe/kolla.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 806ecfed8..f0e09ee76 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -234,7 +234,8 @@ kolla_build_blocks: bifrost_base_header: | ADD additions-archive / grafana_plugins_install: | - RUN grafana-cli plugins install vonage-status-panel + RUN grafana-cli plugins install vonage-status-panel \ + && grafana-cli plugins install grafana-piechart-panel ironic_inspector_header: | ADD additions-archive / From c536ec8f128992dbc3969f478aa9b2096b418b95 Mon Sep 17 00:00:00 2001 From: Grzegorz Koper Date: Mon, 24 Oct 2022 11:49:17 +0200 Subject: [PATCH 101/292] changing release to point to latest build --- etc/kayobe/kolla.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index b4e8afe6b..ad630d963 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -96,7 +96,7 @@ kolla_docker_registry_password: "{{ stackhpc_docker_registry_password }}" # Kolla OpenStack release version. This should be a Docker image tag. # Default is {{ openstack_release }}. -kolla_openstack_release: xena-20220728T100435 +kolla_openstack_release: xena-20221010T103511 # Docker tag applied to built container images. Default is # {{ kolla_openstack_release }}. From ceccb5973c081a7e26bc4382f60a2044bd3ccc8b Mon Sep 17 00:00:00 2001 From: Grzegorz Koper Date: Mon, 24 Oct 2022 12:04:53 +0200 Subject: [PATCH 102/292] dropping not needed overrides --- etc/kayobe/kolla/globals.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 79f05f90c..2c1be0cf5 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -2,8 +2,3 @@ --- docker_yum_baseurl: "{{ stackhpc_repo_docker_url }}" docker_yum_gpgkey: "https://download.docker.com/linux/centos/gpg" - -{% if kolla_base_distro == 'centos' %} -barbican_tag: xena-20220728T143658 -prometheus_tag: xena-20220728T143658 -{% endif %} From 278da4103ecbfa19eaab78c4712e36e029eaa7d9 Mon Sep 17 00:00:00 2001 From: Scott Davidson Date: Mon, 31 Oct 2022 20:16:04 +0000 Subject: [PATCH 103/292] Add basic docs template structure --- tox.ini | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tox.ini b/tox.ini index 787fe1044..ef2a6989c 100644 --- a/tox.ini +++ b/tox.ini @@ -23,3 +23,13 @@ deps = commands = rm -rf releasenotes/build/html sphinx-build -n -W --keep-going -b html releasenotes/source releasenotes/build/html + +[testenv:docs] +deps = + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -r{toxinidir}/doc/requirements.txt +commands = + rm -rf doc/build/html + sphinx-build -b html doc/source doc/build/html + ; sphinx-build -W --keep-going -b html doc/source doc/build/html + ; whereto doc/source/_extra/.htaccess doc/test/redirect-tests.txt \ No newline at end of file From 05df5fa8fa41cb39a7fac70cde2c13225d701c32 Mon Sep 17 00:00:00 2001 From: Scott Davidson Date: Mon, 31 Oct 2022 20:16:04 +0000 Subject: [PATCH 104/292] Add basic docs template structure --- doc/requirements.txt | 9 ++++ doc/source/conf.py | 98 ++++++++++++++++++++++++++++++++++++++++++++ doc/source/index.rst | 27 ++++++++++++ 3 files changed, 134 insertions(+) create mode 100644 doc/requirements.txt create mode 100644 doc/source/conf.py create mode 100644 doc/source/index.rst diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 000000000..51a2c6498 --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1,9 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. + +openstackdocstheme>=2.2.1 # Apache-2.0 +reno>=3.1.0 # Apache-2.0 +sphinx>=2.0.0,!=2.1.0 # BSD +sphinxcontrib-svg2pdfconverter>=0.1.0 # BSD +whereto>=0.3.0 # Apache-2.0 diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 000000000..39fbb51ac --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,98 @@ +# Copyright (c) 2017 StackHPC Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import openstackdocstheme + + +# -- General configuration ---------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ + # 'openstackdocstheme', + #'sphinx.ext.autodoc', + #'sphinx.ext.intersphinx', + 'sphinxcontrib.rsvgconverter', +] + +# autodoc generation is a bit aggressive and a nuisance when doing heavy +# text edit cycles. +# execute "export SPHINX_DEBUG=1" in your terminal to disable + +# The suffix of source filenames. +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'kayobe-config' +# copyright = '2013, OpenStack Foundation' + +# If true, '()' will be appended to :func: etc. cross-reference text. +add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +add_module_names = True + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'native' + +# -- Options for HTML output -------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +# html_theme_path = [] +html_theme = 'default' +# html_static_path = ['static'] + +# Add any paths that contain "extra" files, such as .htaccess or +# robots.txt. +html_extra_path = ['_extra'] + +html_theme_options = { + "show_other_versions": True, +} + +# Output file base name for HTML help builder. +htmlhelp_basename = '%sdoc' % project + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass +# [howto/manual]). +# latex_documents = [ +# ('index', +# 'doc-%s.tex' % project, +# '%s Documentation' % project, +# 'OpenStack Foundation', 'manual'), +# ] + +# Disable usage of xindy https://bugzilla.redhat.com/show_bug.cgi?id=1643664 +latex_use_xindy = False + diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 000000000..b04bb46ad --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,27 @@ +.. kayobe documentation master file, created by + sphinx-quickstart on Tue Jul 9 22:26:36 2013. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +================================================== +Welcome to StackHPC's Kayobe Config documentation! +================================================== + + +Overview +======== + + + +Contents +======== + + + +Intro +===== + + + +Contributors Guide +================= From debe0d6a38c7325eb9ba3d61e8e6d6341a5e4de8 Mon Sep 17 00:00:00 2001 From: Scott Davidson Date: Mon, 31 Oct 2022 20:22:57 +0000 Subject: [PATCH 105/292] Add basic docs template structure --- doc/requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index 51a2c6498..95b160c9c 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -2,8 +2,6 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -openstackdocstheme>=2.2.1 # Apache-2.0 reno>=3.1.0 # Apache-2.0 sphinx>=2.0.0,!=2.1.0 # BSD sphinxcontrib-svg2pdfconverter>=0.1.0 # BSD -whereto>=0.3.0 # Apache-2.0 From 020e29f2075ca4add2eb05fca154bf643ec88efb Mon Sep 17 00:00:00 2001 From: Scott Davidson Date: Mon, 31 Oct 2022 20:22:57 +0000 Subject: [PATCH 106/292] Add basic docs template structure --- doc/source/conf.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 39fbb51ac..e15654d3a 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -26,15 +26,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import openstackdocstheme - - # -- General configuration ---------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ - # 'openstackdocstheme', #'sphinx.ext.autodoc', #'sphinx.ext.intersphinx', 'sphinxcontrib.rsvgconverter', From 0ae82c333a583ecf9fa68b455b4ca2c124ba201e Mon Sep 17 00:00:00 2001 From: Scott Davidson Date: Mon, 31 Oct 2022 20:53:27 +0000 Subject: [PATCH 107/292] Remove openstack reference --- doc/source/conf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index e15654d3a..b3f378449 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -48,7 +48,6 @@ # General information about the project. project = 'kayobe-config' -# copyright = '2013, OpenStack Foundation' # If true, '()' will be appended to :func: etc. cross-reference text. add_function_parentheses = True From 32d3c7988807aa47d75b8a81e84ceef6ba9b33c8 Mon Sep 17 00:00:00 2001 From: Scott Davidson Date: Tue, 1 Nov 2022 13:22:17 +0000 Subject: [PATCH 108/292] Changes based on Mark's comments --- .github/workflows/stackhpc-pull-request.yml | 2 ++ doc/source/conf.py | 6 +++--- doc/source/index.rst | 2 +- tox.ini | 8 ++++---- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/stackhpc-pull-request.yml b/.github/workflows/stackhpc-pull-request.yml index da4760a84..0504103d3 100644 --- a/.github/workflows/stackhpc-pull-request.yml +++ b/.github/workflows/stackhpc-pull-request.yml @@ -14,6 +14,8 @@ jobs: python-version: 3.8 - environment: releasenotes python-version: 3.8 + - environment: docs + python-version: 3.8 name: Tox ${{ matrix.environment }} with Python ${{ matrix.python-version }} if: github.repository == 'stackhpc/stackhpc-kayobe-config' steps: diff --git a/doc/source/conf.py b/doc/source/conf.py index b3f378449..f7214fed0 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -47,7 +47,7 @@ master_doc = 'index' # General information about the project. -project = 'kayobe-config' +project = 'stackhpc-kayobe-config' # If true, '()' will be appended to :func: etc. cross-reference text. add_function_parentheses = True @@ -69,10 +69,10 @@ # Add any paths that contain "extra" files, such as .htaccess or # robots.txt. -html_extra_path = ['_extra'] +# html_extra_path = ['_extra'] html_theme_options = { - "show_other_versions": True, + # "show_other_versions": True, } # Output file base name for HTML help builder. diff --git a/doc/source/index.rst b/doc/source/index.rst index b04bb46ad..24f01975d 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -24,4 +24,4 @@ Intro Contributors Guide -================= +================== diff --git a/tox.ini b/tox.ini index ef2a6989c..5e1f5a7cb 100644 --- a/tox.ini +++ b/tox.ini @@ -25,11 +25,11 @@ commands = sphinx-build -n -W --keep-going -b html releasenotes/source releasenotes/build/html [testenv:docs] +allowlist_externals = rm +skip_install = true deps = - -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/xena} -r{toxinidir}/doc/requirements.txt commands = rm -rf doc/build/html - sphinx-build -b html doc/source doc/build/html - ; sphinx-build -W --keep-going -b html doc/source doc/build/html - ; whereto doc/source/_extra/.htaccess doc/test/redirect-tests.txt \ No newline at end of file + sphinx-build -W --keep-going -b html doc/source doc/build/html \ No newline at end of file From 89ba9beafce07f342a6f04107bf6932aa0a18a60 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 2 Nov 2022 11:26:40 +0000 Subject: [PATCH 109/292] Add initial bifrost tag for Xena --- etc/kayobe/kolla/globals.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index a59e1d492..514bf6aad 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -3,6 +3,10 @@ docker_yum_baseurl: "{{ stackhpc_repo_docker_url }}" docker_yum_gpgkey: "https://download.docker.com/linux/centos/gpg" +{% if kolla_base_distro == 'centos' %} +bifrost_tag: xena-20221102T110017 +{% endif %} + # Monitoring and alerting related settings es_heap_size: 8g -prometheus_cmdline_extras: "--storage.tsdb.retention.time=30d" \ No newline at end of file +prometheus_cmdline_extras: "--storage.tsdb.retention.time=30d" From 23820c4e4afe2e0d90a9823e84530c5e99b8d709 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 2 Nov 2022 11:29:55 +0000 Subject: [PATCH 110/292] Revert "Rocky 8: Install a compatible version of ansible in the kolla venv" This reverts commit 8adf063b2c1e7fb565f1e950e03aee011a3e1f50. It was merged in from stackhpc/wallaby, but is not necessary in Xena. --- etc/kayobe/kolla.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 286a03acc..61b7253e7 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -48,12 +48,7 @@ kolla_ansible_source_version: "{{ stackhpc_kolla_ansible_source_version }}" #kolla_ansible_venv: # Extra requirements to install inside the kolla-ansible virtualenv. -kolla_ansible_venv_extra_requirements: "{{ lookup('vars', 'kolla_ansible_venv_extra_requirements_' ~ os_distribution, default=[]) }}" - -# Rocky specific requirements in the kolla-ansible virtualenv -kolla_ansible_venv_extra_requirements_rocky: - # NOTE(wszumski): This is wallaby specific as we can use ansible-core 2.11 in Xena. - - 'ansible-base@git+https://github.com/stackhpc/ansible@stackhpc/2.10/rocky' +#kolla_ansible_venv_extra_requirements: # Pip requirement specifier for the ansible package. NOTE: This limits the # version of ansible used by kolla-ansible to avoid new releases from breaking From 9dcc4ec836eefa4f2f797a31175cc860248a7107 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 19 Oct 2022 13:56:42 +0100 Subject: [PATCH 111/292] Make prometheus datasource configurable. The use case is when you have a single grafana showing metrics from multiple clouds. This allows you to toggle between the clouds by selecting the datasource. --- .../grafana/dashboards/ceph/ceph_mds.json | 2 +- .../grafana/dashboards/ceph/ceph_osds.json | 2 +- .../dashboards/ceph/ceph_overview.json | 2 +- .../grafana/dashboards/ceph/ceph_pools.json | 2 +- .../dashboards/openstack/blackbox.json | 42 +- .../dashboards/openstack/cadvisor.json | 64 ++- .../dashboards/openstack/database.json | 42 +- .../dashboards/openstack/elasticsearch.json | 390 ++++++++++++++---- .../grafana/dashboards/openstack/fluentd.json | 112 ++--- .../grafana/dashboards/openstack/haproxy.json | 144 ++++--- .../grafana/dashboards/openstack/libvirt.json | 125 +++++- .../dashboards/openstack/memcached.json | 65 ++- .../openstack/node_exporter_full.json | 309 +++++++------- .../openstack/node_exporter_side_by_side.json | 250 ++++++++--- .../dashboards/openstack/openstack.json | 110 ++++- .../dashboards/openstack/prometheus.json | 190 +++++++-- .../openstack/prometheus_alertmanager.json | 2 +- .../openstack/prometheus_benchmark.json | 310 +++++++++++--- .../dashboards/openstack/rabbitmq.json | 130 +++--- .../grafana/dashboards/openstack/redfish.json | 200 +++++++-- 20 files changed, 1831 insertions(+), 662 deletions(-) diff --git a/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_mds.json b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_mds.json index c3e6a0ec6..3dd24041a 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_mds.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_mds.json @@ -240,7 +240,7 @@ }, "hide": 0, "includeAll": false, - "label": "Data Source", + "label": "datasource", "multi": false, "name": "datasource", "options": [], diff --git a/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_osds.json b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_osds.json index 64e0895bc..6c6f525a7 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_osds.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_osds.json @@ -905,7 +905,7 @@ }, "hide": 0, "includeAll": false, - "label": "Data Source", + "label": "datasource", "multi": false, "name": "datasource", "options": [], diff --git a/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_overview.json b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_overview.json index 4693e139f..c329878a8 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_overview.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_overview.json @@ -2787,7 +2787,7 @@ }, "hide": 0, "includeAll": false, - "label": "Data Source", + "label": "datasource", "multi": false, "name": "datasource", "options": [], diff --git a/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_pools.json b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_pools.json index 847350a0f..f2882ed60 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_pools.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_pools.json @@ -684,7 +684,7 @@ }, "hide": 0, "includeAll": false, - "label": "Data Source", + "label": "datasource", "multi": false, "name": "datasource", "options": [], diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/blackbox.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/blackbox.json index dffd96a4c..0f85abdba 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/openstack/blackbox.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/blackbox.json @@ -35,7 +35,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fill": 1, "fillGradient": 0, @@ -129,7 +129,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -223,7 +223,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fill": 1, "fillGradient": 0, @@ -307,7 +307,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fill": 1, "fillGradient": 0, @@ -387,7 +387,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -478,7 +478,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -551,7 +551,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -641,7 +641,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -732,7 +732,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -806,7 +806,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -888,6 +888,26 @@ ], "templating": { "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, { "auto": true, "auto_count": 10, @@ -989,7 +1009,7 @@ }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "definition": "", "hide": 0, diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/cadvisor.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/cadvisor.json index 3ac11afb4..02717e0e7 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/openstack/cadvisor.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/cadvisor.json @@ -51,7 +51,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -129,7 +129,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -193,7 +193,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "exemplar": true, "expr": "count(container_last_seen{job=\"$job\", instance=\"$node\"})", @@ -207,7 +207,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "expr": "", "refId": "B" @@ -242,7 +242,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "editable": true, "error": false, @@ -298,7 +298,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "exemplar": true, "expr": "avg by (instance) (irate(container_cpu_usage_seconds_total{name=~\".+\",job=\"$job\", instance=\"$node\"}[$interval])) * 100", @@ -354,7 +354,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "editable": true, "error": false, @@ -408,7 +408,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(container_memory_usage_bytes{name=~\".+\",job=\"$job\", instance=\"$node\"}) by (instance)", @@ -480,7 +480,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "editable": true, "error": false, @@ -536,7 +536,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "exemplar": true, "expr": "rate(container_network_receive_bytes_total{id=\"/\",job=\"$job\", instance=\"$node\"}[$interval])", @@ -551,7 +551,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "exemplar": true, "expr": "- rate(container_network_transmit_bytes_total{id=\"/\",job=\"$job\", instance=\"$node\"}[$interval])", @@ -619,7 +619,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "editable": true, "error": false, @@ -724,7 +724,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "editable": true, "error": false, @@ -779,7 +779,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "expr": "sum(container_memory_usage_bytes{name=~\".+\",job=\"$job\", instance=\"$node\"}) by (name)", "hide": false, @@ -843,7 +843,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "editable": true, "error": false, @@ -899,7 +899,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "exemplar": true, "expr": "rate(container_network_receive_bytes_total{id=\"/\",job=\"$job\", instance=\"$node\"}[$interval])", @@ -950,7 +950,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "editable": true, "error": false, @@ -1006,7 +1006,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "exemplar": true, "expr": "rate(container_network_transmit_bytes_total{id=\"/\",job=\"$job\", instance=\"$node\"}[$interval])", @@ -1019,7 +1019,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "exemplar": true, "expr": "rate(container_network_transmit_bytes_total{id=\"/\",job=\"$job\", instance=\"$node\"}[$interval])", @@ -1070,6 +1070,26 @@ "tags": [], "templating": { "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, { "allValue": ".+", "current": { @@ -1083,7 +1103,7 @@ }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "definition": "label_values(container_cpu_load_average_10s, name)", "hide": 0, @@ -1213,7 +1233,7 @@ }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "definition": "label_values(container_cpu_usage_seconds_total, job)", "hide": 0, @@ -1240,7 +1260,7 @@ }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "definition": "label_values(container_cpu_usage_seconds_total{job=\"$job\"}, instance)", "hide": 0, diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/database.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/database.json index 70fb95e08..1db2050e1 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/openstack/database.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/database.json @@ -52,7 +52,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -135,7 +135,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -216,7 +216,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -336,7 +336,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -453,7 +453,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -559,7 +559,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -654,7 +654,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -765,7 +765,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "The number of connections that were aborted because the client died without closing the connection properly. See Section B.5.2.10, “Communication Errors and Aborted Connections”.", "fieldConfig": { @@ -859,7 +859,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "The number of failed attempts to connect to the MySQL server. See Section B.5.2.10, “Communication Errors and Aborted Connections”.\n\nFor additional connection-related information, check the Connection_errors_xxx status variables and the host_cache table.", "fieldConfig": { @@ -954,6 +954,26 @@ ], "templating": { "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, { "current": { "selected": false, @@ -961,8 +981,8 @@ "value": "10.3.28" }, "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "type": "prometheus", + "uid": "${datasource}" }, "definition": "label_values(mysql_version_info, innodb_version)", "hide": 0, diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/elasticsearch.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/elasticsearch.json index e4522f306..091c76760 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/openstack/elasticsearch.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/elasticsearch.json @@ -91,7 +91,10 @@ "rgba(237, 129, 40, 0.89)", "#299c46" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "format": "none", "gauge": { "maxValue": 100, @@ -190,7 +193,10 @@ "rgba(237, 129, 40, 0.89)", "#d44a3a" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "decimals": null, "format": "none", "gauge": { @@ -285,7 +291,10 @@ "rgba(237, 129, 40, 0.89)", "rgba(245, 54, 54, 0.9)" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "format": "percent", @@ -374,7 +383,10 @@ "rgba(237, 129, 40, 0.89)", "rgba(245, 54, 54, 0.9)" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "format": "percent", @@ -463,7 +475,10 @@ "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Number of nodes in the cluster", "editable": true, "error": false, @@ -553,7 +568,10 @@ "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Number of data nodes in the cluster", "editable": true, "error": false, @@ -643,7 +661,10 @@ "rgba(237, 129, 40, 0.89)", "rgba(245, 54, 54, 0.9)" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Cluster level changes which have not yet been executed", "editable": true, "error": false, @@ -734,7 +755,10 @@ "rgba(237, 129, 40, 0.89)", "rgba(245, 54, 54, 0.9)" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "format": "short", @@ -832,7 +856,10 @@ "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "The number of primary shards in your cluster. This is an aggregate total across all indices.", "editable": true, "error": false, @@ -922,7 +949,10 @@ "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Aggregate total of all shards across all indices, which includes replica shards", "editable": true, "error": false, @@ -1011,7 +1041,10 @@ "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Count of shards that are being freshly created", "editable": true, "error": false, @@ -1100,7 +1133,10 @@ "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "The number of shards that are currently moving from one node to another node.", "editable": true, "error": false, @@ -1189,7 +1225,10 @@ "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Shards delayed to reduce reallocation overhead", "editable": true, "error": false, @@ -1278,7 +1317,10 @@ "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "The number of shards that exist in the cluster state, but cannot be found in the cluster itself", "editable": true, "error": false, @@ -1378,7 +1420,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -1482,7 +1527,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -1597,7 +1645,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -1679,7 +1730,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -1777,7 +1831,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -1861,7 +1918,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -1968,7 +2028,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -2086,7 +2149,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -2182,7 +2248,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 0, @@ -2293,7 +2362,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 0, @@ -2410,7 +2482,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -2520,7 +2595,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -2642,7 +2720,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "decimals": 2, "editable": true, "error": false, @@ -2739,7 +2820,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -2832,7 +2916,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Count of deleted documents on this node", "editable": true, "error": false, @@ -2928,7 +3015,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "decimals": 2, "editable": true, "error": false, @@ -3023,7 +3113,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -3133,7 +3226,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -3226,7 +3322,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -3319,7 +3418,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -3414,7 +3516,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -3525,7 +3630,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -3692,7 +3800,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -3883,7 +3994,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -3971,7 +4085,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -4059,7 +4176,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -4148,7 +4268,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -4252,7 +4375,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -4347,7 +4473,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -4442,7 +4571,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -4537,7 +4669,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -4632,7 +4767,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -4743,7 +4881,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -4827,7 +4968,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -4927,7 +5071,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -5011,7 +5158,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -5095,7 +5245,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -5195,7 +5348,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -5280,7 +5436,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -5381,7 +5540,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -5465,7 +5627,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -5549,7 +5714,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -5633,7 +5801,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -5733,7 +5904,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -5817,7 +5991,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -5917,7 +6094,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -6001,7 +6181,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -6101,7 +6284,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -6185,7 +6371,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -6285,7 +6474,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -6369,7 +6561,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -6469,7 +6664,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -6553,7 +6751,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -6653,7 +6854,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -6737,7 +6941,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -6837,7 +7044,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -6922,7 +7132,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "gridPos": { "h": 7, @@ -7017,6 +7230,26 @@ ], "templating": { "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, { "auto": true, "auto_count": 30, @@ -7097,7 +7330,10 @@ "text": "kolla_logging", "value": "kolla_logging" }, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "definition": "", "hide": 0, "includeAll": false, @@ -7123,7 +7359,10 @@ "text": "All", "value": "$__all" }, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "definition": "", "hide": 0, "includeAll": true, @@ -7148,7 +7387,10 @@ "text": "10.205.1.4:9108", "value": "10.205.1.4:9108" }, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "definition": "", "hide": 0, "includeAll": false, diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/fluentd.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/fluentd.json index 2339db271..248b8d426 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/openstack/fluentd.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/fluentd.json @@ -5,8 +5,8 @@ { "builtIn": 1, "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "type": "prometheus", + "uid": "${datasource}" }, "enable": true, "hide": true, @@ -30,7 +30,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -50,7 +50,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "If you see errors then you probbaly have serious issues with log processing, see https://docs.fluentd.org/buffer#handling-unrecoverable-errors\n\nRetries are normal but should occur only from time to time, otherwise check for network errors or destination is too slow and requires additional tuning per given provider.", "fieldConfig": { @@ -108,7 +108,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(rate(fluentd_output_status_retry_count[5m]))", @@ -122,7 +122,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "expr": "sum(rate(fluentd_output_status_num_errors[5m]))", "format": "time_series", @@ -170,7 +170,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "Input and output total rates", "fieldConfig": { @@ -276,7 +276,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "This should not reach 0 otherwise logs are blocked from processing or even dropped", "fieldConfig": { @@ -376,7 +376,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "total flush time", "fieldConfig": { @@ -490,7 +490,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "Current total size of stage and queue buffers.\nfluentd_output_status_buffer_total_bytes", "fieldConfig": { @@ -587,7 +587,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -680,7 +680,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -700,7 +700,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "fluentd_output_status_buffer_stage_length", "fieldConfig": { @@ -797,7 +797,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "fluentd_output_status_buffer_stage_byte_size", "fieldConfig": { @@ -891,7 +891,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -911,7 +911,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -1008,7 +1008,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -1105,7 +1105,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "fluentd_output_status_buffer_queue_length", "fieldConfig": { @@ -1202,7 +1202,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "fluentd_output_status_queue_byte_size", "fieldConfig": { @@ -1296,7 +1296,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -1316,7 +1316,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "fluentd_output_status_buffer_available_space_ratio", "fieldConfig": { @@ -1411,7 +1411,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -1431,7 +1431,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "fluentd_output_status_retry_count", "fieldConfig": { @@ -1528,7 +1528,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "fluentd_output_status_emit_records", "fieldConfig": { @@ -1625,7 +1625,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "fluentd_output_status_emit_count", "fieldConfig": { @@ -1722,7 +1722,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "fluentd_output_status_rollback_count", "fieldConfig": { @@ -1819,7 +1819,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "fluentd_output_status_write_count", "fieldConfig": { @@ -1916,7 +1916,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "fluentd_output_status_slow_flush_count", "fieldConfig": { @@ -2013,7 +2013,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "fluentd_output_status_retry_wait", "fieldConfig": { @@ -2110,7 +2110,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "fluentd_output_status_flush_time_count", "fieldConfig": { @@ -2204,7 +2204,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -2224,7 +2224,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "fluentd_output_status_num_errors", "fieldConfig": { @@ -2321,7 +2321,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "fluentd_output_status_buffer_newest_timekey", "fieldConfig": { @@ -2415,7 +2415,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -2435,7 +2435,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -2532,7 +2532,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "fluentd_output_status_buffer_oldest_timekey", "fieldConfig": { @@ -2626,7 +2626,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -2643,7 +2643,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "fluentd_input_status_num_records_total", "fieldConfig": { @@ -2694,8 +2694,8 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "type": "prometheus", + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(rate(fluentd_input_status_num_records_total{cluster=\"$cluster\", instance=\"$instance\"}[5m])) ", @@ -2745,7 +2745,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "fluentd_input_status_num_records_total", "fieldConfig": { @@ -2842,7 +2842,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "fluentd_input_status_num_records_total", "fieldConfig": { @@ -2939,7 +2939,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "fluentd_input_status_num_records_total", "fieldConfig": { @@ -3036,7 +3036,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "fluentd_input_status_num_records_total", "fieldConfig": { @@ -3140,6 +3140,26 @@ ], "templating": { "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, { "current": { "selected": false, @@ -3147,8 +3167,8 @@ "value": "kolla_logging" }, "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "type": "prometheus", + "uid": "${datasource}" }, "definition": "label_values(cluster)", "hide": 0, diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/haproxy.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/haproxy.json index 82f50ffa6..ea735b549 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/openstack/haproxy.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/haproxy.json @@ -56,7 +56,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 0, "fieldConfig": { @@ -184,7 +184,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -322,7 +322,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 1, "description": "", @@ -450,7 +450,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -573,7 +573,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -731,7 +731,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -880,7 +880,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 1, "description": "", @@ -996,7 +996,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 1, "description": "", @@ -1129,7 +1129,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 1, "description": "", @@ -1262,7 +1262,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -1374,7 +1374,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -1492,7 +1492,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -1593,7 +1593,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -1647,7 +1647,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "exemplar": true, "expr": "haproxy_backend_current_queue{backend=~\"$backend\",instance=~\"$host\"}", @@ -1716,7 +1716,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -1838,7 +1838,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -1978,7 +1978,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -2073,7 +2073,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -2185,7 +2185,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -2281,7 +2281,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -2377,7 +2377,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -2472,7 +2472,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -2567,7 +2567,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -2675,7 +2675,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -2783,7 +2783,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -2879,7 +2879,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -3004,7 +3004,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 0, "description": "", @@ -3100,7 +3100,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -3212,7 +3212,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -3260,7 +3260,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "exemplar": true, "expr": "irate(haproxy_server_sessions_total[5m])", @@ -3329,7 +3329,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -3424,7 +3424,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -3536,7 +3536,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "editable": true, "error": false, @@ -3641,7 +3641,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -3784,7 +3784,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "editable": true, "error": false, @@ -3896,7 +3896,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -3991,7 +3991,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -4086,7 +4086,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -4181,7 +4181,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -4293,7 +4293,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -4388,7 +4388,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 1, "description": "", @@ -4483,7 +4483,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 0, "description": "", @@ -4579,7 +4579,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 1, "description": "", @@ -4628,7 +4628,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "exemplar": true, "expr": "haproxy_server_check_duration_milliseconds", @@ -4679,7 +4679,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -4774,7 +4774,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "editable": true, @@ -4886,7 +4886,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "editable": true, "error": false, @@ -4939,7 +4939,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum by (instance) (haproxy_exporter_csv_parse_failures)", @@ -4991,7 +4991,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "editable": true, "error": false, @@ -5044,7 +5044,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum by (instance) (haproxy_exporter_total_scrapes)", @@ -5099,6 +5099,26 @@ ], "templating": { "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, { "current": { "selected": false, @@ -5106,8 +5126,8 @@ "value": "$__all" }, "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "type": "prometheus", + "uid": "${datasource}" }, "definition": "label_values(haproxy_up{cluster='$cluster'}, instance)", "hide": 0, @@ -5138,8 +5158,8 @@ ] }, "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "type": "prometheus", + "uid": "${datasource}" }, "definition": "", "hide": 0, @@ -5170,8 +5190,8 @@ ] }, "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "type": "prometheus", + "uid": "${datasource}" }, "definition": "", "hide": 0, @@ -5201,8 +5221,8 @@ ] }, "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "type": "prometheus", + "uid": "${datasource}" }, "definition": "", "hide": 0, @@ -5233,8 +5253,8 @@ ] }, "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "type": "prometheus", + "uid": "${datasource}" }, "definition": "", "hide": 0, @@ -5317,8 +5337,8 @@ "value": "" }, "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "type": "prometheus", + "uid": "${datasource}" }, "definition": "label_values(haproxy_up, cluster)", "hide": 0, diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/libvirt.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/libvirt.json index 44ae2853f..13a37d3d5 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/openstack/libvirt.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/libvirt.json @@ -38,7 +38,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 8, "fillGradient": 0, "gridPos": { @@ -122,7 +125,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 0, "fillGradient": 0, "gridPos": { @@ -213,7 +219,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 8, "fillGradient": 0, "gridPos": { @@ -297,7 +306,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 0, "fillGradient": 0, "gridPos": { @@ -394,7 +406,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 8, "fillGradient": 0, "gridPos": { @@ -478,7 +493,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 8, "fillGradient": 0, "gridPos": { @@ -562,7 +580,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 8, "fillGradient": 0, "gridPos": { @@ -646,7 +667,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 8, "fillGradient": 0, "gridPos": { @@ -743,7 +767,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 8, "fillGradient": 0, "gridPos": { @@ -827,7 +854,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 8, "fillGradient": 0, "gridPos": { @@ -915,7 +945,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 8, "fillGradient": 0, "gridPos": { @@ -1003,7 +1036,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 8, "fillGradient": 0, "gridPos": { @@ -1093,7 +1129,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 8, "fillGradient": 0, "gridPos": { @@ -1181,7 +1220,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 8, "fillGradient": 0, "gridPos": { @@ -1271,7 +1313,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 8, "fillGradient": 0, "gridPos": { @@ -1355,7 +1400,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 8, "fillGradient": 0, "gridPos": { @@ -1442,7 +1490,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 8, "fillGradient": 0, "gridPos": { @@ -1527,7 +1578,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 8, "fillGradient": 0, "gridPos": { @@ -1609,6 +1663,26 @@ "tags": [], "templating": { "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, { "allValue": null, "current": { @@ -1617,7 +1691,10 @@ "$__all" ] }, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "definition": "label_values(libvirt_up{job=\"libvirt_exporter\"}, instance)", "hide": 0, "includeAll": true, @@ -1644,7 +1721,10 @@ "$__all" ] }, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "definition": "label_values(libvirt_domain_info_virtual_cpus{job=\"libvirt_exporter\"}, domain)", "hide": 0, "includeAll": true, @@ -1671,7 +1751,10 @@ "$__all" ] }, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "definition": "label_values(libvirt_domain_interface_stats_receive_bytes_total{job=\"libvirt_exporter\"}, source_bridge)", "hide": 0, "includeAll": true, diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/memcached.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/memcached.json index afae3d8f7..f701fdd13 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/openstack/memcached.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/memcached.json @@ -26,7 +26,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -125,7 +128,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -223,7 +229,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -319,7 +328,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -419,7 +431,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -527,7 +542,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -636,7 +654,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -735,7 +756,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -836,6 +860,26 @@ ], "templating": { "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, { "allValue": null, "current": { @@ -843,7 +887,10 @@ "text": "All", "value": "$__all" }, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "definition": "", "hide": 0, "includeAll": true, diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_full.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_full.json index e55cc1ce6..08078c31d 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_full.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_full.json @@ -35,7 +35,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -51,7 +51,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "Busy state of all CPU cores together", "fieldConfig": { @@ -118,7 +118,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "exemplar": true, "expr": "(((count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))) - avg(sum by (mode)(irate(node_cpu_seconds_total{mode='idle',instance=\"$node\",job=\"$job\"}[5m])))) * 100) / count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))", @@ -136,7 +136,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "Busy state of all CPU cores together (5 min average)", "fieldConfig": { @@ -215,7 +215,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "Busy state of all CPU cores together (15 min average)", "fieldConfig": { @@ -293,7 +293,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "Non available RAM memory", "fieldConfig": { @@ -372,7 +372,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "Used Swap", "fieldConfig": { @@ -449,7 +449,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "Used Root FS", "fieldConfig": { @@ -527,7 +527,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "Total number of CPU cores", "fieldConfig": { @@ -603,7 +603,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "System uptime", "fieldConfig": { @@ -659,7 +659,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "exemplar": true, "expr": "node_time_seconds{instance=\"$node\",job=\"$job\"} - node_boot_time_seconds{instance=\"$node\",job=\"$job\"}", @@ -677,7 +677,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "Total RootFS", "fieldConfig": { @@ -758,7 +758,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "Total RAM", "fieldConfig": { @@ -833,7 +833,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "Total SWAP", "fieldConfig": { @@ -909,7 +909,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -944,7 +944,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 2, "description": "Basic CPU info", @@ -1018,7 +1018,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, @@ -1035,7 +1035,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, @@ -1113,7 +1113,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 2, "description": "Basic memory usage", @@ -1286,7 +1286,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "Basic network info per interface", "fill": 4, @@ -1391,7 +1391,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 3, "description": "Disk space used of all filesystems mounted", @@ -1482,7 +1482,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -1513,7 +1513,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 2, "description": "", @@ -1566,7 +1566,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "editorMode": "code", "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode=\"system\",instance=\"$node\",job=\"$job\"}[5m])) * 100", @@ -1581,7 +1581,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode='user',instance=\"$node\",job=\"$job\"}[5m])) * 100", "format": "time_series", @@ -1593,7 +1593,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode='nice',instance=\"$node\",job=\"$job\"}[5m])) * 100", "format": "time_series", @@ -1605,7 +1605,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode='iowait',instance=\"$node\",job=\"$job\"}[5m])) * 100", "format": "time_series", @@ -1617,7 +1617,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode='irq',instance=\"$node\",job=\"$job\"}[5m])) * 100", "format": "time_series", @@ -1629,7 +1629,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode='softirq',instance=\"$node\",job=\"$job\"}[5m])) * 100", "format": "time_series", @@ -1641,7 +1641,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode='steal',instance=\"$node\",job=\"$job\"}[5m])) * 100", "format": "time_series", @@ -1653,7 +1653,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode='guest',instance=\"$node\",job=\"$job\"}[5m])) * 100", "format": "time_series", @@ -1665,7 +1665,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode='idle',instance=\"$node\",job=\"$job\"}[5m])) * 100", "format": "time_series", @@ -1737,7 +1737,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 2, "description": "", @@ -1919,7 +1919,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -2025,7 +2025,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 3, "description": "", @@ -2125,7 +2125,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -2318,7 +2318,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 3, "description": "", @@ -2452,7 +2452,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 3, "description": "", @@ -2548,7 +2548,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -2586,7 +2586,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 2, "fieldConfig": { @@ -2709,7 +2709,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 2, "fieldConfig": { @@ -2841,7 +2841,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 2, "fieldConfig": { @@ -2986,7 +2986,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 2, "fieldConfig": { @@ -3116,7 +3116,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 2, "fieldConfig": { @@ -3268,7 +3268,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 2, "fieldConfig": { @@ -3391,7 +3391,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 2, "fieldConfig": { @@ -3524,7 +3524,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 2, "fieldConfig": { @@ -3640,7 +3640,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 2, "fieldConfig": { @@ -3767,7 +3767,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 2, "fieldConfig": { @@ -3892,7 +3892,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 2, "fieldConfig": { @@ -4024,7 +4024,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 2, "fieldConfig": { @@ -4148,7 +4148,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 2, "fieldConfig": { @@ -4282,7 +4282,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 2, "fieldConfig": { @@ -4407,7 +4407,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 2, "fieldConfig": { @@ -4500,7 +4500,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -4520,7 +4520,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -4627,7 +4627,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -4752,7 +4752,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 2, "fieldConfig": { @@ -4891,7 +4891,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 2, "fieldConfig": { @@ -4985,7 +4985,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -5005,7 +5005,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -5126,7 +5126,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -5220,7 +5220,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -5328,7 +5328,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -5428,7 +5428,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -5445,7 +5445,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -5547,7 +5547,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -5642,7 +5642,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -5737,7 +5737,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -5863,7 +5863,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -5973,7 +5973,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -6081,7 +6081,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -6192,7 +6192,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -6212,7 +6212,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -6315,7 +6315,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -6425,7 +6425,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -6532,7 +6532,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -6628,7 +6628,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -6722,7 +6722,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -6816,7 +6816,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -6920,7 +6920,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -6940,7 +6940,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -7084,7 +7084,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -7193,7 +7193,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -7284,7 +7284,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -7304,7 +7304,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -7400,7 +7400,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -7547,7 +7547,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -7567,7 +7567,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -7754,7 +7754,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -7943,7 +7943,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -8134,7 +8134,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -8313,7 +8313,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -8502,7 +8502,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -8689,7 +8689,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -8868,7 +8868,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -9050,7 +9050,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -9070,7 +9070,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "decimals": 3, "description": "", @@ -9187,7 +9187,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -9284,7 +9284,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -9387,7 +9387,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -9486,7 +9486,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -9591,7 +9591,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -9616,7 +9616,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -9725,7 +9725,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -9836,7 +9836,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -9947,7 +9947,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -10058,7 +10058,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -10161,7 +10161,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -10272,7 +10272,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -10376,7 +10376,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -10474,7 +10474,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -10521,7 +10521,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "editorMode": "code", "expr": "irate(node_network_transmit_colls_total{instance=\"$node\",job=\"$job\"}[5m])", @@ -10572,7 +10572,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -10627,7 +10627,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "expr": "node_nf_conntrack_entries{instance=\"$node\",job=\"$job\"}", "format": "time_series", @@ -10639,7 +10639,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "expr": "node_nf_conntrack_entries_limit{instance=\"$node\",job=\"$job\"}", "format": "time_series", @@ -10686,7 +10686,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -10738,7 +10738,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "editorMode": "code", "expr": "node_arp_entries{instance=\"$node\",job=\"$job\"}", @@ -10756,7 +10756,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -10806,7 +10806,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "editorMode": "code", "exemplar": false, @@ -10827,7 +10827,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -10897,7 +10897,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "editorMode": "code", "exemplar": false, @@ -10917,7 +10917,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -10968,7 +10968,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "editorMode": "code", "expr": "node_network_transmit_queue_length{instance=\"$node\",job=\"$job\"}", @@ -10990,7 +10990,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -11105,7 +11105,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -11204,7 +11204,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -11306,7 +11306,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -11326,7 +11326,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -11461,7 +11461,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -11577,7 +11577,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -11675,7 +11675,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -11782,7 +11782,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -11895,7 +11895,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -11915,7 +11915,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -12029,7 +12029,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -12084,7 +12084,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "exemplar": true, "expr": "irate(node_netstat_Ip_Forwarding{instance=\"$node\",job=\"$job\"}[5m])", @@ -12135,7 +12135,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -12248,7 +12248,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -12352,7 +12352,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -12469,7 +12469,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -12601,7 +12601,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -12719,7 +12719,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -12853,7 +12853,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -12969,7 +12969,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -13096,7 +13096,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -13201,7 +13201,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -13221,7 +13221,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -13318,7 +13318,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -13437,11 +13437,12 @@ "text": "Prometheus", "value": "Prometheus" }, + "description": "The prometheus datasource used for queries.", "hide": 0, "includeAll": false, "label": "datasource", "multi": false, - "name": "Prometheus", + "name": "datasource", "options": [], "query": "prometheus", "queryValue": "", @@ -13457,8 +13458,8 @@ "value": "node" }, "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "type": "prometheus", + "uid": "${datasource}" }, "definition": "label_values(node_cpu_seconds_total, job)", "hide": 0, @@ -13484,8 +13485,8 @@ "value": "kef1p-phycon0001" }, "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "type": "prometheus", + "uid": "${datasource}" }, "definition": "label_values(node_cpu_seconds_total{job=\"$job\"}, instance)", "hide": 0, diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_side_by_side.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_side_by_side.json index 00da9e539..8547db825 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_side_by_side.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_side_by_side.json @@ -115,7 +115,10 @@ "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "format": "none", @@ -212,7 +215,10 @@ "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "format": "none", @@ -311,7 +317,10 @@ "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "format": "none", @@ -406,7 +415,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "decimals": 3, "editable": true, "error": false, @@ -587,7 +599,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "decimals": 3, "editable": true, "error": false, @@ -770,7 +785,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "decimals": 3, "editable": true, "error": false, @@ -956,7 +974,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "decimals": 2, "editable": true, "error": false, @@ -1178,7 +1199,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "decimals": 2, "editable": true, "error": false, @@ -1402,7 +1426,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "decimals": 2, "editable": true, "error": false, @@ -1623,7 +1650,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -1727,7 +1757,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -1833,7 +1866,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -1939,7 +1975,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "decimals": 3, "editable": true, "error": false, @@ -2044,7 +2083,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "decimals": 3, "editable": true, "error": false, @@ -2151,7 +2193,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "decimals": 3, "editable": true, "error": false, @@ -2258,7 +2303,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -2363,7 +2411,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -2470,7 +2521,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -2577,7 +2631,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -2697,7 +2754,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -2819,7 +2879,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -2941,7 +3004,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -3060,7 +3126,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -3181,7 +3250,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -3302,7 +3374,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -3407,7 +3482,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -3514,7 +3592,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -3621,7 +3702,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -3725,7 +3809,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -3831,7 +3918,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -3937,7 +4027,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -4061,7 +4154,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -4187,7 +4283,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -4313,7 +4412,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -4433,7 +4535,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -4555,7 +4660,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -4677,7 +4785,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -4779,7 +4890,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -4883,7 +4997,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -4987,7 +5104,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -5120,7 +5240,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -5255,7 +5378,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -5390,7 +5516,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -5492,7 +5621,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -5596,7 +5728,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fill": 1, @@ -5704,6 +5839,26 @@ ], "templating": { "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, { "allFormat": "glob", "allValue": null, @@ -5715,7 +5870,10 @@ "10.225.1.3:9100" ] }, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "definition": "", "hide": 0, "includeAll": false, diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/openstack.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/openstack.json index 6dc3cb865..28a1fcb3f 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/openstack/openstack.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/openstack.json @@ -44,7 +44,10 @@ "rgba(237, 129, 40, 0.89)", "#d44a3a" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -125,7 +128,10 @@ }, { "columns": [], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -300,7 +306,10 @@ "rgba(237, 129, 40, 0.89)", "#299c46" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -388,7 +397,10 @@ "rgba(237, 129, 40, 0.89)", "#d44a3a" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -469,7 +481,10 @@ }, { "columns": [], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -660,7 +675,10 @@ "rgba(237, 129, 40, 0.89)", "#299c46" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -748,7 +766,10 @@ "rgba(237, 129, 40, 0.89)", "#d44a3a" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -828,7 +849,10 @@ }, { "columns": [], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -1002,7 +1026,10 @@ "rgba(237, 129, 40, 0.89)", "#299c46" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -1099,7 +1126,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -1208,7 +1238,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -1316,7 +1349,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -1423,7 +1459,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -1542,7 +1581,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -1654,7 +1696,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -1761,7 +1806,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -1867,7 +1915,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -1961,7 +2012,10 @@ } }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "", "fieldConfig": { "defaults": { @@ -2489,6 +2543,26 @@ ], "templating": { "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, { "auto": true, "auto_count": 10, diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus.json index 1ff5724fd..862165643 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus.json @@ -12,7 +12,10 @@ "type": "dashboard" }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "enable": true, "expr": "sum(changes(prometheus_config_last_reload_success_timestamp_seconds{instance=~\"$instance\"}[10m])) by (instance)", "hide": false, @@ -24,7 +27,10 @@ "type": "alert" }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "enable": false, "expr": "count(sum(up{instance=\"$instance\"}) by (instance) < 1)", "hide": false, @@ -69,7 +75,10 @@ "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "decimals": 3, "description": "Percentage of uptime during the most recent $interval period. Change the period with the 'interval' dropdown above.", "format": "none", @@ -147,7 +156,10 @@ }, { "columns": [], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Servers which are DOWN RIGHT NOW! \nFIX THEM!!", "fontSize": "100%", "gridPos": { @@ -236,7 +248,10 @@ "rgba(237, 129, 40, 0.89)", "rgba(245, 54, 54, 0.9)" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Total number of time series in prometheus", "format": "none", "gauge": { @@ -319,7 +334,10 @@ "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "format": "none", "gauge": { "maxValue": 100, @@ -416,7 +434,10 @@ "rgba(237, 129, 40, 0.89)", "rgba(245, 54, 54, 0.9)" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "The total number of rule group evaluations missed due to slow rule group evaluation.", "format": "none", "gauge": { @@ -499,7 +520,10 @@ "rgba(237, 129, 40, 0.89)", "rgba(245, 54, 54, 0.9)" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "The total number of rule group evaluations skipped due to throttled metric storage.", "format": "none", "gauge": { @@ -582,7 +606,10 @@ "rgba(237, 129, 40, 0.89)", "rgba(245, 54, 54, 0.9)" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Total number of scrapes that hit the sample limit and were rejected.", "format": "none", "gauge": { @@ -665,7 +692,10 @@ "rgba(237, 129, 40, 0.89)", "rgba(245, 54, 54, 0.9)" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Number of times the database failed to reload block data from disk.", "format": "none", "gauge": { @@ -748,7 +778,10 @@ "rgba(237, 129, 40, 0.89)", "rgba(245, 54, 54, 0.9)" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Sum of all skipped scrapes", "format": "none", "gauge": { @@ -842,7 +875,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "All non-zero failures and errors", "fill": 1, "fillGradient": 0, @@ -1101,7 +1137,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -1194,7 +1233,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -1300,7 +1342,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -1391,7 +1436,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -1512,7 +1560,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Rate of total number of appended samples", "fill": 1, "fillGradient": 0, @@ -1619,7 +1670,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Total number of syncs that were executed on a scrape pool.", "fill": 1, "fillGradient": 0, @@ -1712,7 +1766,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Actual interval to sync the scrape pool.", "fill": 1, "fillGradient": 0, @@ -1819,7 +1876,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -1910,7 +1970,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Total number of rejected scrapes", "fill": 1, "fillGradient": 0, @@ -2045,7 +2108,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "The duration of rule group evaluations", "fill": 1, "fillGradient": 0, @@ -2137,7 +2203,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -2228,7 +2297,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -2319,7 +2391,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Rule-group evaluations \n - total\n - missed due to slow rule group evaluation\n - skipped due to throttled metric storage", "fill": 1, "fillGradient": 0, @@ -2442,7 +2517,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -2548,7 +2626,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -2639,7 +2720,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -2746,7 +2830,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "GC invocation durations", "fill": 1, "fillGradient": 0, @@ -2853,7 +2940,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "This is probably wrong! Please help.", "fill": 1, "fillGradient": 0, @@ -3088,7 +3178,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -3179,7 +3272,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -3272,6 +3368,26 @@ "tags": [], "templating": { "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, { "allValue": null, "current": { @@ -3280,7 +3396,10 @@ "$__all" ] }, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "definition": "", "hide": 0, "includeAll": true, @@ -3307,7 +3426,10 @@ "$__all" ] }, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "definition": "", "hide": 0, "includeAll": true, diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus_alertmanager.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus_alertmanager.json index 510af9a72..8db3f73a3 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus_alertmanager.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus_alertmanager.json @@ -11182,7 +11182,7 @@ }, "hide": 0, "includeAll": false, - "label": "Prometheus datasource", + "label": "datasource", "multi": false, "name": "datasource", "options": [], diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus_benchmark.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus_benchmark.json index 836a429b0..b2eb04de1 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus_benchmark.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus_benchmark.json @@ -23,7 +23,10 @@ "panels": [ { "collapsed": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "gridPos": { "h": 1, "w": 24, @@ -40,7 +43,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -138,7 +144,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -236,7 +245,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -331,7 +343,10 @@ }, { "collapsed": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "gridPos": { "h": 1, "w": 24, @@ -354,7 +369,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -462,7 +480,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -567,7 +588,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -679,7 +703,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -792,7 +819,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -914,7 +944,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -1015,7 +1048,10 @@ }, { "collapsed": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "gridPos": { "h": 1, "w": 24, @@ -1040,7 +1076,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -1172,7 +1211,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -1283,7 +1325,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -1398,7 +1443,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -1508,7 +1556,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -1631,7 +1682,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -1801,7 +1855,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -1910,7 +1967,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -2020,7 +2080,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -2137,7 +2200,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -2246,7 +2312,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -2355,7 +2424,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -2456,7 +2528,10 @@ }, { "collapsed": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "gridPos": { "h": 1, "w": 24, @@ -2484,7 +2559,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "decimals": null, "editable": true, "error": false, @@ -2623,7 +2701,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -2728,7 +2809,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "decimals": 2, "editable": true, "error": false, @@ -2844,7 +2928,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "decimals": 2, "editable": true, "error": false, @@ -2953,7 +3040,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "decimals": 2, "editable": true, "error": false, @@ -3065,7 +3155,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -3167,7 +3260,10 @@ }, { "collapsed": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "gridPos": { "h": 1, "w": 24, @@ -3184,7 +3280,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -3283,7 +3382,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -3384,7 +3486,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -3482,7 +3587,10 @@ }, { "collapsed": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "gridPos": { "h": 1, "w": 24, @@ -3499,7 +3607,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -3600,7 +3711,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -3725,7 +3839,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -3821,7 +3938,10 @@ }, { "collapsed": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "gridPos": { "h": 1, "w": 24, @@ -3843,7 +3963,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Time spent in each mode, per second", "editable": true, "error": false, @@ -3955,7 +4078,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -4076,7 +4202,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -4179,7 +4308,10 @@ }, { "collapsed": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "gridPos": { "h": 1, "w": 24, @@ -4201,7 +4333,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -4312,7 +4447,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -4423,7 +4561,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -4534,7 +4675,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -4643,7 +4787,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -4754,7 +4901,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -4865,7 +5015,10 @@ }, { "collapsed": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "gridPos": { "h": 1, "w": 24, @@ -4887,7 +5040,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "", "editable": true, "error": false, @@ -4999,7 +5155,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "", "editable": true, "error": false, @@ -5111,7 +5270,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "", "editable": true, "error": false, @@ -5225,7 +5387,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -5340,7 +5505,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "editable": true, "error": false, "fieldConfig": { @@ -5450,6 +5618,26 @@ "tags": [], "templating": { "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, { "allValue": null, "current": { @@ -5457,7 +5645,10 @@ "text": "10.103.1.13", "value": "10.103.1.13" }, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "definition": "", "description": null, "error": null, @@ -5488,7 +5679,10 @@ "text": "All", "value": "$__all" }, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "definition": "", "description": null, "error": null, diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/rabbitmq.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/rabbitmq.json index 7d8779a88..71966f624 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/openstack/rabbitmq.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/rabbitmq.json @@ -45,7 +45,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -107,7 +107,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(rabbitmq_queue_messages_ready * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", @@ -126,7 +126,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -194,7 +194,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(irate(rabbitmq_channel_messages_published_total[5m]) * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", @@ -212,7 +212,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -292,7 +292,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -373,7 +373,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -453,7 +453,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -534,7 +534,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -602,7 +602,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(rate(rabbitmq_channel_messages_redelivered_total[5m]) * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_channel_messages_delivered_total[5m]) * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_channel_messages_delivered_ack_total[5m]) * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_channel_get_total[5m]) * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_channel_get_ack_total[5m]) * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", @@ -621,7 +621,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -701,7 +701,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -781,7 +781,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -862,7 +862,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -879,7 +879,7 @@ "columns": [], "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "fontSize": "100%", "gridPos": { @@ -1089,7 +1089,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "If the value is zero or less, the memory alarm will be triggered and all publishing connections across all cluster nodes will be blocked.\n\nThis value can temporarily go negative because the memory alarm is triggered with a slight delay.\n\nThe kernel's view of the amount of memory used by the node can differ from what the node itself can observe. This means that this value can be negative for a sustained period of time.\n\nBy default nodes use resident set size (RSS) to compute how much memory they use. This strategy can be changed (see the guides below).\n\n* [Alarms](https://www.rabbitmq.com/alarms.html)\n* [Memory Alarms](https://www.rabbitmq.com/memory.html)\n* [Reasoning About Memory Use](https://www.rabbitmq.com/memory-use.html)\n* [Blocked Connection Notifications](https://www.rabbitmq.com/connection-blocked.html)", "fill": 0, @@ -1234,7 +1234,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "This metric is reported for the partition where the RabbitMQ data directory is stored.\n\nIf the value is zero or less, the disk alarm will be triggered and all publishing connections across all cluster nodes will be blocked.\n\nThis value can temporarily go negative because the free disk space alarm is triggered with a slight delay.\n\n* [Alarms](https://www.rabbitmq.com/alarms.html)\n* [Disk Space Alarms](https://www.rabbitmq.com/disk-alarms.html)\n* [Disk Space](https://www.rabbitmq.com/production-checklist.html#resource-limits-disk-space)\n* [Persistence Configuration](https://www.rabbitmq.com/persistence-conf.html)\n* [Blocked Connection Notifications](https://www.rabbitmq.com/connection-blocked.html)", "fill": 0, @@ -1379,7 +1379,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "When this value reaches zero, new connections will not be accepted and disk write operations may fail.\n\nClient libraries, peer nodes and CLI tools will not be able to connect when the node runs out of available file descriptors.\n\n* [Open File Handles Limit](https://www.rabbitmq.com/production-checklist.html#resource-limits-file-handle-limit)", "fill": 0, @@ -1526,7 +1526,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "When this value reaches zero, new connections will not be accepted.\n\nClient libraries, peer nodes and CLI tools will not be able to connect when the node runs out of available file descriptors.\n\n* [Networking and RabbitMQ](https://www.rabbitmq.com/networking.html)", "fill": 0, @@ -1668,7 +1668,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -1688,7 +1688,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "Total number of ready messages ready to be delivered to consumers.\n\nAim to keep this value as low as possible. RabbitMQ behaves best when messages are flowing through it. It's OK for publishers to occasionally outpace consumers, but the expectation is that consumers will eventually process all ready messages.\n\nIf this metric keeps increasing, your system will eventually run out of memory and/or disk space. Consider using TTL or Queue Length Limit to prevent unbounded message growth.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Consumers](https://www.rabbitmq.com/consumers.html)\n* [Queue Length Limit](https://www.rabbitmq.com/maxlength.html)\n* [Time-To-Live and Expiration](https://www.rabbitmq.com/ttl.html)", "fill": 10, @@ -1819,7 +1819,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "The total number of messages that are either in-flight to consumers, currently being processed by consumers or simply waiting for the consumer acknowledgements to be processed by the queue. Until the queue processes the message acknowledgement, the message will remain unacknowledged.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Confirms and Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [Consumer Prefetch](https://www.rabbitmq.com/consumer-prefetch.html)", "fill": 10, @@ -1947,7 +1947,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -1967,7 +1967,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "The incoming message rate before any routing rules are applied.\n\nIf this value is lower than the number of messages published to queues, it may indicate that some messages are delivered to more than one queue.\n\nIf this value is higher than the number of messages published to queues, messages cannot be routed and will either be dropped or returned to publishers.\n\n* [Publishers](https://www.rabbitmq.com/publishers.html)", "fill": 10, @@ -2099,7 +2099,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "The rate of messages confirmed by the broker to publishers. Publishers must opt-in to receive message confirmations.\n\nIf this metric is consistently at zero it may suggest that publisher confirms are not used by clients. The safety of published messages is likely to be at risk.\n\n* [Publisher Confirms](https://www.rabbitmq.com/confirms.html#publisher-confirms)\n* [Publisher Confirms and Data Safety](https://www.rabbitmq.com/publishers.html#data-safety)\n* [When Will Published Messages Be Confirmed by the Broker?](https://www.rabbitmq.com/confirms.html#when-publishes-are-confirmed)", "fill": 10, @@ -2231,7 +2231,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "The rate of messages received from publishers and successfully routed to the master queue replicas.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Publishers](https://www.rabbitmq.com/publishers.html)", "fill": 10, @@ -2363,7 +2363,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "The rate of messages received from publishers that have publisher confirms enabled and the broker has not confirmed yet.\n\n* [Publishers](https://www.rabbitmq.com/publishers.html)\n* [Confirms and Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [When Will Published Messages Be Confirmed by the Broker?](https://www.rabbitmq.com/confirms.html#when-publishes-are-confirmed)", "fill": 10, @@ -2495,7 +2495,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "The rate of messages that cannot be routed and are dropped. \n\nAny value above zero means message loss and likely suggests a routing problem on the publisher end.\n\n* [Unroutable Message Handling](https://www.rabbitmq.com/publishers.html#unroutable)", "fill": 10, @@ -2599,7 +2599,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "The rate of messages that cannot be routed and are returned back to publishers.\n\nSustained values above zero may indicate a routing problem on the publisher end.\n\n* [Unroutable Message Handling](https://www.rabbitmq.com/publishers.html#unroutable)\n* [When Will Published Messages Be Confirmed by the Broker?](https://www.rabbitmq.com/confirms.html#when-publishes-are-confirmed)", "fill": 10, @@ -2700,7 +2700,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -2720,7 +2720,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "The rate of messages delivered to consumers. It includes messages that have been redelivered.\n\nThis metric does not include messages that have been fetched by consumers using `basic.get` (consumed by polling).\n\n* [Consumers](https://www.rabbitmq.com/consumers.html)", "fill": 10, @@ -2851,7 +2851,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "The rate of messages that have been redelivered to consumers. It includes messages that have been requeued automatically and redelivered due to channel exceptions or connection closures.\n\nHaving some redeliveries is expected, but if this metric is consistently non-zero, it is worth investigating why.\n\n* [Negative Acknowledgement and Requeuing of Deliveries](https://www.rabbitmq.com/confirms.html#consumer-nacks-requeue)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", "fill": 10, @@ -2999,7 +2999,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "The rate of message deliveries to consumers that use manual acknowledgement mode.\n\nWhen this mode is used, RabbitMQ waits for consumers to acknowledge messages before more messages can be delivered.\n\nThis is the safest way of consuming messages.\n\n* [Consumer Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [Consumer Prefetch](https://www.rabbitmq.com/consumer-prefetch.html)\n* [Consumer Acknowledgement Modes, Prefetch and Throughput](https://www.rabbitmq.com/confirms.html#channel-qos-prefetch-throughput)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", "fill": 10, @@ -3131,7 +3131,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "The rate of message deliveries to consumers that use automatic acknowledgement mode.\n\nWhen this mode is used, RabbitMQ does not wait for consumers to acknowledge message deliveries.\n\nThis mode is fire-and-forget and does not offer any delivery safety guarantees. It tends to provide higher throughput and it may lead to consumer overload and higher consumer memory usage.\n\n* [Consumer Acknowledgement Modes, Prefetch and Throughput](https://www.rabbitmq.com/confirms.html#channel-qos-prefetch-throughput)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", "fill": 10, @@ -3263,7 +3263,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "The rate of message acknowledgements coming from consumers that use manual acknowledgement mode.\n\n* [Consumer Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [Consumer Prefetch](https://www.rabbitmq.com/consumer-prefetch.html)\n* [Consumer Acknowledgement Modes, Prefetch and Throughput](https://www.rabbitmq.com/confirms.html#channel-qos-prefetch-throughput)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", "fill": 10, @@ -3395,7 +3395,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "The rate of messages delivered to polling consumers that use automatic acknowledgement mode.\n\nThe use of polling consumers is highly inefficient and therefore strongly discouraged.\n\n* [Fetching individual messages](https://www.rabbitmq.com/consumers.html#fetching)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", "fill": 10, @@ -3499,7 +3499,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "The rate of polling consumer operations that yield no result.\n\nAny value above zero means that RabbitMQ resources are wasted by polling consumers.\n\nCompare this metric to the other polling consumer metrics to see the inefficiency rate.\n\nThe use of polling consumers is highly inefficient and therefore strongly discouraged.\n\n* [Fetching individual messages](https://www.rabbitmq.com/consumers.html#fetching)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", "fill": 10, @@ -3604,7 +3604,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "The rate of messages delivered to polling consumers that use manual acknowledgement mode.\n\nThe use of polling consumers is highly inefficient and therefore strongly discouraged.\n\n* [Fetching individual messages](https://www.rabbitmq.com/consumers.html#fetching)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", "fill": 10, @@ -3705,7 +3705,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -3725,7 +3725,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "Total number of queue masters per node. \n\nThis metric makes it easy to see sub-optimal queue distribution in a cluster.\n\n* [Queue Masters, Data Locality](https://www.rabbitmq.com/ha.html#master-migration-data-locality)\n* [Queues](https://www.rabbitmq.com/queues.html)", "fill": 10, @@ -3857,7 +3857,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "The rate of queue declarations performed by clients.\n\nLow sustained values above zero are to be expected. High rates may be indicative of queue churn or high rates of connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [Queues](https://www.rabbitmq.com/queues.html)", "fill": 10, @@ -4005,7 +4005,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "The rate of new queues created (as opposed to redeclarations).\n\nLow sustained values above zero are to be expected. High rates may be indicative of queue churn or high rates of connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [Queues](https://www.rabbitmq.com/queues.html)", "fill": 10, @@ -4153,7 +4153,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "The rate of queues deleted.\n\nLow sustained values above zero are to be expected. High rates may be indicative of queue churn or high rates of connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [Queues](https://www.rabbitmq.com/queues.html)", "fill": 10, @@ -4298,7 +4298,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -4318,7 +4318,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "Total number of channels on all currently opened connections.\n\nIf this metric grows monotonically it is highly likely a channel leak in one of the applications. Confirm channel leaks by using the _Channels opened_ and _Channels closed_ metrics.\n\n* [Channel Leak](https://www.rabbitmq.com/channels.html#channel-leaks)\n* [Channels](https://www.rabbitmq.com/channels.html)", "fill": 10, @@ -4449,7 +4449,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "The rate of new channels opened by applications across all connections. Channels are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of channel churn or mass connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [High Channel Churn](https://www.rabbitmq.com/channels.html#high-channel-churn)\n* [Channels](https://www.rabbitmq.com/channels.html)", "fill": 10, @@ -4597,7 +4597,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "The rate of channels closed by applications across all connections. Channels are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of channel churn or mass connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [High Channel Churn](https://www.rabbitmq.com/channels.html#high-channel-churn)\n* [Channels](https://www.rabbitmq.com/channels.html)", "fill": 10, @@ -4742,7 +4742,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "gridPos": { "h": 1, @@ -4762,7 +4762,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "Total number of client connections.\n\nIf this metric grows monotonically it is highly likely a connection leak in one of the applications. Confirm connection leaks by using the _Connections opened_ and _Connections closed_ metrics.\n\n* [Connection Leak](https://www.rabbitmq.com/connections.html#monitoring)\n* [Connections](https://www.rabbitmq.com/connections.html)", "fill": 10, @@ -4893,7 +4893,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "The rate of new connections opened by clients. Connections are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of connection churn or mass connection recovery.\n\n* [Connection Leak](https://www.rabbitmq.com/connections.html#monitoring)\n* [Connections](https://www.rabbitmq.com/connections.html)", "fill": 10, @@ -5041,7 +5041,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${datasource}" }, "description": "The rate of connections closed. Connections are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of connection churn or mass connection recovery.\n\n* [Connections](https://www.rabbitmq.com/connections.html)", "fill": 10, @@ -5191,6 +5191,26 @@ ], "templating": { "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, { "current": { "isNone": true, @@ -5199,8 +5219,8 @@ "value": "" }, "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "type": "prometheus", + "uid": "${datasource}" }, "definition": "label_values(rabbitmq_identity_info, namespace)", "hide": 0, @@ -5229,8 +5249,8 @@ "value": "rabbit@kef1p-phycon0001" }, "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "type": "prometheus", + "uid": "${datasource}" }, "definition": "label_values(rabbitmq_identity_info, rabbitmq_node)", "hide": 0, diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/redfish.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/redfish.json index 9cd65e141..b3f68ac97 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/openstack/redfish.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/redfish.json @@ -21,7 +21,10 @@ "links": [], "panels": [ { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": { @@ -119,7 +122,10 @@ "type": "stat" }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": { @@ -208,7 +214,10 @@ "type": "stat" }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": { @@ -297,7 +306,10 @@ "type": "stat" }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": { @@ -386,7 +398,10 @@ "type": "stat" }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": { @@ -493,7 +508,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -590,7 +608,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -687,7 +708,10 @@ }, { "columns": [], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -907,7 +931,10 @@ }, { "columns": [], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -1017,7 +1044,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -1119,7 +1149,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "", "fieldConfig": { "defaults": { @@ -1224,7 +1257,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -1328,7 +1364,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": { @@ -1451,7 +1490,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": { @@ -1582,7 +1624,10 @@ "mode": "spectrum" }, "dataFormat": "timeseries", - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -1656,7 +1701,10 @@ "mode": "spectrum" }, "dataFormat": "timeseries", - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "", "fieldConfig": { "defaults": { @@ -1731,7 +1779,10 @@ "mode": "spectrum" }, "dataFormat": "timeseries", - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "", "fieldConfig": { "defaults": { @@ -1806,7 +1857,10 @@ "mode": "spectrum" }, "dataFormat": "timeseries", - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "", "fieldConfig": { "defaults": { @@ -1873,7 +1927,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": { @@ -1990,7 +2047,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "", "fieldConfig": { "defaults": { @@ -2089,7 +2149,10 @@ }, { "columns": [], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -2228,7 +2291,10 @@ }, { "collapsed": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "gridPos": { "h": 1, "w": 24, @@ -2249,7 +2315,10 @@ "type": "row" }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -2342,7 +2411,10 @@ "type": "gauge" }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -2431,7 +2503,10 @@ "type": "gauge" }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -2520,7 +2595,10 @@ "type": "gauge" }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -2609,7 +2687,10 @@ "type": "gauge" }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -2698,7 +2779,10 @@ "type": "gauge" }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -2800,7 +2884,10 @@ "#299c46", "#299c46" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -2907,7 +2994,10 @@ "#299c46", "#d44a3a" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -3019,7 +3109,10 @@ "#299c46", "#299c46" ], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -3126,7 +3219,10 @@ "cacheTimeout": null, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "decimals": 1, "fieldConfig": { "defaults": { @@ -3251,7 +3347,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {}, @@ -3375,7 +3474,10 @@ { "cacheTimeout": null, "columns": [], - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -3601,7 +3703,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "decimals": 0, "fieldConfig": { "defaults": { @@ -3715,13 +3820,36 @@ "tags": [], "templating": { "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, { "allValue": null, "current": { "text": "cpu-e-1041", "value": "cpu-e-1041" }, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "definition": "label_values(redfish_exporter_collector_duration_seconds, server)", "hide": 0, "includeAll": false, From 7cd8ea0b10843d1a4188187c8592ff2f60b90c55 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 23 Nov 2022 08:42:03 +0000 Subject: [PATCH 112/292] Upgrade Pulp container to 3.21 --- etc/kayobe/seed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index ce16140fa..78cb92e79 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -106,7 +106,7 @@ seed_pulp_container: image: pulp/pulp pre: "{{ kayobe_config_path }}/containers/pulp/pre.yml" post: "{{ kayobe_config_path }}/containers/pulp/post.yml" - tag: "3.16" + tag: "3.21" network_mode: host volumes: - /opt/kayobe/containers/pulp:/etc/pulp From 6e4d042e3c97bd40c18836ef72e6f1524098de7a Mon Sep 17 00:00:00 2001 From: Scott Davidson Date: Wed, 23 Nov 2022 12:08:53 +0000 Subject: [PATCH 113/292] Change default setting --- etc/kayobe/cephadm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/cephadm.yml b/etc/kayobe/cephadm.yml index b82111cb7..371336310 100644 --- a/etc/kayobe/cephadm.yml +++ b/etc/kayobe/cephadm.yml @@ -91,7 +91,7 @@ kolla_ceph_services: kolla_ceph_cinder_volume_required: "{{ kolla_enable_cinder | bool }}" # Whether to generate Ceph configuration for Cinder backup. -kolla_ceph_cinder_backup_required: "{{ kolla_enable_cinder_backup | default(false) | bool }}" +kolla_ceph_cinder_backup_required: "{{ kolla_enable_cinder_backup | default(true) | bool }}" # Whether to generate Ceph configuration for Glance. kolla_ceph_glance_required: "{{ kolla_enable_glance | bool }}" From c5fa5645e726b2acf160e4feb2b08338e4146182 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Mon, 28 Nov 2022 09:30:27 +0000 Subject: [PATCH 114/292] Switch to using absolute limit for low memory alert (#247) * Switch to using absolute limit for low memory alert Nodes with alot of ram can end up triggering this low memory alert even when they have a significant amount of free memory e.g for a node with 512GiB ram, when we hit the current alert (which is con figured to be 80%) we still have 102.5GiB free. * Apply suggestions from code review * Rewrap comment --- etc/kayobe/kolla/config/prometheus/system.rules | 4 ++-- etc/kayobe/kolla/globals.yml | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/kolla/config/prometheus/system.rules b/etc/kayobe/kolla/config/prometheus/system.rules index 76d301fbf..a8fb59aaa 100644 --- a/etc/kayobe/kolla/config/prometheus/system.rules +++ b/etc/kayobe/kolla/config/prometheus/system.rules @@ -16,13 +16,13 @@ groups: description: "{{ $labels.device }} is {{ $value }}% full." - alert: LowMemory - expr: ( ( node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes - (node_memory_Hugepagesize_bytes * node_memory_HugePages_Total)) / (node_memory_MemTotal_bytes - (node_memory_Hugepagesize_bytes * node_memory_HugePages_Total))) * 100 >= 80 + expr: (node_memory_MemAvailable_bytes / 1024^3) < {{ alertmanager_low_memory_threshold_gib }} for: 1m labels: severity: alert annotations: summary: "Prometheus exporter at {{ $labels.instance }} reports low memory" - description: "Memory is {{ $value }}% full." + description: "Available memory is {{ $value }}." - alert: HostOomKillDetected expr: increase(node_vmstat_oom_kill[5m]) > 0 diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 514bf6aad..f52d530e0 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -7,6 +7,13 @@ docker_yum_gpgkey: "https://download.docker.com/linux/centos/gpg" bifrost_tag: xena-20221102T110017 {% endif %} +############################################################################# # Monitoring and alerting related settings + es_heap_size: 8g prometheus_cmdline_extras: "--storage.tsdb.retention.time=30d" +# Threshold to trigger a LowMemory alert in Gibibytes (GiB). When the amount +# of free memory is lower than this value an alert will be triggered. +alertmanager_low_memory_threshold_gib: 5 + +############################################################################# From 8a1c39bae66288f72d9740bbad30b84b34af210a Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 28 Nov 2022 10:29:26 +0000 Subject: [PATCH 115/292] Move README content into documentation --- README.rst | 443 +----------------- doc/requirements.txt | 2 +- doc/source/configuration/index.rst | 11 + doc/source/configuration/release-train.rst | 155 ++++++ doc/source/contributor/environments.rst | 11 + .../contributor/environments/ci-aio.rst | 113 +++++ .../contributor/environments/ci-builder.rst | 115 +++++ .../contributor/environments/ci-multinode.rst | 55 +++ doc/source/contributor/index.rst | 10 + doc/source/index.rst | 13 +- doc/source/usage.rst | 59 +++ .../environments/ci-multinode/README.md | 48 -- tox.ini | 5 +- 13 files changed, 542 insertions(+), 498 deletions(-) create mode 100644 doc/source/configuration/index.rst create mode 100644 doc/source/configuration/release-train.rst create mode 100644 doc/source/contributor/environments.rst create mode 100644 doc/source/contributor/environments/ci-aio.rst create mode 100644 doc/source/contributor/environments/ci-builder.rst create mode 100644 doc/source/contributor/environments/ci-multinode.rst create mode 100644 doc/source/contributor/index.rst create mode 100644 doc/source/usage.rst delete mode 100644 etc/kayobe/environments/ci-multinode/README.md diff --git a/README.rst b/README.rst index dfd27bfa6..c441e9eda 100644 --- a/README.rst +++ b/README.rst @@ -5,449 +5,14 @@ StackHPC Kayobe Configuration This repository provides a base Kayobe configuration for the Xena release of StackHPC OpenStack. -StackHPC release train -====================== - -StackHPC provides packages and container images for OpenStack via `Ark -`__. - -Deployments should use a local `Pulp `__ repository -server to synchronise content from Ark and serve it locally. Access to the -repositories on Ark is controlled via X.509 certificates issued by StackHPC. - -This configuration is a base, and should be merged with any existing Kayobe -configuration. It currently provides the following: - -* Configuration to deploy a local Pulp service as a container on the seed -* Pulp repository definitions for CentOS Stream 8 -* Playbooks to synchronise a local Pulp service with Ark -* Configuration to use the local Pulp repository mirrors on control plane hosts -* Configuration to use the local Pulp container registry on control plane hosts - -This configuration defines two `Pulp distributions -`__ for -packages, ``development`` and ``production``. This allows packages to be -updated and tested in a development or staging environment before rolling them -out to production. - -How to consume this configuration -================================= - -This configuration is not a complete Kayobe configuration, rather it should be -treated as a base, in place of the `upstream kayobe-config -`__. Indeed, this repository is -based on the upstream kayobe-config, with some opinionated configuration -changes applied. - -Since this repository makes changes to the base configuration, it works best -when used with Kayobe's `multiple environments -`__ -feature. - -This configuration should be consumed using the `StackHPC Kayobe fork -`__, which includes -backported support for Ansible collections. - -New deployments ---------------- - -If starting a new deployment, clone this repository as the starting point for -your configuration. - -.. code-block:: console - - git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/xena - -Existing deployments --------------------- - -If migrating an existing deployment to StackHPC Kayobe configuration, you will -need to merge the changes in this repository into your repository. - -.. code-block:: console - - git remote add stackhpc https://github.com/stackhpc/stackhpc-kayobe-config - git fetch stackhpc - git merge stackhpc/stackhpc/xena - -Updating --------- - -This base configuration will be updated over time, to update repository -versions, container image tags, and other configuration. Deployments may -consume these updates by merging in the changes with their local configuration. +Documentation for this project is provided in the ``doc`` directory. To build +it, run: .. code-block:: console - git fetch stackhpc - git merge stackhpc/stackhpc/xena - -The intention is to avoid merge conflicts where possible, but there may be -cases where this is difficult. We are open to discussion on how best to -approach this on both sides. - -Configuration -============= - -Local Pulp server ------------------ - -The URL and credentials of the local Pulp server are configured in -``etc/kayobe/pulp.yml`` via ``pulp_url``, ``pulp_username`` and -``pulp_password``. In most cases, the default values should be sufficient. -An admin password must be generated and set as the value of a -``secrets_pulp_password`` variable, typically in an Ansible Vault encrypted -``etc/kayobe/secrets.yml`` file. This password will be automatically set on -Pulp startup. - -StackHPC Ark ------------- - -The container image registry credentials issued by StackHPC should be -configured in ``etc/kayobe/pulp.yml``, using Ansible Vault to encrypt the -password: - -.. code-block:: yaml - - stackhpc_release_pulp_username: - stackhpc_release_pulp_password: - -The client certificate and private key issued by StackHPC should be stored in -``etc/kayobe/ansible/certs/ark.stackhpc.com/client-cert.pem`` and -``etc/kayobe/ansible/certs/ark.stackhpc.com/client-key.pem``, respectively, -with the private key encrypted via Ansible Vault. - -The distribution name for the environment should be configured as either -``development`` or ``production`` via ``stackhpc_repo_distribution`` in -``etc/kayobe/stackhpc.yml``. - -Usage -===== - -The local Pulp service will be deployed as a `Seed custom container -`__ -on next ``kayobe seed service deploy`` or ``kayobe seed service upgrade``. - -The following custom playbooks are provided in ``etc/kayobe/ansible/``: - -See the Kayobe `custom playbook documentation -`__ -for information on how to run them. - -* ``pulp-repo-sync.yml``: Pull packages from Ark to the local Pulp. This will - create a new repository version (snapshot) for each repository in the local - Pulp server when new packages are available. The new packages will not be - available to cloud nodes until they have been published. -* ``pulp-repo-publish.yml``: Publish synchronised packages to the - ``development`` distribution in the local Pulp. This will make synchronised - packages available to cloud nodes using the ``development`` distribution - (typically a development or staging environment). The new packages will not - be available to cloud nodes using the ``production`` distribution until they - have been promoted. -* ``pulp-repo-promote-production.yml``: Promote packages in the ``development`` - distribution to the ``production`` distribution in the local Pulp. This will - make all packages currently available to cloud nodes using the - ``development`` distribution also available to cloud nodes using the - ``production`` distribution. Typically this would be done only once the new - packages have been validated in a development or staging environment. -* ``pulp-container-sync.yml``: Pull container images from Ark to the local - Pulp. This will create a new repository version (snapshot) for each - repository in the local Pulp server when new image tags are available. The - new image tags will not be available to cloud nodes until they have been - published. -* ``pulp-container-publish.yml``: Publish synchronised container images in the - local Pulp. This will make synchonised container images available to cloud - nodes. - -Working with pulp -================= - -The `pulp CLI -`__ tool can be used to administer your local -pulp installation. Please follow the upstream documentation for installation -instructions. - -pulp CLI tricks ---------------- - -Saving credentials -~~~~~~~~~~~~~~~~~~ - -This is useful to avoid the need to always supply your credentials when running commands -from the command line: - -.. code-block:: console - - (venv-pulp) [stack@seed ~]$ pulp config create --username admin --base-url http://:8080 --password - - -Troubleshooting --------------- - -HTTP Error 400: Bad Request {"name":["This field must be unique."]} -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If you have previously tried to push an image to pulp e.g for local testing, you may -see this message when you later try to run ``pulp-container-sync.yml``: - -.. code-block:: console - - TASK [stackhpc.pulp.pulp_repository : Setup container repositories] ***************************** - failed: [localhost] (item=stackhpc/centos-source-prometheus-jiralert) => changed=false - ansible_loop_var: item - item: - name: stackhpc/centos-source-prometheus-jiralert - policy: on_demand - remote_password: password - remote_username: username - state: present - url: https://ark.stackhpc.com - msg: 'HTTP Error 400: Bad Request b''{"name":["This field must be unique."]}''' - -The issue is that pushing an image automatically creates a `container-push repository -`__ -which conflicts with the creation of a regular container repository of the same -name. You can resolve this conflict by deleting the distribution associated -with the push repository using the pulp CLI: - -.. code-block:: console - - (venv-pulp) [stack@seed ~]$ pulp --base-url http://:8080--username admin --password container distribution destroy --name stackhpc/centos-source-prometheus-jiralert - Started background task /pulp/api/v3/tasks/1f0a474a-b7c0-44b4-9ef4-ed633077f4d8/ - .Done. - -Environments -============ - -The following Kayobe environments are provided with this configuration: - -* ``ci-aio``: deploys an all-in-one converged control/compute cloud for testing -* ``ci-builder``: builds container images - -ci-aio ------- - -Prerequisites -^^^^^^^^^^^^^ - -* a CentOS Stream 8 or Ubuntu Focal 20.04 host -* access to the local Pulp server - -Setup -^^^^^ - -Access the host via SSH. - -Install package dependencies. - -On CentOS: - -.. code-block:: console - - sudo dnf install -y python3-virtualenv - -On Ubuntu: - -.. code-block:: console - - sudo apt update - sudo apt install -y python3-virtualenv - -Clone the Kayobe and Kayobe configuration repositories (this one): - -.. code-block:: console - - cd - mkdir -p src - pushd src - git clone https://github.com/stackhpc/kayobe.git -b stackhpc/xena - git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/xena kayobe-config - popd - -Create a virtual environment and install Kayobe: - -.. code-block:: console - - cd - mkdir -p venvs - pushd venvs - virtualenv kayobe - source kayobe/bin/activate - pip install -U pip - pip install ../src/kayobe - popd - -Add initial network configuration: - -.. code-block:: console - - sudo ip l add breth1 type bridge - sudo ip l set breth1 up - sudo ip a add 192.168.33.3/24 dev breth1 - sudo ip l add dummy1 type dummy - sudo ip l set dummy1 up - sudo ip l set dummy1 master breth1 - -Installation -^^^^^^^^^^^^ - -Acquire the Ansible Vault password for this repository, and store a copy at -``~/vault-pw``. - -The following commands install Kayobe and its dependencies, and prepare the -Ansible control host. - -.. code-block:: console - - export KAYOBE_VAULT_PASSWORD=$(cat ~/vault-pw) - pushd ~/venvs/kayobe - source bin/activate - popd - pushd ~/src/kayobe-config - source kayobe-env --environment ci-aio - kayobe control host bootstrap - -Deployment -^^^^^^^^^^ - -Next, configure the host OS & services. - -.. code-block:: console - - kayobe overcloud host configure - -Finally, deploy the overcloud services. - -.. code-block:: console - - kayobe overcloud service deploy - -The control plane should now be running. - -Testing -^^^^^^^ - -Run a smoke test: - -.. code-block:: console - - cd ~/kayobe - ./dev/overcloud-test-vm.sh - -ci-builder ----------- - -The ``ci-builder`` Kayobe environment is used to build Kolla container images. -Images are built using package repositories in the StackHPC development Pulp -service, and pushed there once built. - -Prerequisites -^^^^^^^^^^^^^ - -* a CentOS Stream 8 or Ubuntu Focal 20.04 host -* access to the local Pulp server - -Setup -^^^^^ - -Access the host via SSH. - -Install package dependencies. - -On CentOS: - -.. code-block:: console - - sudo dnf install -y python3-virtualenv - -On Ubuntu: - -.. code-block:: console - - sudo apt update - sudo apt install -y python3-virtualenv - -Clone the Kayobe and Kayobe configuration repositories (this one): - -.. code-block:: console - - cd - mkdir -p src - pushd src - git clone https://github.com/stackhpc/kayobe.git -b stackhpc/xena - git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/xena kayobe-config - popd - -Create a virtual environment and install Kayobe: - -.. code-block:: console - - cd - mkdir -p venvs - pushd venvs - virtualenv kayobe - source kayobe/bin/activate - pip install -U pip - pip install ../src/kayobe - popd - -Add initial network configuration: - -.. code-block:: console - - sudo ip l add breth1 type bridge - sudo ip l set breth1 up - sudo ip a add 192.168.33.3/24 dev breth1 - sudo ip l add dummy1 type dummy - sudo ip l set dummy1 up - sudo ip l set dummy1 master breth1 - -Installation -^^^^^^^^^^^^ - -Acquire the Ansible Vault password for this repository, and store a copy at -``~/vault-pw``. - -The following commands install Kayobe and its dependencies, and prepare the -Ansible control host. - -.. code-block:: console - - export KAYOBE_VAULT_PASSWORD=$(cat ~/vault-pw) - pushd ~/venvs/kayobe - source bin/activate - popd - pushd ~/src/kayobe-config - source kayobe-env --environment ci-builder - kayobe control host bootstrap - -Deployment -^^^^^^^^^^ - -Next, configure the host OS & services. - -.. code-block:: console - - kayobe seed host configure - -Building images -^^^^^^^^^^^^^^^ - -At this point you are ready to build and push some container images. - -.. code-block:: console - - kayobe seed container image build --push - kayobe overcloud container image build --push - -The container images are tagged as ``xena-``. This Kayobe -configuration includes a hook that writes the tag to ``~/kolla_tag``, since -it is not always simple to determine which tag was last applied to built -images. + tox -e docs -To use the new images, edit -``~/src/kayobe-config/etc/kayobe/kolla.yml`` to set the above -tag as the value of the ``kolla_openstack_release`` variable. +The HTML will be written to ``doc/build``. Resources ========= diff --git a/doc/requirements.txt b/doc/requirements.txt index 95b160c9c..842c709a9 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -3,5 +3,5 @@ # process, which may cause wedges in the gate later. reno>=3.1.0 # Apache-2.0 -sphinx>=2.0.0,!=2.1.0 # BSD +sphinx>=4.2.0 # BSD sphinxcontrib-svg2pdfconverter>=0.1.0 # BSD diff --git a/doc/source/configuration/index.rst b/doc/source/configuration/index.rst new file mode 100644 index 000000000..133045779 --- /dev/null +++ b/doc/source/configuration/index.rst @@ -0,0 +1,11 @@ +=================== +Configuration Guide +=================== + +This configuration guide describes how to use this configuration, and covers +the various features provided. + +.. toctree:: + :maxdepth: 1 + + release-train diff --git a/doc/source/configuration/release-train.rst b/doc/source/configuration/release-train.rst new file mode 100644 index 000000000..8a25f0653 --- /dev/null +++ b/doc/source/configuration/release-train.rst @@ -0,0 +1,155 @@ +====================== +StackHPC Release Train +====================== + +StackHPC provides packages and container images for OpenStack via `Ark +`__. + +Deployments should use a local `Pulp `__ repository +server to synchronise content from Ark and serve it locally. Access to the +repositories on Ark is controlled via X.509 certificates issued by StackHPC. + +This configuration is a base, and should be merged with any existing Kayobe +configuration. It currently provides the following: + +* Configuration to deploy a local Pulp service as a container on the seed +* Pulp repository definitions for CentOS Stream 8 +* Playbooks to synchronise a local Pulp service with Ark +* Configuration to use the local Pulp repository mirrors on control plane hosts +* Configuration to use the local Pulp container registry on control plane hosts + +This configuration defines two `Pulp distributions +`__ for +packages, ``development`` and ``production``. This allows packages to be +updated and tested in a development or staging environment before rolling them +out to production. + +Configuration +============= + +Local Pulp server +----------------- + +The URL and credentials of the local Pulp server are configured in +``etc/kayobe/pulp.yml`` via ``pulp_url``, ``pulp_username`` and +``pulp_password``. In most cases, the default values should be sufficient. +An admin password must be generated and set as the value of a +``secrets_pulp_password`` variable, typically in an Ansible Vault encrypted +``etc/kayobe/secrets.yml`` file. This password will be automatically set on +Pulp startup. + +StackHPC Ark +------------ + +The container image registry credentials issued by StackHPC should be +configured in ``etc/kayobe/pulp.yml``, using Ansible Vault to encrypt the +password: + +.. code-block:: yaml + + stackhpc_release_pulp_username: + stackhpc_release_pulp_password: + +The client certificate and private key issued by StackHPC should be stored in +``etc/kayobe/ansible/certs/ark.stackhpc.com/client-cert.pem`` and +``etc/kayobe/ansible/certs/ark.stackhpc.com/client-key.pem``, respectively, +with the private key encrypted via Ansible Vault. + +The distribution name for the environment should be configured as either +``development`` or ``production`` via ``stackhpc_repo_distribution`` in +``etc/kayobe/stackhpc.yml``. + +Usage +===== + +The local Pulp service will be deployed as a `Seed custom container +`__ +on next ``kayobe seed service deploy`` or ``kayobe seed service upgrade``. + +The following custom playbooks are provided in ``etc/kayobe/ansible/``: + +See the Kayobe `custom playbook documentation +`__ +for information on how to run them. + +* ``pulp-repo-sync.yml``: Pull packages from Ark to the local Pulp. This will + create a new repository version (snapshot) for each repository in the local + Pulp server when new packages are available. The new packages will not be + available to cloud nodes until they have been published. +* ``pulp-repo-publish.yml``: Publish synchronised packages to the + ``development`` distribution in the local Pulp. This will make synchronised + packages available to cloud nodes using the ``development`` distribution + (typically a development or staging environment). The new packages will not + be available to cloud nodes using the ``production`` distribution until they + have been promoted. +* ``pulp-repo-promote-production.yml``: Promote packages in the ``development`` + distribution to the ``production`` distribution in the local Pulp. This will + make all packages currently available to cloud nodes using the + ``development`` distribution also available to cloud nodes using the + ``production`` distribution. Typically this would be done only once the new + packages have been validated in a development or staging environment. +* ``pulp-container-sync.yml``: Pull container images from Ark to the local + Pulp. This will create a new repository version (snapshot) for each + repository in the local Pulp server when new image tags are available. The + new image tags will not be available to cloud nodes until they have been + published. +* ``pulp-container-publish.yml``: Publish synchronised container images in the + local Pulp. This will make synchonised container images available to cloud + nodes. + +Working with pulp +================= + +The `pulp CLI +`__ tool can be used to administer your local +pulp installation. Please follow the upstream documentation for installation +instructions. + +pulp CLI tricks +--------------- + +Saving credentials +~~~~~~~~~~~~~~~~~~ + +This is useful to avoid the need to always supply your credentials when running commands +from the command line: + +.. code-block:: console + + (venv-pulp) [stack@seed ~]$ pulp config create --username admin --base-url http://:8080 --password + + +Troubleshooting +--------------- + +HTTP Error 400: Bad Request {"name":["This field must be unique."]} +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you have previously tried to push an image to pulp e.g for local testing, you may +see this message when you later try to run ``pulp-container-sync.yml``: + +.. code-block:: console + + TASK [stackhpc.pulp.pulp_repository : Setup container repositories] ***************************** + failed: [localhost] (item=stackhpc/centos-source-prometheus-jiralert) => changed=false + ansible_loop_var: item + item: + name: stackhpc/centos-source-prometheus-jiralert + policy: on_demand + remote_password: password + remote_username: username + state: present + url: https://ark.stackhpc.com + msg: 'HTTP Error 400: Bad Request b''{"name":["This field must be unique."]}''' + +The issue is that pushing an image automatically creates a `container-push repository +`__ +which conflicts with the creation of a regular container repository of the same +name. You can resolve this conflict by deleting the distribution associated +with the push repository using the pulp CLI: + +.. code-block:: console + + (venv-pulp) [stack@seed ~]$ pulp --base-url http://:8080--username admin --password container distribution destroy --name stackhpc/centos-source-prometheus-jiralert + Started background task /pulp/api/v3/tasks/1f0a474a-b7c0-44b4-9ef4-ed633077f4d8/ + .Done. diff --git a/doc/source/contributor/environments.rst b/doc/source/contributor/environments.rst new file mode 100644 index 000000000..9d99d7a79 --- /dev/null +++ b/doc/source/contributor/environments.rst @@ -0,0 +1,11 @@ +Environments +============ + +The following Kayobe environments are provided with this configuration: + +.. toctree:: + :maxdepth: 1 + + environments/ci-aio + environments/ci-builder + environments/ci-multinode diff --git a/doc/source/contributor/environments/ci-aio.rst b/doc/source/contributor/environments/ci-aio.rst new file mode 100644 index 000000000..b34bad4d6 --- /dev/null +++ b/doc/source/contributor/environments/ci-aio.rst @@ -0,0 +1,113 @@ +====== +ci-aio +====== + +This environment deploys an all-in-one converged control/compute cloud for +testing. + +Prerequisites +============= + +* a CentOS Stream 8 or Ubuntu Focal 20.04 host +* access to the local Pulp server + +Setup +===== + +Access the host via SSH. + +Install package dependencies. + +On CentOS: + +.. code-block:: console + + sudo dnf install -y python3-virtualenv + +On Ubuntu: + +.. code-block:: console + + sudo apt update + sudo apt install -y python3-virtualenv + +Clone the Kayobe and Kayobe configuration repositories (this one): + +.. code-block:: console + + cd + mkdir -p src + pushd src + git clone https://github.com/stackhpc/kayobe.git -b stackhpc/xena + git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/xena kayobe-config + popd + +Create a virtual environment and install Kayobe: + +.. code-block:: console + + cd + mkdir -p venvs + pushd venvs + virtualenv kayobe + source kayobe/bin/activate + pip install -U pip + pip install ../src/kayobe + popd + +Add initial network configuration: + +.. code-block:: console + + sudo ip l add breth1 type bridge + sudo ip l set breth1 up + sudo ip a add 192.168.33.3/24 dev breth1 + sudo ip l add dummy1 type dummy + sudo ip l set dummy1 up + sudo ip l set dummy1 master breth1 + +Installation +============ + +Acquire the Ansible Vault password for this repository, and store a copy at +``~/vault-pw``. + +The following commands install Kayobe and its dependencies, and prepare the +Ansible control host. + +.. code-block:: console + + export KAYOBE_VAULT_PASSWORD=$(cat ~/vault-pw) + pushd ~/venvs/kayobe + source bin/activate + popd + pushd ~/src/kayobe-config + source kayobe-env --environment ci-aio + kayobe control host bootstrap + +Deployment +========== + +Next, configure the host OS & services. + +.. code-block:: console + + kayobe overcloud host configure + +Finally, deploy the overcloud services. + +.. code-block:: console + + kayobe overcloud service deploy + +The control plane should now be running. + +Testing +======= + +Run a smoke test: + +.. code-block:: console + + cd ~/kayobe + ./dev/overcloud-test-vm.sh diff --git a/doc/source/contributor/environments/ci-builder.rst b/doc/source/contributor/environments/ci-builder.rst new file mode 100644 index 000000000..f146b2107 --- /dev/null +++ b/doc/source/contributor/environments/ci-builder.rst @@ -0,0 +1,115 @@ +========== +ci-builder +========== + +The ``ci-builder`` Kayobe environment is used to build Kolla container images. +Images are built using package repositories in the StackHPC development Pulp +service, and pushed there once built. + +Prerequisites +============= + +* a CentOS Stream 8 or Ubuntu Focal 20.04 host +* access to the local Pulp server + +Setup +===== + +Access the host via SSH. + +Install package dependencies. + +On CentOS: + +.. code-block:: console + + sudo dnf install -y python3-virtualenv + +On Ubuntu: + +.. code-block:: console + + sudo apt update + sudo apt install -y python3-virtualenv + +Clone the Kayobe and Kayobe configuration repositories (this one): + +.. code-block:: console + + cd + mkdir -p src + pushd src + git clone https://github.com/stackhpc/kayobe.git -b stackhpc/xena + git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/xena kayobe-config + popd + +Create a virtual environment and install Kayobe: + +.. code-block:: console + + cd + mkdir -p venvs + pushd venvs + virtualenv kayobe + source kayobe/bin/activate + pip install -U pip + pip install ../src/kayobe + popd + +Add initial network configuration: + +.. code-block:: console + + sudo ip l add breth1 type bridge + sudo ip l set breth1 up + sudo ip a add 192.168.33.3/24 dev breth1 + sudo ip l add dummy1 type dummy + sudo ip l set dummy1 up + sudo ip l set dummy1 master breth1 + +Installation +============ + +Acquire the Ansible Vault password for this repository, and store a copy at +``~/vault-pw``. + +The following commands install Kayobe and its dependencies, and prepare the +Ansible control host. + +.. code-block:: console + + export KAYOBE_VAULT_PASSWORD=$(cat ~/vault-pw) + pushd ~/venvs/kayobe + source bin/activate + popd + pushd ~/src/kayobe-config + source kayobe-env --environment ci-builder + kayobe control host bootstrap + +Deployment +========== + +Next, configure the host OS & services. + +.. code-block:: console + + kayobe seed host configure + +Building images +=============== + +At this point you are ready to build and push some container images. + +.. code-block:: console + + kayobe seed container image build --push + kayobe overcloud container image build --push + +The container images are tagged as ``xena-``. This Kayobe +configuration includes a hook that writes the tag to ``~/kolla_tag``, since +it is not always simple to determine which tag was last applied to built +images. + +To use the new images, edit +``~/src/kayobe-config/etc/kayobe/kolla.yml`` to set the above +tag as the value of the ``kolla_openstack_release`` variable. diff --git a/doc/source/contributor/environments/ci-multinode.rst b/doc/source/contributor/environments/ci-multinode.rst new file mode 100644 index 000000000..72aecc3e7 --- /dev/null +++ b/doc/source/contributor/environments/ci-multinode.rst @@ -0,0 +1,55 @@ +========================== +Multinode Test Environment +========================== + +Set up hosts +============ +1. Create four baremetal instances with a centos 8 stream LVM image, and a + Centos 8 stream vm +2. SSH into each baremetal and run ``sudo chown -R centos:.`` in the home + directory, then add the lines:: + + 10.0.0.34 pelican pelican.service.compute.sms-lab.cloud + 10.205.3.187 pulp-server pulp-server.internal.sms-cloud + + to ``/etc/hosts`` (if you're waiting on them starting up, you can progress + until ``kayobe overcloud host configure`` without this step) + +Basic Kayobe Setup +================== +1. SSH into the VM +2. ``sudo dnf install -y python3-virtualenv`` +3. ``mkdir src`` and ``cd src`` +4. Clone https://github.com/stackhpc/stackhpc-kayobe-config.git, then checkout + commit f31df6256f1b1fea99c84547d44f06c4cb74b161 +5. ``cd ..`` and ``mkdir venvs`` +6. ``virtualenv venvs/kayobe`` and source ``venvs/kayobe/bin/activate`` +7. ``pip install -U pip`` +8. ``pip install ./src/kayobe`` +9. Acquire the Ansible Vault password for this repository, and store a copy at + ``~/vault-pw`` +10. ``export KAYOBE_VAULT_PASSWORD=$(cat ~/vault-pw)`` + +Config changes +============== +1. In etc/kayobe/ansible/requirements.yml remove version from vxlan +2. In etc/kayobe/ansible/configure-vxlan.yml, change the group of + vxlan_interfaces so that the last octet is different e.g. 224.0.0.15 +3. Also under vxlan_interfaces, add vni:x where x is between 500 and 1000 +4. Also under vxlan_interfaces, check vxlan_dstport is not 4789 (this causes + conflicts, change to 4790) +5. In /etc/kayobe/environments/ci-multinode/tf-networks.yml, edit admin_ips so + that the compute and controller IPs line up with the + instances that were created earlier, remove the other IPs for seed and + cephOSD +6. In /etc/kayobe/environments/ci-multinode/network-allocation.yml, remove all + the entries and just assign ``aio_ips:`` an empty set ``[]`` +7. In etc/kayobe/environments/ci-multinode/inventory/hosts, remove the seed +8. run stackhpc-kayobe-config/etc/kayobe/ansible/growroot.yml (if this fails, + manually increase the partition size on each host) + +Final steps +=========== +1. ``source kayobe-env --environment ci-aio`` +2. Run ``kayobe overcloud host configure`` +3. Run ``kayobe overcloud service deploy`` diff --git a/doc/source/contributor/index.rst b/doc/source/contributor/index.rst new file mode 100644 index 000000000..d80f0c4ce --- /dev/null +++ b/doc/source/contributor/index.rst @@ -0,0 +1,10 @@ +================= +Contributor Guide +================= + +This guide is for contributors of the StackHPC Kayobe configuration project. + +.. toctree:: + :maxdepth: 1 + + environments diff --git a/doc/source/index.rst b/doc/source/index.rst index 24f01975d..a9906b35f 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -16,12 +16,9 @@ Overview Contents ======== +.. toctree:: + :maxdepth: 2 - -Intro -===== - - - -Contributors Guide -================== + usage + configuration/index + contributor/index diff --git a/doc/source/usage.rst b/doc/source/usage.rst new file mode 100644 index 000000000..f65b98af1 --- /dev/null +++ b/doc/source/usage.rst @@ -0,0 +1,59 @@ +===== +Usage +===== + +How to consume this configuration +================================= + +This configuration is not a complete Kayobe configuration, rather it should be +treated as a base, in place of the `upstream kayobe-config +`__. Indeed, this repository is +based on the upstream kayobe-config, with some opinionated configuration +changes applied. + +Since this repository makes changes to the base configuration, it works best +when used with Kayobe's `multiple environments +`__ +feature. + +This configuration should be consumed using the `StackHPC Kayobe fork +`__, which includes +backported support for Ansible collections. + +New deployments +--------------- + +If starting a new deployment, clone this repository as the starting point for +your configuration. + +.. code-block:: console + + git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/xena + +Existing deployments +-------------------- + +If migrating an existing deployment to StackHPC Kayobe configuration, you will +need to merge the changes in this repository into your repository. + +.. code-block:: console + + git remote add stackhpc https://github.com/stackhpc/stackhpc-kayobe-config + git fetch stackhpc + git merge stackhpc/stackhpc/xena + +Updating +-------- + +This base configuration will be updated over time, to update repository +versions, container image tags, and other configuration. Deployments may +consume these updates by merging in the changes with their local configuration. + +.. code-block:: console + + git fetch stackhpc + git merge stackhpc/stackhpc/xena + +The intention is to avoid merge conflicts where possible, but there may be +cases where this is difficult. We are open to discussion on how best to +approach this on both sides. diff --git a/etc/kayobe/environments/ci-multinode/README.md b/etc/kayobe/environments/ci-multinode/README.md deleted file mode 100644 index c2511150c..000000000 --- a/etc/kayobe/environments/ci-multinode/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# Multinode Test Environment - -## Set up hosts -1. Create four baremetal instances with a centos 8 stream LVM image, and a -Centos 8 stream vm -2. SSH into each baremetal and run `sudo chown -R centos:.` in the home directory, - then add the lines - >`10.0.0.34 pelican pelican.service.compute.sms-lab.cloud` - >`10.205.3.187 pulp-server pulp-server.internal.sms-cloud` - - to /etc/hosts (if you're waiting on them starting up, you can progress until - `kayobe overcloud host configure` without this step) - -## Basic Kayobe Setup -1. SSH into the VM -1. `sudo dnf install -y python3-virtualenv` -1. `mkdir src` and `cd src` -1. Clone https://github.com/stackhpc/stackhpc-kayobe-config.git, then checkout - commit f31df6256f1b1fea99c84547d44f06c4cb74b161 -1. `cd ..` and `mkdir venvs` -1. `virtualenv venvs/kayobe` and source `venvs/kayobe/bin/activate` -1. `pip install -U pip` -1. `pip install ./src/kayobe` -1. Acquire the Ansible Vault password for this repository, and store a copy at -``~/vault-pw`` -1. `export KAYOBE_VAULT_PASSWORD=$(cat ~/vault-pw)` - -## Config changes -1. In etc/kayobe/ansible/requirements.yml remove version from vxlan -4. In etc/kayobe/ansible/configure-vxlan.yml, change the group of -vxlan_interfaces so that the last octet is different e.g. 224.0.0.15 -5. Also under vxlan_interfaces, add vni:x where x is between 500 and 1000 -5. Also under vxlan_interfaces, check vxlan_dstport is not 4789 (this causes -conflicts, change to 4790) -6. In /etc/kayobe/environments/ci-multinode/tf-networks.yml, - edit admin_ips so that the compute and controller IPs line up with the - instances that were created earlier, remove the other IPs for seed and - cephOSD -7. In /etc/kayobe/environments/ci-multinode/network-allocation.yml, remove all -the entries and just assign `aio_ips:` an empty set `[]` -8. In etc/kayobe/environments/ci-multinode/inventory/hosts, remove the seed -9. run stackhpc-kayobe-config/etc/kayobe/ansible/growroot.yml (if this fails, -manually increase the partition size on each host) - -## Final steps -1. `source kayobe-env --environment ci-aio` -10. Run `kayobe overcloud host configure` -11. Run `kayobe overcloud service deploy` diff --git a/tox.ini b/tox.ini index 5e1f5a7cb..b4dd6e620 100644 --- a/tox.ini +++ b/tox.ini @@ -27,9 +27,10 @@ commands = [testenv:docs] allowlist_externals = rm skip_install = true +# NOTE: Need yoga UC for sphinx 4.2.0 for python 3.10 support deps = - -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/xena} + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/yoga} -r{toxinidir}/doc/requirements.txt commands = rm -rf doc/build/html - sphinx-build -W --keep-going -b html doc/source doc/build/html \ No newline at end of file + sphinx-build -W --keep-going -b html doc/source doc/build/html From 32bc9172982ec2108748d9bf157d6bf9b4a57410 Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Mon, 28 Nov 2022 13:07:00 +0100 Subject: [PATCH 116/292] bifrost: update tag after backport merge Ref: https://github.com/stackhpc/bifrost/pull/16 --- etc/kayobe/kolla/globals.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index f52d530e0..c740e0e65 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -4,7 +4,7 @@ docker_yum_baseurl: "{{ stackhpc_repo_docker_url }}" docker_yum_gpgkey: "https://download.docker.com/linux/centos/gpg" {% if kolla_base_distro == 'centos' %} -bifrost_tag: xena-20221102T110017 +bifrost_tag: xena-20221128T101757 {% endif %} ############################################################################# From 45dd43eb35a30ec2dfd0f37e8feb8053f4f3aea6 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 28 Nov 2022 12:20:34 +0000 Subject: [PATCH 117/292] CI: Ubuntu images not available yet --- .github/workflows/stackhpc-pull-request.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stackhpc-pull-request.yml b/.github/workflows/stackhpc-pull-request.yml index a534d413d..4b027f44f 100644 --- a/.github/workflows/stackhpc-pull-request.yml +++ b/.github/workflows/stackhpc-pull-request.yml @@ -107,7 +107,8 @@ jobs: vm_image: Ubuntu-20.04 OS_CLOUD: sms-lab-release secrets: inherit - if: github.repository == 'stackhpc/stackhpc-kayobe-config' + # NOTE: Ubuntu images not available yet. + if: false && github.repository == 'stackhpc/stackhpc-kayobe-config' all-in-one-ubuntu-ovn: name: aio (Ubuntu OVN) @@ -121,4 +122,5 @@ jobs: vm_image: Ubuntu-20.04 OS_CLOUD: sms-lab-release secrets: inherit - if: github.repository == 'stackhpc/stackhpc-kayobe-config' + # NOTE: Ubuntu images not available yet. + if: false && github.repository == 'stackhpc/stackhpc-kayobe-config' From b528368d44180edfd9aa5111103f683be798a917 Mon Sep 17 00:00:00 2001 From: Bartosz Bezak Date: Tue, 29 Nov 2022 11:07:54 +0100 Subject: [PATCH 118/292] remove ceph from storage group this can leads to problem in hyperconverged setup where one can already have ceph subgroups members --- etc/kayobe/inventory/groups | 3 --- 1 file changed, 3 deletions(-) diff --git a/etc/kayobe/inventory/groups b/etc/kayobe/inventory/groups index 570779a93..e51195309 100644 --- a/etc/kayobe/inventory/groups +++ b/etc/kayobe/inventory/groups @@ -101,9 +101,6 @@ hs-switches ############################################################################### # Ceph groups. -[storage:children] -ceph - [ceph:children] mons mgrs From 10cedc9220fcc3718e06db9252e052fb6e291ee1 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Tue, 29 Nov 2022 14:59:33 +0000 Subject: [PATCH 119/292] Fix templating issue Kolla doesn't template these rules files: https://opendev.org/openstack/kolla-ansible/src/branch/master/ansible/roles/prometheus/tasks/config.yml#L42-L56 --- etc/kayobe/kolla/config/prometheus/system.rules | 2 +- etc/kayobe/kolla/globals.yml | 3 --- etc/kayobe/stackhpc-monitoring.yml | 11 +++++++++++ 3 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 etc/kayobe/stackhpc-monitoring.yml diff --git a/etc/kayobe/kolla/config/prometheus/system.rules b/etc/kayobe/kolla/config/prometheus/system.rules index a8fb59aaa..fe3a2b9ac 100644 --- a/etc/kayobe/kolla/config/prometheus/system.rules +++ b/etc/kayobe/kolla/config/prometheus/system.rules @@ -16,7 +16,7 @@ groups: description: "{{ $labels.device }} is {{ $value }}% full." - alert: LowMemory - expr: (node_memory_MemAvailable_bytes / 1024^3) < {{ alertmanager_low_memory_threshold_gib }} + expr: (node_memory_MemAvailable_bytes / 1024^3) < {% endraw %}{{ alertmanager_low_memory_threshold_gib }}{% raw %} for: 1m labels: severity: alert diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index c740e0e65..774da38ba 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -12,8 +12,5 @@ bifrost_tag: xena-20221128T101757 es_heap_size: 8g prometheus_cmdline_extras: "--storage.tsdb.retention.time=30d" -# Threshold to trigger a LowMemory alert in Gibibytes (GiB). When the amount -# of free memory is lower than this value an alert will be triggered. -alertmanager_low_memory_threshold_gib: 5 ############################################################################# diff --git a/etc/kayobe/stackhpc-monitoring.yml b/etc/kayobe/stackhpc-monitoring.yml new file mode 100644 index 000000000..43f1f309f --- /dev/null +++ b/etc/kayobe/stackhpc-monitoring.yml @@ -0,0 +1,11 @@ +--- +# StackHPC monitoring configuration + +############################################################################### +# Alert configuration + +# Threshold to trigger a LowMemory alert in Gibibytes (GiB). When the amount +# of free memory is lower than this value an alert will be triggered. +alertmanager_low_memory_threshold_gib: 5 + +############################################################################### From 6f9cbaf7a38feaf16904e839ba83b76d02a40c9a Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Tue, 29 Nov 2022 15:13:12 +0000 Subject: [PATCH 120/292] Docs for monitoring configuration --- doc/source/configuration/index.rst | 1 + doc/source/configuration/monitoring.rst | 13 +++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 doc/source/configuration/monitoring.rst diff --git a/doc/source/configuration/index.rst b/doc/source/configuration/index.rst index 133045779..c84f735cb 100644 --- a/doc/source/configuration/index.rst +++ b/doc/source/configuration/index.rst @@ -9,3 +9,4 @@ the various features provided. :maxdepth: 1 release-train + monitoring diff --git a/doc/source/configuration/monitoring.rst b/doc/source/configuration/monitoring.rst new file mode 100644 index 000000000..7e53629f1 --- /dev/null +++ b/doc/source/configuration/monitoring.rst @@ -0,0 +1,13 @@ +======================== +Monitoring Configuration +======================== + +StackHPC kayobe config includes a reference monitoring stack based on +Prometheus. Whilst this often works out of the box, there are some tunables +which can be customised to adapt the configuration to a particular deployment. + +The configuration options can be found in +``etc/kayobe/stackhpc-monitoring.yml``: + +.. literalinclude:: ../../../etc/kayobe/stackhpc-monitoring.yml + :language: yaml \ No newline at end of file From 15024b27c1771a52ad080f2b6f02f67cc525baef Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 29 Nov 2022 19:59:30 +0000 Subject: [PATCH 121/292] CI: symlink to prometheus & grafana config in ci-aio & ci-multinode Symlinks created via: ln -s ../../../../kolla/config/grafana/ etc/kayobe/environments/ci-aio/kolla/config/ ln -s ../../../../kolla/config/prometheus/ etc/kayobe/environments/ci-aio/kolla/config/ ln -s ../../../../kolla/config/grafana/ etc/kayobe/environments/ci-multinode/kolla/config/ ln -s ../../../../kolla/config/prometheus/ etc/kayobe/environments/ci-multinode/kolla/config/ --- etc/kayobe/environments/ci-aio/kolla/config/grafana | 1 + etc/kayobe/environments/ci-aio/kolla/config/prometheus | 1 + etc/kayobe/environments/ci-multinode/kolla/config/grafana | 1 + etc/kayobe/environments/ci-multinode/kolla/config/prometheus | 1 + 4 files changed, 4 insertions(+) create mode 120000 etc/kayobe/environments/ci-aio/kolla/config/grafana create mode 120000 etc/kayobe/environments/ci-aio/kolla/config/prometheus create mode 120000 etc/kayobe/environments/ci-multinode/kolla/config/grafana create mode 120000 etc/kayobe/environments/ci-multinode/kolla/config/prometheus diff --git a/etc/kayobe/environments/ci-aio/kolla/config/grafana b/etc/kayobe/environments/ci-aio/kolla/config/grafana new file mode 120000 index 000000000..0e711c2ae --- /dev/null +++ b/etc/kayobe/environments/ci-aio/kolla/config/grafana @@ -0,0 +1 @@ +../../../../kolla/config/grafana/ \ No newline at end of file diff --git a/etc/kayobe/environments/ci-aio/kolla/config/prometheus b/etc/kayobe/environments/ci-aio/kolla/config/prometheus new file mode 120000 index 000000000..9a40a2c64 --- /dev/null +++ b/etc/kayobe/environments/ci-aio/kolla/config/prometheus @@ -0,0 +1 @@ +../../../../kolla/config/prometheus/ \ No newline at end of file diff --git a/etc/kayobe/environments/ci-multinode/kolla/config/grafana b/etc/kayobe/environments/ci-multinode/kolla/config/grafana new file mode 120000 index 000000000..0e711c2ae --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/kolla/config/grafana @@ -0,0 +1 @@ +../../../../kolla/config/grafana/ \ No newline at end of file diff --git a/etc/kayobe/environments/ci-multinode/kolla/config/prometheus b/etc/kayobe/environments/ci-multinode/kolla/config/prometheus new file mode 120000 index 000000000..9a40a2c64 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/kolla/config/prometheus @@ -0,0 +1 @@ +../../../../kolla/config/prometheus/ \ No newline at end of file From d481c17c347e745c3d47dfb0840f36f04ecd05c4 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 1 Dec 2022 20:17:07 +0000 Subject: [PATCH 122/292] docs: add Cephadm section Converted from the wiki https://wiki.stackhpc.com/doc/cephadm-kayobe-LHOSFUOLvp --- doc/source/configuration/cephadm.rst | 296 +++++++++++++++++++++++++++ doc/source/configuration/index.rst | 1 + 2 files changed, 297 insertions(+) create mode 100644 doc/source/configuration/cephadm.rst diff --git a/doc/source/configuration/cephadm.rst b/doc/source/configuration/cephadm.rst new file mode 100644 index 000000000..e207dd5b7 --- /dev/null +++ b/doc/source/configuration/cephadm.rst @@ -0,0 +1,296 @@ +================ +Cephadm & Kayobe +================ + +This section describes how to use the Cephadm integration included in StackHPC +Kayobe configuration since Xena to deploy Ceph. + +The Cephadm integration takes the form of custom playbooks that wrap +around the Ansible `stackhpc.cephadm collection +`_ and provide a means to +create or modify Ceph cluster deployments. Supported features are: + +- creating a new cluster from scratch (RedHat/Debian family distros + supported) +- creating pools, users, CRUSH rules and EC profiles +- modifying the OSD spec after initial deployment +- destroying the cluster + +Resources +========= + +- https://docs.ceph.com/en/pacific/cephadm/index.html +- https://docs.ceph.com/en/pacific/ +- https://github.com/stackhpc/ansible-collection-cephadm + +Configuration +============= + +Inventory +--------- + +The collection assumes a set of group entries in Ansible’s inventory. +The following groups are defined in the Kayobe base configuration +inventory ``groups`` file: + +- ``ceph`` (parent for all Ceph nodes) +- ``mons`` +- ``mgrs`` +- ``osds`` +- ``rgws`` (optional) + +Ceph hosts should be added to these groups as appropriate. Typically at +least the ``mons``, ``mgrs``, and ``osds`` groups should be populated. + +Example: Separate monitors +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For a system with separate monitor hosts, the following could be added +to ``etc/kayobe/environments//inventory/groups``, to define two +top-level Ceph host groups: + +.. code:: ini + + [ceph-mons] + [ceph-osds] + + [mons:children] + ceph-mons + + [mgrs:children] + ceph-mons + + [osds:children] + ceph-osds + +Then, populate the ``ceph-mons`` and ``ceph-osds`` groups with the +necessary hosts, e.g. in +``etc/kayobe/environments//inventory/hosts``. + +Example: Colocated monitors +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For a system with only colocated monitor and OSD hosts, the following +might be appropriate: + +NOTE: we are using ``storage`` rather than ``ceph``, since ``ceph`` +is already required by the cephadm collection, and redefining it would +introduce a circular dependency between groups. + +.. code:: ini + + [storage] + + [mons:children] + storage + + [mgrs:children] + storage + + [osds:children] + storage + +Then populate the ``storage`` group with the necessary hosts, +e.g. in ``etc/kayobe/environments//inventory/hosts``. + +Ceph deployment configuration +----------------------------- + +Default variables for configuring Ceph are provided in +``etc/kayobe/cephadm.yml``. Many of these defaults will be sufficient, +but you will likely need to set ``cephadm_osd_spec`` to define the OSD +specification. + +OSD specification +~~~~~~~~~~~~~~~~~ + +The following example is a basic OSD spec that adds OSDs for all +available disks: + +.. code:: yaml + + cephadm_osd_spec: + service_type: osd + service_id: osd_spec_default + placement: + host_pattern: "*" + data_devices: + all: true + +More information about OSD service placement is available +`here `__. + +Container image +~~~~~~~~~~~~~~~ + +The container image to be deployed by Cephadm is defined by +``cephadm_image``, and the tag by ``cephadm_image_tag``. The StackHPC +Kayobe configuration provides defaults for both of these. + +Firewalld +~~~~~~~~~ + +If the Ceph storage hosts are running firewalld, it may be helpful to +set ``cephadm_enable_firewalld`` to ``true`` to enable configuration of +firewall rules for Ceph services. + +Ceph post-deployment configuration +---------------------------------- + +The ``stackhpc.cephadm`` collection also provides roles for +post-deployment configuration of pools, users, CRUSH rules and EC +profiles. + +EC profiles +~~~~~~~~~~~ + +An Erasure Coding (EC) profile is required in order to use Erasure Coded +storage pools. Example EC profile: + +.. code:: yaml + + # List of Ceph erasure coding profiles. See stackhpc.cephadm.ec_profiles role + # for format. + cephadm_ec_profiles: + - name: ec_4_2_hdd + k: 4 + m: 2 + crush_device_class: hdd + +CRUSH rules +~~~~~~~~~~~ + +CRUSH rules may not be required in a simple setup with a homogeneous +pool of storage. They are useful when there are different tiers of +storage. The following example CRUSH rules define separate tiers for +Hard Disk Drives (HDDs) and Solid State Drives (SSDs). + +.. code:: yaml + + # List of Ceph CRUSH rules. See stackhpc.cephadm.crush_rules role for format. + cephadm_crush_rules: + - name: replicated_hdd + bucket_root: default + bucket_type: host + device_class: hdd + rule_type: replicated + state: present + - name: replicated_ssd + bucket_root: default + bucket_type: host + device_class: ssd + rule_type: replicated + state: present + +Pools +~~~~~ + +The following example pools should be sufficient to work with the +default `external Ceph +configuration `__ +for Cinder, Cinder backup, Glance, and Nova in Kolla Ansible. + +.. code:: yaml + + # List of Ceph pools. See stackhpc.cephadm.pools role for format. + cephadm_pools: + - name: backups + application: rbd + state: present + - name: images + application: rbd + state: present + - name: volumes + application: rbd + state: present + - name: vms + application: rbd + state: present + +If a pool needs to use a particular CRUSH rule, this can be defined via +``rule_name: ``. + +Keys +~~~~ + +The following example keys should be sufficient to work with the default +`external Ceph +configuration `__ +for Cinder, Cinder backup, Glance, and Nova in Kolla Ansible. + +.. code:: yaml + + # List of Cephx keys. See stackhpc.cephadm.keys role for format. + cephadm_keys: + - name: client.cinder + caps: + mon: "profile rbd" + osd: "profile rbd pool=volumes, profile rbd pool=vms, profile rbd-read-only pool=images" + mgr: "profile rbd pool=volumes, profile rbd pool=vms" + - name: client.cinder-backup + caps: + mon: "profile rbd" + osd: "profile rbd pool=volumes, profile rbd pool=backups" + mgr: "profile rbd pool=volumes, profile rbd pool=backups" + - name: client.glance + caps: + mon: "profile rbd" + osd: "profile rbd pool=images" + mgr: "profile rbd pool=images" + state: present + +Deployment +========== + +Host configuration +------------------ + +Configure the Ceph hosts: + +.. code:: bash + + kayobe overcloud host configure --limit storage --kolla-limit storage + +Ceph deployment +--------------- + +.. + **FIXME**: Wait for Ceph to come up, so that we can just run cephadm.yml + +Deploy the Ceph services: + +.. code:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-deploy.yml + +You can check the status of Ceph via Cephadm on the storage nodes: + +.. code:: bash + + sudo cephadm shell -- ceph -s + +Once the Ceph cluster has finished initialising, run the full +cephadm.yml playbook to perform post-deployment configuration: + +.. code:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm.yml + +The ``cephadm.yml`` playbook imports various other playbooks, which may +also be run individually to perform specific tasks. + +Configuration generation +------------------------ + +Generate keys and configuration for Kolla Ansible: + +.. code:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-gather-keys.yml + +This will generate Ceph keys and configuration under +``etc/kayobe/environments//kolla/config/``, which should be +committed to the configuration. + +This configuration will be used during +``kayobe overcloud service deploy``. diff --git a/doc/source/configuration/index.rst b/doc/source/configuration/index.rst index c84f735cb..21bff56f7 100644 --- a/doc/source/configuration/index.rst +++ b/doc/source/configuration/index.rst @@ -9,4 +9,5 @@ the various features provided. :maxdepth: 1 release-train + cephadm monitoring From 2de8f98e94082f440ab661945352d33681cb16eb Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 1 Dec 2022 20:27:50 +0000 Subject: [PATCH 123/292] docs: add walled garden section Converted from the wiki https://wiki.stackhpc.com/doc/walled-garden-deployment-guide-CbVw69ANfD --- doc/source/configuration/index.rst | 1 + doc/source/configuration/walled-garden.rst | 119 +++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 doc/source/configuration/walled-garden.rst diff --git a/doc/source/configuration/index.rst b/doc/source/configuration/index.rst index c84f735cb..54e48a166 100644 --- a/doc/source/configuration/index.rst +++ b/doc/source/configuration/index.rst @@ -8,5 +8,6 @@ the various features provided. .. toctree:: :maxdepth: 1 + walled-garden release-train monitoring diff --git a/doc/source/configuration/walled-garden.rst b/doc/source/configuration/walled-garden.rst new file mode 100644 index 000000000..0df8ef8b5 --- /dev/null +++ b/doc/source/configuration/walled-garden.rst @@ -0,0 +1,119 @@ +============================== +Walled Garden deployment guide +============================== + +This document describes how to configure Kayobe for a “walled garden” +deployment, where most hosts do not have external network access. + +Routing +======= + +With the exception of hosts running HAProxy (usually controllers), +overcloud hosts typically do not have external network access. + +The hosts running HAProxy typically have a gateway on the public API +network for external network access. + +NTP +=== + +Overcloud hosts initially use the seed’s NTP server. + +In ``etc/kayobe/inventory/group_vars/overcloud/time``: + +.. code:: yaml + + --- + # NTP services for overcloud hosts + # During early initialisation we use the seed Node + # Following deployment we include the OpenStack VIP + + chrony_ntp_servers: + - server: "{{ admin_oc_net_name | net_ip(inventory_hostname=groups['seed'][0]) }}" + +Proxy +===== + +We use a Squid caching proxy to access external web resources. + +In ``etc/kayobe/seed.yml``, enable the Squid proxy container on the +seed: + +.. code:: yaml + + # Seed container running a Squid caching proxy. This can be used to proxy + # HTTP(S) requests from control plane hosts. + seed_squid_container_enabled: true + +In some environments we have found that squid’s preference for IPv6 can +cause problems. It can be forced to prefer IPv4, by adding the following +in ``etc/kayobe/containers/squid_proxy/squid.conf``: + +.. code:: none + + dns_v4_first on + +In ``etc/kayobe/inventory/group_vars/overcloud/proxy`` (and any other +groups that need to use the proxy), configure overcloud hosts to use the +proxy: + +.. code:: yaml + + --- + # HTTP proxy URL (format: http(s)://[user:password@]proxy_name:port). By + # default no proxy is used. + http_proxy: "http://{{ admin_oc_net_name | net_ip(inventory_hostname=groups['seed'][0]) }}:3128" + + # HTTPS proxy URL (format: http(s)://[user:password@]proxy_name:port). By + # default no proxy is used. + https_proxy: "{{ http_proxy }}" + + # List of domains, hostnames, IP addresses and networks for which no proxy is + # used. Defaults to ["127.0.0.1", "localhost", "{{ ('http://' ~ + # docker_registry) | urlsplit('hostname') }}"] if docker_registry is set, or + # ["127.0.0.1", "localhost"] otherwise. This is configured only if either + # http_proxy or https_proxy is set. + no_proxy: + - "127.0.0.1" + - "localhost" + - "{{ ('http://' ~ docker_registry) | urlsplit('hostname') if docker_registry else '' }}" + - "{{ admin_oc_net_name | net_ip(inventory_hostname=groups['seed'][0]) }}" + - "{{ kolla_external_fqdn }}" + - "{{ kolla_internal_fqdn }}" + + # PyPI proxy URL (format: http(s)://[user:password@]proxy_name:port) + pip_proxy: "{{ https_proxy }}" + +We typically don’t use the proxy for DNF package updates, or for +container image downloads, since the Pulp server is hosted on the seed. +The ``no_proxy`` setting should handle this. + +For Ubuntu hosts, where package repos are not hosted in a local Pulp +server, you will also want to proxy APT requests. This can be done by +adding the following in +``etc/kayobe/inventory/group_vars/overcloud/proxy``: + +.. code:: yaml + + # Apt proxy URL for HTTP. Default is empty (no proxy). + apt_proxy_http: "{{ http_proxy }}" + + # Apt proxy URL for HTTPS. Default is {{ apt_proxy_http }}. + apt_proxy_https: "{{ https_proxy }}" + +Typically, container images are pulled from the local Pulp server. If +you need to be able to pull container images from external sources, it +may be necessary to add proxy configuration for Docker. This is Kolla +Ansible configuration, rather than Kayobe, in +``etc/kayobe/kolla/inventory/group_vars/overcloud``: + +.. code:: yaml + + --- + # Use a proxy for external Docker image pulls + docker_http_proxy: "http://:3128" + docker_https_proxy: "http://:3128" + docker_no_proxy: + - "127.0.0.1" + - "localhost" + - "{{ ('http://' ~ docker_registry) | urlsplit('hostname') if docker_registry else '' }}" From d4b449cd30235c1bded079480c1eb1a93356a632 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Mon, 5 Dec 2022 13:25:29 +0100 Subject: [PATCH 124/292] Replace hardcoded threshold by temp_max value The temp_max value is dynamically gathered from the device [1]. With Xeon CPUs (coretemp driver), it is often 90C, but sometimes lower. This can help reduce the frequency of alerts with busy hypervisors. [1] https://docs.kernel.org/hwmon/coretemp.html --- etc/kayobe/kolla/config/prometheus/system.rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/kolla/config/prometheus/system.rules b/etc/kayobe/kolla/config/prometheus/system.rules index fe3a2b9ac..ffc7d25a3 100644 --- a/etc/kayobe/kolla/config/prometheus/system.rules +++ b/etc/kayobe/kolla/config/prometheus/system.rules @@ -34,7 +34,7 @@ groups: description: "OOM kill detected" - alert: Overheating - expr: node_hwmon_temp_celsius >= 85 + expr: node_hwmon_temp_celsius >= node_hwmon_temp_max_celsius for: 1m labels: severity: warning From e3f83f5111f7565342de31276a68c106fa20af08 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 5 Dec 2022 17:08:35 +0000 Subject: [PATCH 125/292] Adding support for pulp RBAC --- README.rst | 32 +++++++-- .../certs/ark.stackhpc.com/client-cert.pem | 0 .../certs/ark.stackhpc.com/client-key.pem | 0 etc/kayobe/ansible/requirements.yml | 4 +- etc/kayobe/pulp.yml | 67 +++++++++---------- 5 files changed, 58 insertions(+), 45 deletions(-) delete mode 100644 etc/kayobe/ansible/certs/ark.stackhpc.com/client-cert.pem delete mode 100644 etc/kayobe/ansible/certs/ark.stackhpc.com/client-key.pem diff --git a/README.rst b/README.rst index 304a8edbc..6f62d93d9 100644 --- a/README.rst +++ b/README.rst @@ -13,7 +13,7 @@ StackHPC provides packages and container images for OpenStack via `Ark Deployments should use a local `Pulp `__ repository server to synchronise content from Ark and serve it locally. Access to the -repositories on Ark is controlled via X.509 certificates issued by StackHPC. +repositories on Ark is controlled via user accounts issued by StackHPC. This configuration is a base, and should be merged with any existing Kayobe configuration. It currently provides the following: @@ -112,11 +112,6 @@ password: stackhpc_release_pulp_username: stackhpc_release_pulp_password: -The client certificate and private key issued by StackHPC should be stored in -``etc/kayobe/ansible/certs/ark.stackhpc.com/client-cert.pem`` and -``etc/kayobe/ansible/certs/ark.stackhpc.com/client-key.pem``, respectively, -with the private key encrypted via Ansible Vault. - The distribution name for the environment should be configured as either ``development`` or ``production`` via ``stackhpc_repo_distribution`` in ``etc/kayobe/stackhpc.yml``. @@ -216,6 +211,31 @@ with the push repository using the pulp CLI: Started background task /pulp/api/v3/tasks/1f0a474a-b7c0-44b4-9ef4-ed633077f4d8/ .Done. +HTTP Error 404: Not Found +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If your login credentials are incorrect, or lack the required permissions, +you will see a 404 error during ``pulp-repo-sync.yml``: + +.. code-block:: console + TASK [stackhpc.pulp.pulp_repository : Sync RPM remotes into repositories] **************************************************************************************************************************************** + An exception occurred during task execution. To see the full traceback, use -vvv. The error was: Exception: Task failed to complete. (failed; 404, message='Not Found', url=URL('https://ark.stackhpc.com/pulp/content/centos/8-stream/BaseOS/x86_64/os/20211122T102435')) + failed: [localhost] (item=centos-stream-8-baseos-development) => changed=false + ansible_loop_var: item + item: + name: centos-stream-8-baseos-development + policy: on_demand + proxy_url: __omit_place_holder__d35452c39719f081229941a64fd2cdce1188a287 + remote_password: + remote_username: + required: true + state: present + sync_policy: mirror_complete + url: https://ark.stackhpc.com/pulp/content/centos/8-stream/BaseOS/x86_64/os/20211122T102435 + msg: Task failed to complete. (failed; 404, message='Not Found', url=URL('https://ark.stackhpc.com/pulp/content/centos/8-stream/BaseOS/x86_64/os/20211122T102435')) ''' +The issue can be rectified by updating the ``stackhpc_release_pulp_username`` +and ``stackhpc_release_pulp_password`` variables + Environments ============ diff --git a/etc/kayobe/ansible/certs/ark.stackhpc.com/client-cert.pem b/etc/kayobe/ansible/certs/ark.stackhpc.com/client-cert.pem deleted file mode 100644 index e69de29bb..000000000 diff --git a/etc/kayobe/ansible/certs/ark.stackhpc.com/client-key.pem b/etc/kayobe/ansible/certs/ark.stackhpc.com/client-key.pem deleted file mode 100644 index e69de29bb..000000000 diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index 8293d49dc..0ced4d527 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -1,6 +1,4 @@ --- collections: - name: stackhpc.pulp - version: 0.4.0 - - name: pulp.squeezer - version: 0.0.11 + version: 0.4.1 diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index cdc62c85f..d6017895e 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -19,15 +19,10 @@ pulp_proxy_url: "{{ omit }}" # Base URL of the StackHPC Pulp service. stackhpc_release_pulp_url: "https://ark.stackhpc.com" -# Credentials used to access the StackHPC Ark container image registry. +# Credentials used to access the StackHPC Ark pulp server. stackhpc_release_pulp_username: stackhpc_release_pulp_password: -# Client certificates used to access StackHPC Ark repositories. -# They are trusted by the 'release' cert guard's CA. -stackhpc_release_pulp_client_cert: "{{ lookup('file', kayobe_config_path ~ '/ansible/certs/ark.stackhpc.com/client-cert.pem') | trim }}" -stackhpc_release_pulp_client_key: "{{ lookup('file', kayobe_config_path ~ '/ansible/certs/ark.stackhpc.com/client-key.pem') | trim }}" - # Content URL of the StackHPC Pulp service. stackhpc_release_pulp_content_url: "{{ stackhpc_release_pulp_url }}/pulp/content" @@ -44,8 +39,8 @@ stackhpc_pulp_repository_deb_repos: # Base Ubuntu Focal repositories - name: Ubuntu focal url: "{{ stackhpc_release_pulp_content_url }}/ubuntu/focal/{{ stackhpc_pulp_repo_ubuntu_focal_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand proxy_url: "{{ pulp_proxy_url }}" architectures: amd64 @@ -57,8 +52,8 @@ stackhpc_pulp_repository_deb_repos: - name: Ubuntu focal security url: "{{ stackhpc_release_pulp_content_url }}/ubuntu/focal-security/{{ stackhpc_pulp_repo_ubuntu_focal_security_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand proxy_url: "{{ pulp_proxy_url }}" architectures: amd64 @@ -71,8 +66,8 @@ stackhpc_pulp_repository_deb_repos: # Ubuntu Cloud Archive (UCA) repositories - name: Ubuntu Cloud Archive url: "{{ stackhpc_release_pulp_content_url }}/ubuntu-cloud-archive/{{ stackhpc_pulp_repo_ubuntu_cloud_archive_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand proxy_url: "{{ pulp_proxy_url }}" architectures: amd64 @@ -85,8 +80,8 @@ stackhpc_pulp_repository_deb_repos: # Third-party repositories - name: Docker CE for Ubuntu url: "{{ stackhpc_release_pulp_content_url }}/docker-ce/ubuntu/{{ stackhpc_pulp_repo_docker_ce_ubuntu_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand proxy_url: "{{ pulp_proxy_url }}" architectures: amd64 @@ -193,8 +188,8 @@ stackhpc_pulp_repository_rpm_repos: # Base CentOS 8 Stream repositories - name: CentOS Stream 8 - AppStream url: "{{ stackhpc_release_pulp_content_url }}/centos/8-stream/AppStream/x86_64/os/{{ stackhpc_pulp_repo_centos_stream_8_appstream_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_complete @@ -202,8 +197,8 @@ stackhpc_pulp_repository_rpm_repos: required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}" - name: CentOS Stream 8 - BaseOS url: "{{ stackhpc_release_pulp_content_url }}/centos/8-stream/BaseOS/x86_64/os/{{ stackhpc_pulp_repo_centos_stream_8_baseos_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_complete @@ -211,8 +206,8 @@ stackhpc_pulp_repository_rpm_repos: required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}" - name: CentOS Stream 8 - Extras url: "{{ stackhpc_release_pulp_content_url }}/centos/8-stream/extras/x86_64/os/{{ stackhpc_pulp_repo_centos_stream_8_extras_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_complete @@ -222,40 +217,40 @@ stackhpc_pulp_repository_rpm_repos: # Base Rocky 8 repositories - name: Rocky Linux 8 - AppStream url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/AppStream/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_appstream_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand sync_policy: mirror_complete state: present required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" - name: Rocky Linux 8 - BaseOS url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/BaseOS/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_baseos_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand sync_policy: mirror_complete state: present required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" - name: Rocky Linux 8 - Extras url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/extras/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_extras_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand sync_policy: mirror_complete state: present required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" - name: Rocky Linux 8 - NFV url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/nfv/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_nfv_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand sync_policy: mirror_complete state: present required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" - name: Rocky Linux 8 - PowerTools url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/PowerTools/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_powertools_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand sync_policy: mirror_complete state: present @@ -264,8 +259,8 @@ stackhpc_pulp_repository_rpm_repos: # EPEL repositories - name: Extra Packages for Enterprise Linux 8 - x86_64 url: "{{ stackhpc_release_pulp_content_url }}/epel/8/Everything/x86_64/{{ stackhpc_pulp_repo_epel_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_content_only @@ -273,8 +268,8 @@ stackhpc_pulp_repository_rpm_repos: required: "{{ stackhpc_pulp_sync_el_8 | bool }}" - name: Extra Packages for Enterprise Linux Modular 8 - x86_64 url: "{{ stackhpc_release_pulp_content_url }}/epel/8/Modular/x86_64/{{ stackhpc_pulp_repo_epel_modular_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_complete @@ -284,8 +279,8 @@ stackhpc_pulp_repository_rpm_repos: # Third-party repositories - name: Docker CE for CentOS 8 url: "{{ stackhpc_release_pulp_content_url }}/docker-ce/centos/8/x86_64/stable/{{ stackhpc_pulp_repo_docker_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_complete From 7437660f99595b3b93ae571ba2b8e5ee589b862a Mon Sep 17 00:00:00 2001 From: Matt Anson Date: Tue, 6 Dec 2022 11:22:54 +0000 Subject: [PATCH 126/292] Set Pulp 3.21 init: false --- etc/kayobe/seed.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index 78cb92e79..33e2637ac 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -108,6 +108,9 @@ seed_pulp_container: post: "{{ kayobe_config_path }}/containers/pulp/post.yml" tag: "3.21" network_mode: host + # Override deploy_containers_defaults.init == true to ensure + # s6-overlay-suexec starts as pid 1 + init: false volumes: - /opt/kayobe/containers/pulp:/etc/pulp - pulp_storage:/var/lib/pulp From 60f8588d5db698081a337d4fc32546855a2e560e Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 5 Dec 2022 17:29:18 +0000 Subject: [PATCH 127/292] updating docs --- doc/source/configuration/release-train.rst | 37 +++++++++++++++++----- etc/kayobe/ansible/advise-run.yml | 2 +- etc/kayobe/ansible/requirements.yml | 1 - 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/doc/source/configuration/release-train.rst b/doc/source/configuration/release-train.rst index 8a25f0653..39bb846c3 100644 --- a/doc/source/configuration/release-train.rst +++ b/doc/source/configuration/release-train.rst @@ -7,7 +7,7 @@ StackHPC provides packages and container images for OpenStack via `Ark Deployments should use a local `Pulp `__ repository server to synchronise content from Ark and serve it locally. Access to the -repositories on Ark is controlled via X.509 certificates issued by StackHPC. +repositories on Ark is controlled via user accounts issued by StackHPC. This configuration is a base, and should be merged with any existing Kayobe configuration. It currently provides the following: @@ -41,19 +41,14 @@ Pulp startup. StackHPC Ark ------------ -The container image registry credentials issued by StackHPC should be -configured in ``etc/kayobe/pulp.yml``, using Ansible Vault to encrypt the -password: +The Ark pulp credentials issued by StackHPC should be configured in +``etc/kayobe/pulp.yml``, using Ansible Vault to encrypt the password: .. code-block:: yaml stackhpc_release_pulp_username: stackhpc_release_pulp_password: -The client certificate and private key issued by StackHPC should be stored in -``etc/kayobe/ansible/certs/ark.stackhpc.com/client-cert.pem`` and -``etc/kayobe/ansible/certs/ark.stackhpc.com/client-key.pem``, respectively, -with the private key encrypted via Ansible Vault. The distribution name for the environment should be configured as either ``development`` or ``production`` via ``stackhpc_repo_distribution`` in @@ -153,3 +148,29 @@ with the push repository using the pulp CLI: (venv-pulp) [stack@seed ~]$ pulp --base-url http://:8080--username admin --password container distribution destroy --name stackhpc/centos-source-prometheus-jiralert Started background task /pulp/api/v3/tasks/1f0a474a-b7c0-44b4-9ef4-ed633077f4d8/ .Done. + +HTTP Error 404: Not Found +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If your login credentials are incorrect, or lack the required permissions, +you will see a 404 error during ``pulp-repo-sync.yml``: + +.. code-block:: console + TASK [stackhpc.pulp.pulp_repository : Sync RPM remotes into repositories] **************************************************************************************************************************************** + An exception occurred during task execution. To see the full traceback, use -vvv. The error was: Exception: Task failed to complete. (failed; 404, message='Not Found', url=URL('https://ark.stackhpc.com/pulp/content/centos/8-stream/BaseOS/x86_64/os/20211122T102435')) + failed: [localhost] (item=centos-stream-8-baseos-development) => changed=false + ansible_loop_var: item + item: + name: centos-stream-8-baseos-development + policy: on_demand + proxy_url: __omit_place_holder__d35452c39719f081229941a64fd2cdce1188a287 + remote_password: + remote_username: + required: true + state: present + sync_policy: mirror_complete + url: https://ark.stackhpc.com/pulp/content/centos/8-stream/BaseOS/x86_64/os/20211122T102435 + msg: Task failed to complete. (failed; 404, message='Not Found', url=URL('https://ark.stackhpc.com/pulp/content/centos/8-stream/BaseOS/x86_64/os/20211122T102435')) ''' + +The issue can be rectified by updating the ``stackhpc_release_pulp_username`` +and ``stackhpc_release_pulp_password`` variables diff --git a/etc/kayobe/ansible/advise-run.yml b/etc/kayobe/ansible/advise-run.yml index 09440aeb9..d0ad2eee1 100644 --- a/etc/kayobe/ansible/advise-run.yml +++ b/etc/kayobe/ansible/advise-run.yml @@ -8,7 +8,7 @@ venv: "~/venvs/advise-review" input_dir: "{{ lookup('env', 'PWD') }}/overcloud-introspection-data" output_dir: "{{ lookup('env', 'PWD') }}/review" - advise_pattern: ".*.eval" # Uses regex + advise_pattern: ".*.eval" # Uses regex tasks: - name: Install dependencies pip: diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index 2ab9ab661..77aaa9f99 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -6,4 +6,3 @@ collections: version: 0.4.1 roles: - src: stackhpc.vxlan - From 25d50f722b90ddf1dd895888e48ada825c95cc91 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 11 Nov 2022 09:31:24 +0100 Subject: [PATCH 128/292] Wait longer before raising Elasticsearch alerts --- etc/kayobe/kolla/config/prometheus/elasticsearch.rules | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/etc/kayobe/kolla/config/prometheus/elasticsearch.rules b/etc/kayobe/kolla/config/prometheus/elasticsearch.rules index 974bf4e99..42a196b9a 100644 --- a/etc/kayobe/kolla/config/prometheus/elasticsearch.rules +++ b/etc/kayobe/kolla/config/prometheus/elasticsearch.rules @@ -44,7 +44,7 @@ groups: - alert: ElasticsearchClusterRed expr: elasticsearch_cluster_health_status{color="red"} == 1 - for: 0m + for: 5m labels: severity: critical annotations: @@ -53,7 +53,7 @@ groups: - alert: ElasticsearchClusterYellow expr: elasticsearch_cluster_health_status{color="yellow"} == 1 - for: 5m + for: 15m labels: severity: warning annotations: @@ -80,7 +80,7 @@ groups: - alert: ElasticsearchUnassignedShards expr: elasticsearch_cluster_health_unassigned_shards > 0 - for: 0m + for: 5m labels: severity: critical annotations: From 61f26484758f012107a271c2716e0ac0bf139e10 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 7 Dec 2022 13:50:34 +0000 Subject: [PATCH 129/292] Fix release-train docs formatting --- doc/source/configuration/release-train.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/configuration/release-train.rst b/doc/source/configuration/release-train.rst index 39bb846c3..04b583410 100644 --- a/doc/source/configuration/release-train.rst +++ b/doc/source/configuration/release-train.rst @@ -156,6 +156,7 @@ If your login credentials are incorrect, or lack the required permissions, you will see a 404 error during ``pulp-repo-sync.yml``: .. code-block:: console + TASK [stackhpc.pulp.pulp_repository : Sync RPM remotes into repositories] **************************************************************************************************************************************** An exception occurred during task execution. To see the full traceback, use -vvv. The error was: Exception: Task failed to complete. (failed; 404, message='Not Found', url=URL('https://ark.stackhpc.com/pulp/content/centos/8-stream/BaseOS/x86_64/os/20211122T102435')) failed: [localhost] (item=centos-stream-8-baseos-development) => changed=false From 59192f8106ba64c83533c4d2ef61a617eee09d8e Mon Sep 17 00:00:00 2001 From: Rafal Lewandowski Date: Thu, 8 Dec 2022 14:26:31 +0100 Subject: [PATCH 130/292] overcloud-dib.yml syntax fix --- etc/kayobe/overcloud-dib.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/overcloud-dib.yml b/etc/kayobe/overcloud-dib.yml index 78fff9471..9c34650ff 100644 --- a/etc/kayobe/overcloud-dib.yml +++ b/etc/kayobe/overcloud-dib.yml @@ -70,7 +70,7 @@ overcloud_dib_host_packages_extra: # elements. See stackhpc.os-images role for usage. Default is empty. overcloud_dib_git_elements_extra: - repo: "https://github.com/stackhpc/stackhpc-image-elements" - local: "{{ source_checkout_path }}/stackhpc-image-elememts" + local: "{{ source_checkout_path }}/stackhpc-image-elements" version: "v1.4.0" elements_path: "elements" From 75ba2f1cd2d74afe5573c311892b606c055c8412 Mon Sep 17 00:00:00 2001 From: Scott Davidson Date: Wed, 30 Nov 2022 10:17:52 +0000 Subject: [PATCH 131/292] Enable running of additional ceph commands before cephadm deployment --- etc/kayobe/ansible/cephadm-commands.yml | 11 +++++++++++ etc/kayobe/ansible/cephadm.yml | 1 + 2 files changed, 12 insertions(+) create mode 100644 etc/kayobe/ansible/cephadm-commands.yml diff --git a/etc/kayobe/ansible/cephadm-commands.yml b/etc/kayobe/ansible/cephadm-commands.yml new file mode 100644 index 000000000..311f55fd2 --- /dev/null +++ b/etc/kayobe/ansible/cephadm-commands.yml @@ -0,0 +1,11 @@ +--- +- name: Ensure additional Ceph commands are run + gather_facts: false + hosts: mons + become: true + tags: + - cephadm + - cephadm-commands + tasks: + - import_role: + name: stackhpc.cephadm.commands \ No newline at end of file diff --git a/etc/kayobe/ansible/cephadm.yml b/etc/kayobe/ansible/cephadm.yml index 96c667f62..48b0c126d 100644 --- a/etc/kayobe/ansible/cephadm.yml +++ b/etc/kayobe/ansible/cephadm.yml @@ -1,5 +1,6 @@ --- # Deploy Ceph via Cephadm. Create EC profiles, CRUSH rules, pools and keys. +- import_playbook: cephadm-commands.yml - import_playbook: cephadm-deploy.yml - import_playbook: cephadm-ec-profiles.yml - import_playbook: cephadm-crush-rules.yml From e26f1a220a62dd340aa53ab80ce98f3d4a27e4d5 Mon Sep 17 00:00:00 2001 From: Scott Davidson Date: Fri, 9 Dec 2022 09:04:36 +0000 Subject: [PATCH 132/292] Fix playbook order --- etc/kayobe/ansible/cephadm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/cephadm.yml b/etc/kayobe/ansible/cephadm.yml index 48b0c126d..1cb4d56e9 100644 --- a/etc/kayobe/ansible/cephadm.yml +++ b/etc/kayobe/ansible/cephadm.yml @@ -1,7 +1,7 @@ --- # Deploy Ceph via Cephadm. Create EC profiles, CRUSH rules, pools and keys. -- import_playbook: cephadm-commands.yml - import_playbook: cephadm-deploy.yml +- import_playbook: cephadm-commands.yml - import_playbook: cephadm-ec-profiles.yml - import_playbook: cephadm-crush-rules.yml - import_playbook: cephadm-pools.yml From e8f77680cfcbdf32985be5b3612225c9c61ecee7 Mon Sep 17 00:00:00 2001 From: Scott Davidson Date: Wed, 30 Nov 2022 10:01:09 +0000 Subject: [PATCH 133/292] Bypass pulp on seed hypervisor and seed vm This avoids the problem of trying to install packages from the local pulp server on the seed VM before that server exists. --- etc/kayobe/dnf.yml | 16 +++++++++++++++- .../group_vars/overcloud/stackhpc-dnf-repos | 5 +++++ etc/kayobe/kolla/globals.yml | 8 ++++++-- 3 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 etc/kayobe/inventory/group_vars/overcloud/stackhpc-dnf-repos diff --git a/etc/kayobe/dnf.yml b/etc/kayobe/dnf.yml index c63276a28..375fa30ff 100644 --- a/etc/kayobe/dnf.yml +++ b/etc/kayobe/dnf.yml @@ -41,7 +41,7 @@ # file: myrepo # gpgkey: http://gpgkey # gpgcheck: yes -dnf_custom_repos: "{{ dnf_custom_repos_el8 | combine(lookup('vars', 'dnf_custom_repos_' ~ ansible_facts.distribution | lower )) }}" +stackhpc_dnf_repos: "{{ dnf_custom_repos_el8 | combine(lookup('vars', 'dnf_custom_repos_' ~ ansible_facts.distribution | lower )) }}" # Custom repositories shared between all RHEL derivatives. dnf_custom_repos_el8: @@ -59,6 +59,13 @@ dnf_custom_repos_el8: file: epel-modular gpgkey: "{{ dnf_epel_gpg_key_url }}" gpgcheck: yes + docker: + baseurl: "{{ stackhpc_repo_docker_url }}" + description: "Package repository for installing docker" + enabled: "{{ dnf_enable_docker | bool }}" + file: docker + gpgkey: "{{ dnf_docker_gpg_key_url }}" + gpgcheck: yes # CentOS Stream 8 specific repositories dnf_custom_repos_centos: @@ -112,6 +119,13 @@ dnf_epel_gpg_key_url: "https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-8" # systems only. Default value is 'true'. dnf_install_epel: false +# Whether to enable docker dnf repo in stackhpc_dnf_repos +dnf_enable_docker: true + +#URL of docker repo GPG key +dnf_docker_gpg_key_url: "https://download.docker.com/linux/centos/gpg" + + ############################################################################### # DNF Automatic configuration. diff --git a/etc/kayobe/inventory/group_vars/overcloud/stackhpc-dnf-repos b/etc/kayobe/inventory/group_vars/overcloud/stackhpc-dnf-repos new file mode 100644 index 000000000..c4f70a863 --- /dev/null +++ b/etc/kayobe/inventory/group_vars/overcloud/stackhpc-dnf-repos @@ -0,0 +1,5 @@ +# Only use local pulp mirrors for overcloud hosts +# to avoid situations where the seed-hypervisor +# tries to use a local pulp repo on the seed VM +# before the seed vm has been provisioned +dnf_custom_repos: "{{ stackhpc_dnf_repos }}" diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 774da38ba..de970ea16 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -1,7 +1,11 @@ # yamllint disable-file --- -docker_yum_baseurl: "{{ stackhpc_repo_docker_url }}" -docker_yum_gpgkey: "https://download.docker.com/linux/centos/gpg" + +# To work around issue of trying to install docker from +# empty pulp server, use upstream docker dnf repo on +# non-overcloud hosts +enable_docker_repo: {% raw %}"{{ 'overcloud' not in group_names }}"{% endraw %} + {% if kolla_base_distro == 'centos' %} bifrost_tag: xena-20221128T101757 From 46f98c80b51d2275f5a25b4e86a6bc3e39fffb6b Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 12 Dec 2022 16:34:57 +0000 Subject: [PATCH 134/292] Make pep8 happy --- etc/kayobe/ansible/cephadm-commands.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/cephadm-commands.yml b/etc/kayobe/ansible/cephadm-commands.yml index 311f55fd2..32742a343 100644 --- a/etc/kayobe/ansible/cephadm-commands.yml +++ b/etc/kayobe/ansible/cephadm-commands.yml @@ -8,4 +8,4 @@ - cephadm-commands tasks: - import_role: - name: stackhpc.cephadm.commands \ No newline at end of file + name: stackhpc.cephadm.commands From a96ee4fc0000d4bc6f93ec569d74cb358a62872f Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 12 Dec 2022 19:45:21 +0000 Subject: [PATCH 135/292] Ubuntu: always use kolla-ansible to install docker-ce repo Currently Release Train does not provide Ubuntu package repos. --- etc/kayobe/kolla/globals.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index de970ea16..b30ddd013 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -4,7 +4,7 @@ # To work around issue of trying to install docker from # empty pulp server, use upstream docker dnf repo on # non-overcloud hosts -enable_docker_repo: {% raw %}"{{ 'overcloud' not in group_names }}"{% endraw %} +enable_docker_repo: {% raw %}"{{ 'overcloud' not in group_names or ansible_facts.os_family == 'Debian' }}"{% endraw %} {% if kolla_base_distro == 'centos' %} From fc547a3f6e88c5dc49657fd2860de4e8efd40e31 Mon Sep 17 00:00:00 2001 From: Matt Anson Date: Wed, 14 Dec 2022 07:47:58 +0000 Subject: [PATCH 136/292] Add Ubuntu image tags --- etc/kayobe/kolla.yml | 2 +- etc/kayobe/kolla/globals.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 61b7253e7..59fc4d50e 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -96,7 +96,7 @@ kolla_docker_registry_password: "{{ stackhpc_docker_registry_password }}" # Kolla OpenStack release version. This should be a Docker image tag. # Default is {{ openstack_release }}. -kolla_openstack_release: xena-20221010T103511 +kolla_openstack_release: "{% if kolla_base_distro == 'centos' %}xena-20221010T103511{% else %}xena-20221213T204703{% endif %}" # Docker tag applied to built container images. Default is # {{ kolla_openstack_release }}. diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 774da38ba..c8fc2413a 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -5,6 +5,8 @@ docker_yum_gpgkey: "https://download.docker.com/linux/centos/gpg" {% if kolla_base_distro == 'centos' %} bifrost_tag: xena-20221128T101757 +{% else %} +bifrost_tag: xena-20221213T224057 {% endif %} ############################################################################# From 9773d874c5dca4da275407c7ba966ffce3985511 Mon Sep 17 00:00:00 2001 From: Matt Anson Date: Wed, 14 Dec 2022 07:48:22 +0000 Subject: [PATCH 137/292] Enable Ubuntu AIO CI --- .github/workflows/stackhpc-pull-request.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/stackhpc-pull-request.yml b/.github/workflows/stackhpc-pull-request.yml index 4b027f44f..a534d413d 100644 --- a/.github/workflows/stackhpc-pull-request.yml +++ b/.github/workflows/stackhpc-pull-request.yml @@ -107,8 +107,7 @@ jobs: vm_image: Ubuntu-20.04 OS_CLOUD: sms-lab-release secrets: inherit - # NOTE: Ubuntu images not available yet. - if: false && github.repository == 'stackhpc/stackhpc-kayobe-config' + if: github.repository == 'stackhpc/stackhpc-kayobe-config' all-in-one-ubuntu-ovn: name: aio (Ubuntu OVN) @@ -122,5 +121,4 @@ jobs: vm_image: Ubuntu-20.04 OS_CLOUD: sms-lab-release secrets: inherit - # NOTE: Ubuntu images not available yet. - if: false && github.repository == 'stackhpc/stackhpc-kayobe-config' + if: github.repository == 'stackhpc/stackhpc-kayobe-config' From 6791190cd311974fa733ed7213c1d4fa7cf0ec28 Mon Sep 17 00:00:00 2001 From: Dawud M <7688823+technowhizz@users.noreply.github.com> Date: Fri, 18 Nov 2022 17:05:51 +0000 Subject: [PATCH 138/292] Add SMART Monitoring with dash and alerts Enabled Textfile collector in node exporter in kolla/globals.yml Added smartmon script as is from the prometheus-community github and then removed NVME support from this script in favour of using the nvme-cli script, which has also been added in. This is because the nvme-cli script provides better metrics than the smartmon script does. The script also adds the serial number of the disk as a label to all SMART metrics. Added a Kayobe custom playbook to easily deploy the script and associated cron job. This playbook installs smartmontool and nvmecli then copies these over to the hosts and sets up a cronjob which runs the scripts and stores the metrics in the docker volume for node exporter. The playbook changes the way the metrics are saved to a file by making use of the mv command as it is atomic. This was needed as at times prometheus would read a partially completed file. Added a prometheus alert to alert when a drive is reported as not healthy for more than 10 minutes. Added a Grafana dashboard to display the number of healthy and unhealthy drives reported in prometheus. (cherry picked from commit d83ecde1c8f38fe79da70041c637140c58f9dc9b) Add docs for SMART Monitoring (cherry picked from commit 595429ad15b77ee9777899e2593c41431ce37a82) Update doc/source/configuration/monitoring.rst Fix kayobe command Co-authored-by: Will Szumski (cherry picked from commit 9a5fc53c05de12b30dd360acfdcb023aedb84491) Update doc/source/configuration/monitoring.rst Fix Spelling Co-authored-by: Will Szumski (cherry picked from commit ef25d6f4d556809cfdc4a0be42300cd1249fdde1) Add release note (cherry picked from commit 3d4d01117dce76d668ac351a61f4843efe72ca0c) Amend docs and add release note (cherry picked from commit b6cb511d911f031de4e2b3c08fa5afb95e0f6d9a) Move SMART prometheus alert to own file (cherry picked from commit b353fd32c4799f325d1a5a0c8476690a12e30ff2) Fix typo (cherry picked from commit 611f2fb891802f2ff4d7248bcd7363acd244154c) fixup --- doc/source/configuration/monitoring.rst | 46 +- etc/kayobe/ansible/scripts/nvmemon.sh | 97 ++++ etc/kayobe/ansible/scripts/smartmon.sh | 202 +++++++ etc/kayobe/ansible/smartmon-tools.yml | 43 ++ .../openstack/hardware_overview.json | 543 ++++++++++++++++++ .../kolla/config/prometheus/smart.rules | 12 + etc/kayobe/kolla/globals.yml | 7 + .../notes/smart-mon-db8fa642c3af74b1.yaml | 4 + 8 files changed, 952 insertions(+), 2 deletions(-) create mode 100644 etc/kayobe/ansible/scripts/nvmemon.sh create mode 100644 etc/kayobe/ansible/scripts/smartmon.sh create mode 100644 etc/kayobe/ansible/smartmon-tools.yml create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json create mode 100644 etc/kayobe/kolla/config/prometheus/smart.rules create mode 100644 releasenotes/notes/smart-mon-db8fa642c3af74b1.yaml diff --git a/doc/source/configuration/monitoring.rst b/doc/source/configuration/monitoring.rst index 7e53629f1..f358ea084 100644 --- a/doc/source/configuration/monitoring.rst +++ b/doc/source/configuration/monitoring.rst @@ -1,4 +1,7 @@ -======================== +=========== +Monitoring +=========== + Monitoring Configuration ======================== @@ -10,4 +13,43 @@ The configuration options can be found in ``etc/kayobe/stackhpc-monitoring.yml``: .. literalinclude:: ../../../etc/kayobe/stackhpc-monitoring.yml - :language: yaml \ No newline at end of file + :language: yaml + +SMART Drive Monitoring +======================= + +StackHPC kayobe config also includes drive monitoring for spinning disks and +NVME's. + +By default, node exporter doesn't provide SMART metrics, hence we make use +of 2 scripts (one for NVME’s and one for spinning drives), which are run by +a cronjob, to output the metrics and we use node exporter's Textfile collector +to report the metrics output by the scripts to Prometheus. These metrics can +then be visualised in Grafana with the bundled dashboard. + +After pulling in the latest changes into your local kayobe config, reconfigure +Prometheus and Grafana + +.. code-block:: console + + kayobe overcloud service reconfigure -kt grafana,prometheus + +(Note: If you run into an error when reconfiguring Grafana, it could be due to +`this `__ bug and at +present, the workaround is to go into each node running Grafana and manually +restart the process with ``docker restart grafana`` and then try the reconfigure +command again.)  + +Once the reconfigure has completed you can now run the custom playbook which +copies over the scripts and sets up the cron jobs to start SMART monitoring +on the overcloud hosts: + +.. code-block:: console + + (kayobe) [stack@node ~]$ cd etc/kayobe + (kayobe) [stack@node kayobe]$ kayobe playbook run ansible/smartmontools.yml + +SMART reporting should now be enabled along with a Prometheus alert for +unhealthy disks and a Grafana dashboard called ``Hardware Overview``.  + + diff --git a/etc/kayobe/ansible/scripts/nvmemon.sh b/etc/kayobe/ansible/scripts/nvmemon.sh new file mode 100644 index 000000000..9ab727b0b --- /dev/null +++ b/etc/kayobe/ansible/scripts/nvmemon.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +set -eu + +# Dependencies: nvme-cli, jq (packages) +# Based on code from +# - https://github.com/prometheus/node_exporter/blob/master/text_collector_examples/smartmon.sh +# - https://github.com/prometheus/node_exporter/blob/master/text_collector_examples/mellanox_hca_temp +# - https://github.com/vorlon/check_nvme/blob/master/check_nvme.sh +# +# Author: Henk + +# Check if we are root +if [ "$EUID" -ne 0 ]; then + echo "${0##*/}: Please run as root!" >&2 + exit 1 +fi + +# Check if programs are installed +if ! command -v nvme >/dev/null 2>&1; then + echo "${0##*/}: nvme is not installed. Aborting." >&2 + exit 1 +fi + +output_format_awk="$( + cat <<'OUTPUTAWK' +BEGIN { v = "" } +v != $1 { + print "# HELP nvme_" $1 " SMART metric " $1; + if ($1 ~ /_total$/) + print "# TYPE nvme_" $1 " counter"; + else + print "# TYPE nvme_" $1 " gauge"; + v = $1 +} +{print "nvme_" $0} +OUTPUTAWK +)" + +format_output() { + sort | awk -F'{' "${output_format_awk}" +} + +# Get the nvme-cli version +nvme_version="$(nvme version | awk '$1 == "nvme" {print $3}')" +echo "nvmecli{version=\"${nvme_version}\"} 1" | format_output + +# Get devices +device_list="$(nvme list -o json | jq -r '.Devices | .[].DevicePath')" + +# Loop through the NVMe devices +for device in ${device_list}; do + json_check="$(nvme smart-log -o json "${device}")" + disk="${device##*/}" + + # The temperature value in JSON is in Kelvin, we want Celsius + value_temperature="$(echo "$json_check" | jq '.temperature - 273')" + echo "temperature_celsius{device=\"${disk}\"} ${value_temperature}" + + value_available_spare="$(echo "$json_check" | jq '.avail_spare / 100')" + echo "available_spare_ratio{device=\"${disk}\"} ${value_available_spare}" + + value_available_spare_threshold="$(echo "$json_check" | jq '.spare_thresh / 100')" + echo "available_spare_threshold_ratio{device=\"${disk}\"} ${value_available_spare_threshold}" + + value_percentage_used="$(echo "$json_check" | jq '.percent_used / 100')" + echo "percentage_used_ratio{device=\"${disk}\"} ${value_percentage_used}" + + value_critical_warning="$(echo "$json_check" | jq '.critical_warning')" + echo "critical_warning_total{device=\"${disk}\"} ${value_critical_warning}" + + value_media_errors="$(echo "$json_check" | jq '.media_errors')" + echo "media_errors_total{device=\"${disk}\"} ${value_media_errors}" + + value_num_err_log_entries="$(echo "$json_check" | jq '.num_err_log_entries')" + echo "num_err_log_entries_total{device=\"${disk}\"} ${value_num_err_log_entries}" + + value_power_cycles="$(echo "$json_check" | jq '.power_cycles')" + echo "power_cycles_total{device=\"${disk}\"} ${value_power_cycles}" + + value_power_on_hours="$(echo "$json_check" | jq '.power_on_hours')" + echo "power_on_hours_total{device=\"${disk}\"} ${value_power_on_hours}" + + value_controller_busy_time="$(echo "$json_check" | jq '.controller_busy_time')" + echo "controller_busy_time_seconds{device=\"${disk}\"} ${value_controller_busy_time}" + + value_data_units_written="$(echo "$json_check" | jq '.data_units_written')" + echo "data_units_written_total{device=\"${disk}\"} ${value_data_units_written}" + + value_data_units_read="$(echo "$json_check" | jq '.data_units_read')" + echo "data_units_read_total{device=\"${disk}\"} ${value_data_units_read}" + + value_host_read_commands="$(echo "$json_check" | jq '.host_read_commands')" + echo "host_read_commands_total{device=\"${disk}\"} ${value_host_read_commands}" + + value_host_write_commands="$(echo "$json_check" | jq '.host_write_commands')" + echo "host_write_commands_total{device=\"${disk}\"} ${value_host_write_commands}" +done | format_output diff --git a/etc/kayobe/ansible/scripts/smartmon.sh b/etc/kayobe/ansible/scripts/smartmon.sh new file mode 100644 index 000000000..bcac8b8b3 --- /dev/null +++ b/etc/kayobe/ansible/scripts/smartmon.sh @@ -0,0 +1,202 @@ +#!/bin/bash +# Script informed by the collectd monitoring script for smartmontools (using smartctl) +# by Samuel B. (c) 2012 +# source at: http://devel.dob.sk/collectd-scripts/ + +# TODO: This probably needs to be a little more complex. The raw numbers can have more +# data in them than you'd think. +# http://arstechnica.com/civis/viewtopic.php?p=22062211 + +# Formatting done via shfmt -i 2 +# https://github.com/mvdan/sh + +parse_smartctl_attributes_awk="$( + cat <<'SMARTCTLAWK' +$1 ~ /^ *[0-9]+$/ && $2 ~ /^[a-zA-Z0-9_-]+$/ { + gsub(/-/, "_"); + printf "%s_value{%s,smart_id=\"%s\"} %d\n", $2, labels, $1, $4 + printf "%s_worst{%s,smart_id=\"%s\"} %d\n", $2, labels, $1, $5 + printf "%s_threshold{%s,smart_id=\"%s\"} %d\n", $2, labels, $1, $6 + printf "%s_raw_value{%s,smart_id=\"%s\"} %e\n", $2, labels, $1, $10 +} +SMARTCTLAWK +)" + +smartmon_attrs="$( + cat <<'SMARTMONATTRS' +airflow_temperature_cel +command_timeout +current_pending_sector +end_to_end_error +erase_fail_count +g_sense_error_rate +hardware_ecc_recovered +host_reads_32mib +host_reads_mib +host_writes_32mib +host_writes_mib +load_cycle_count +media_wearout_indicator +nand_writes_1gib +offline_uncorrectable +power_cycle_count +power_on_hours +program_fail_cnt_total +program_fail_count +raw_read_error_rate +reallocated_event_count +reallocated_sector_ct +reported_uncorrect +runtime_bad_block +sata_downshift_count +seek_error_rate +spin_retry_count +spin_up_time +start_stop_count +temperature_case +temperature_celsius +temperature_internal +total_lbas_read +total_lbas_written +udma_crc_error_count +unsafe_shutdown_count +unused_rsvd_blk_cnt_tot +wear_leveling_count +workld_host_reads_perc +workld_media_wear_indic +workload_minutes +SMARTMONATTRS +)" +smartmon_attrs="$(echo "${smartmon_attrs}" | xargs | tr ' ' '|')" + +parse_smartctl_attributes() { + local disk="$1" + local disk_type="$2" + local serial="$3" + local labels="disk=\"${disk}\",type=\"${disk_type}\",serial_number=\"${serial}\"" + sed 's/^ \+//g' | + awk -v labels="${labels}" "${parse_smartctl_attributes_awk}" 2>/dev/null | + tr '[:upper:]' '[:lower:]' | + grep -E "(${smartmon_attrs})" +} + +parse_smartctl_scsi_attributes() { + local disk="$1" + local disk_type="$2" + local serial="$3" + local labels="disk=\"${disk}\",type=\"${disk_type}\",serial_number=\"${serial}\"" + while read -r line; do + attr_type="$(echo "${line}" | tr '=' ':' | cut -f1 -d: | sed 's/^ \+//g' | tr ' ' '_')" + attr_value="$(echo "${line}" | tr '=' ':' | cut -f2 -d: | sed 's/^ \+//g')" + case "${attr_type}" in + number_of_hours_powered_up_) power_on="$(echo "${attr_value}" | awk '{ printf "%e\n", $1 }')" ;; + Current_Drive_Temperature) temp_cel="$(echo "${attr_value}" | cut -f1 -d' ' | awk '{ printf "%e\n", $1 }')" ;; + Blocks_sent_to_initiator_) lbas_read="$(echo "${attr_value}" | awk '{ printf "%e\n", $1 }')" ;; + Blocks_received_from_initiator_) lbas_written="$(echo "${attr_value}" | awk '{ printf "%e\n", $1 }')" ;; + Accumulated_start-stop_cycles) power_cycle="$(echo "${attr_value}" | awk '{ printf "%e\n", $1 }')" ;; + Elements_in_grown_defect_list) grown_defects="$(echo "${attr_value}" | awk '{ printf "%e\n", $1 }')" ;; + esac + done + [ -n "$power_on" ] && echo "power_on_hours_raw_value{${labels},smart_id=\"9\"} ${power_on}" + [ -n "$temp_cel" ] && echo "temperature_celsius_raw_value{${labels},smart_id=\"194\"} ${temp_cel}" + [ -n "$lbas_read" ] && echo "total_lbas_read_raw_value{${labels},smart_id=\"242\"} ${lbas_read}" + [ -n "$lbas_written" ] && echo "total_lbas_written_raw_value{${labels},smart_id=\"241\"} ${lbas_written}" + [ -n "$power_cycle" ] && echo "power_cycle_count_raw_value{${labels},smart_id=\"12\"} ${power_cycle}" + [ -n "$grown_defects" ] && echo "grown_defects_count_raw_value{${labels},smart_id=\"-1\"} ${grown_defects}" +} + +parse_smartctl_info() { + local -i smart_available=0 smart_enabled=0 smart_healthy= + local disk="$1" disk_type="$2" + local model_family='' device_model='' serial_number='' fw_version='' vendor='' product='' revision='' lun_id='' + while read -r line; do + info_type="$(echo "${line}" | cut -f1 -d: | tr ' ' '_')" + info_value="$(echo "${line}" | cut -f2- -d: | sed 's/^ \+//g' | sed 's/"/\\"/')" + case "${info_type}" in + Model_Family) model_family="${info_value}" ;; + Device_Model) device_model="${info_value}" ;; + Serial_Number) serial_number="${info_value}" ;; + Firmware_Version) fw_version="${info_value}" ;; + Vendor) vendor="${info_value}" ;; + Product) product="${info_value}" ;; + Revision) revision="${info_value}" ;; + Logical_Unit_id) lun_id="${info_value}" ;; + esac + if [[ "${info_type}" == 'SMART_support_is' ]]; then + case "${info_value:0:7}" in + Enabled) smart_available=1; smart_enabled=1 ;; + Availab) smart_available=1; smart_enabled=0 ;; + Unavail) smart_available=0; smart_enabled=0 ;; + esac + fi + if [[ "${info_type}" == 'SMART_overall-health_self-assessment_test_result' ]]; then + case "${info_value:0:6}" in + PASSED) smart_healthy=1 ;; + *) smart_healthy=0 ;; + esac + elif [[ "${info_type}" == 'SMART_Health_Status' ]]; then + case "${info_value:0:2}" in + OK) smart_healthy=1 ;; + *) smart_healthy=0 ;; + esac + fi + done + echo "device_info{disk=\"${disk}\",type=\"${disk_type}\",vendor=\"${vendor}\",product=\"${product}\",revision=\"${revision}\",lun_id=\"${lun_id}\",model_family=\"${model_family}\",device_model=\"${device_model}\",serial_number=\"${serial_number}\",firmware_version=\"${fw_version}\"} 1" + echo "device_smart_available{disk=\"${disk}\",type=\"${disk_type}\",serial_number=\"${serial_number}\"} ${smart_available}" + [[ "${smart_available}" == "1" ]] && echo "device_smart_enabled{disk=\"${disk}\",type=\"${disk_type}\",serial_number=\"${serial_number}\"} ${smart_enabled}" + [[ "${smart_available}" == "1" ]] && [[ "${smart_healthy}" != "" ]] && echo "device_smart_healthy{disk=\"${disk}\",type=\"${disk_type}\",serial_number=\"${serial_number}\"} ${smart_healthy}" +} + +output_format_awk="$( + cat <<'OUTPUTAWK' +BEGIN { v = "" } +v != $1 { + print "# HELP smartmon_" $1 " SMART metric " $1; + print "# TYPE smartmon_" $1 " gauge"; + v = $1 +} +{print "smartmon_" $0} +OUTPUTAWK +)" + +format_output() { + sort | + awk -F'{' "${output_format_awk}" +} + +smartctl_version="$(/usr/sbin/smartctl -V | head -n1 | awk '$1 == "smartctl" {print $2}')" + +echo "smartctl_version{version=\"${smartctl_version}\"} 1" | format_output + +if [[ "$(expr "${smartctl_version}" : '\([0-9]*\)\..*')" -lt 6 ]]; then + exit +fi + +device_list="$(/usr/sbin/smartctl --scan-open | awk '/^\/dev/{print $1 "|" $3}')" + +for device in ${device_list}; do + disk="$(echo "${device}" | cut -f1 -d'|')" + type="$(echo "${device}" | cut -f2 -d'|')" + # Use REGEX to extract the serial number from the parsed information and save that to a variable + serial_number="$(/usr/sbin/smartctl -i -H -d "${type}" "${disk}" | parse_smartctl_info "${disk}" "${type}"| sed -E ':a;N;$!ba;s/.*serial_number=\"([^"]+)\".*/\1/g' | sed -E 's/^device_info\{.*//g')" + active=1 + echo "smartctl_run{disk=\"${disk}\",type=\"${type}\"}" "$(TZ=UTC date '+%s')" + # Check if the device is in a low-power mode + /usr/sbin/smartctl -n standby -d "${type}" "${disk}" > /dev/null || active=0 + echo "device_active{disk=\"${disk}\",type=\"${type}\"}" "${active}" + # Skip further metrics to prevent the disk from spinning up + test ${active} -eq 0 && continue + # Get the SMART information and health + /usr/sbin/smartctl -i -H -d "${type}" "${disk}" | parse_smartctl_info "${disk}" "${type}" + # Get the SMART attributes + case ${type} in + sat) /usr/sbin/smartctl -A -d "${type}" "${disk}" | parse_smartctl_attributes "${disk}" "${type}" "${serial_number}" ;; + sat+megaraid*) /usr/sbin/smartctl -A -d "${type}" "${disk}" | parse_smartctl_attributes "${disk}" "${type}" "${serial_number}" ;; + scsi) /usr/sbin/smartctl -A -d "${type}" "${disk}" | parse_smartctl_scsi_attributes "${disk}" "${type}" "${serial_number}" ;; + megaraid*) /usr/sbin/smartctl -A -d "${type}" "${disk}" | parse_smartctl_scsi_attributes "${disk}" "${type}" "${serial_number}" ;; + *) + (>&2 echo "disk type is not sat, scsi or megaraid but ${type}") + exit + ;; + esac +done | format_output \ No newline at end of file diff --git a/etc/kayobe/ansible/smartmon-tools.yml b/etc/kayobe/ansible/smartmon-tools.yml new file mode 100644 index 000000000..6b275c264 --- /dev/null +++ b/etc/kayobe/ansible/smartmon-tools.yml @@ -0,0 +1,43 @@ +--- +- hosts: overcloud + + tasks: + - name: Ensure smartmon-tools and nvme-cli is installed + package: + name: + - smartmontools + - nvme-cli + - jq + state: present + become: true + + - name: Copy smartmon.sh and nvmemon.sh from scripts folder + copy: + src: "scripts/{{ item }}" + dest: /usr/local/bin/ + owner: 'root' + group: 'root' + mode: '0700' + loop: + - smartmon.sh + - nvmemon.sh + become: yes + + - name: Set PATH Variable for cron + cron: + name: PATH + user: root + env: yes + job: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + become: yes + + - name: Schedule cronjob to run both scripts every 5 minutes and save output to file + cron: + name: "SMART metrics for drive monitoring using {{ item }}" + user: root + minute: "*/5" + job: "/usr/local/bin/{{ item }}.sh > /var/lib/docker/volumes/textfile/_data/{{ item }}.prom.temp && mv /var/lib/docker/volumes/textfile/_data/{{ item }}.prom.temp /var/lib/docker/volumes/textfile/_data/{{ item }}.prom" + loop: + - smartmon + - nvmemon + become: yes diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json new file mode 100644 index 000000000..e4f78aee3 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json @@ -0,0 +1,543 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of healthy drives", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 0 + }, + "hideTimeOverride": false, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.1.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(smartmon_device_smart_healthy > 0)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Healthy Drives", + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of healthy drives", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 0 + }, + "hideTimeOverride": false, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.1.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(smartmon_device_smart_healthy < 1) ", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Unhealthy Drives", + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of healthy drives", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 0 + }, + "hideTimeOverride": false, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.1.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(smartmon_device_smart_healthy)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total Drives", + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "align": "center", + "displayMode": "auto", + "filterable": false, + "inspect": false + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Failed" + }, + "1": { + "color": "dark-green", + "index": 0, + "text": "Ok" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "custom.displayMode", + "value": "color-background-solid" + }, + { + "id": "custom.width" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "type" + }, + "properties": [ + { + "id": "custom.width", + "value": 153 + }, + { + "id": "displayName", + "value": "Type" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "serial_number" + }, + "properties": [ + { + "id": "custom.width", + "value": 208 + }, + { + "id": "displayName", + "value": "Serial Number" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "disk" + }, + "properties": [ + { + "id": "custom.width", + "value": 146 + }, + { + "id": "displayName", + "value": "Disk" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.width", + "value": 203 + }, + { + "id": "displayName", + "value": "Hostname" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Type" + }, + "properties": [ + { + "id": "custom.width" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Serial Number" + }, + "properties": [ + { + "id": "custom.width" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hostname" + }, + "properties": [ + { + "id": "custom.width" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Disk" + }, + "properties": [ + { + "id": "custom.width" + } + ] + } + ] + }, + "gridPos": { + "h": 13, + "w": 18, + "x": 0, + "y": 7 + }, + "id": 2, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.1.2", + "targets": [ + { + "$$hashKey": "object:40", + "aggregation": "Last", + "alias": "Healthy", + "crit": 0, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 0, + "displayAliasType": "Warning / Critical", + "displayType": "Regular", + "displayValueWithAlias": "Never", + "editorMode": "code", + "exemplar": false, + "expr": "smartmon_device_smart_healthy", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "range": false, + "refId": "A", + "units": "none", + "valueHandler": "Number Threshold", + "warn": 0 + } + ], + "title": "Panel Title", + "transparent": true, + "type": "table" + } + ], + "refresh": false, + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "filters": [], + "hide": 0, + "name": "Filters", + "skipUrlSync": false, + "type": "adhoc" + }, + { + "current": { + "selected": true, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Hardware Overview", + "uid": "TCN51Y25P", + "version": 1, + "weekStart": "" +} +{% endraw %} \ No newline at end of file diff --git a/etc/kayobe/kolla/config/prometheus/smart.rules b/etc/kayobe/kolla/config/prometheus/smart.rules new file mode 100644 index 000000000..0b6552598 --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/smart.rules @@ -0,0 +1,12 @@ +{% raw %} + +- alert: DiskSmartStatusUnhealthy + expr: smartmon_device_smart_healthy < 1 + for: 10m + labels: + severity: alert + annotations: + summary: "SMART monitor reports bad disk on (instance {{ $labels.instance }})" + description: "{{ $labels.instance }} is reporting unhealthy for the disk at {{ $labels.disk }}. Disk serial number is: {{ $labels.serial_number }}" + +{% endraw %} diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index b30ddd013..3081a2e5e 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -17,4 +17,11 @@ bifrost_tag: xena-20221128T101757 es_heap_size: 8g prometheus_cmdline_extras: "--storage.tsdb.retention.time=30d" +# Additional command line flags for node exporter to enable texfile collector for disk metrics and create textfile docker volume +prometheus_node_exporter_extra_volumes: + - "textfile:/var/lib/node_exporter/textfile_collector" +prometheus_node_exporter_cmdline_extras: "--collector.textfile.directory=/var/lib/node_exporter/textfile_collector" + + ############################################################################# + diff --git a/releasenotes/notes/smart-mon-db8fa642c3af74b1.yaml b/releasenotes/notes/smart-mon-db8fa642c3af74b1.yaml new file mode 100644 index 000000000..feaec4dbe --- /dev/null +++ b/releasenotes/notes/smart-mon-db8fa642c3af74b1.yaml @@ -0,0 +1,4 @@ +--- + +features: + - Enables SMART monitoring. Manual action is required, please see the monitoring documentation for the procedure. From 5a7d68d3d41d33f5b9449a1d30f2e33a82acdeb3 Mon Sep 17 00:00:00 2001 From: Matt Anson Date: Fri, 11 Nov 2022 09:17:02 +0000 Subject: [PATCH 139/292] Increase job timeout for kolla image build GHA (cherry picked from commit 807e935dd4dc622ffbaf0a1df2a86c93bacf8b4c) --- .github/workflows/stackhpc-container-image-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml index 65428c039..f69626d27 100644 --- a/.github/workflows/stackhpc-container-image-build.yml +++ b/.github/workflows/stackhpc-container-image-build.yml @@ -34,6 +34,7 @@ jobs: name: Build Kolla container images if: github.repository == 'stackhpc/stackhpc-kayobe-config' runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder] + timeout-minutes: 720 steps: - uses: actions/checkout@v3 with: From a089c3ebcc1aecae7a1a508cf5cceec361da420b Mon Sep 17 00:00:00 2001 From: Dawud M <7688823+technowhizz@users.noreply.github.com> Date: Fri, 2 Dec 2022 16:04:05 +0000 Subject: [PATCH 140/292] Fix oom-killer graph Changes the oom-killer graph from a smoothed irate to a discrete delta function. Change-Id: I2e4a8576c628610409ade4aad2bd98754bec3860 (cherry picked from commit ef1a449034bb4501e333ae41910e8d77cb5b4ad8) --- .../grafana/dashboards/openstack/node_exporter_full.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_full.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_full.json index 08078c31d..66d630b8d 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_full.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_full.json @@ -4886,7 +4886,7 @@ "Total Swap": "#614D93", "VmallocUsed": "#EA6460" }, - "bars": false, + "bars": true, "dashLength": 10, "dashes": false, "datasource": { @@ -4921,7 +4921,7 @@ "total": false, "values": true }, - "lines": true, + "lines": false, "linewidth": 1, "links": [], "maxPerRow": 6, @@ -4940,9 +4940,9 @@ "steppedLine": false, "targets": [ { - "expr": "irate(node_vmstat_oom_kill{instance=\"$node\",job=\"$job\"}[5m])", + "expr": "max_over_time(node_vmstat_oom_kill{instance=\"$node\",job=\"$job\"}[$__rate_interval:]) - (min_over_time(node_vmstat_oom_kill{instance=\"$node\",job=\"$job\"}[$__rate_interval:]))", "format": "time_series", - "interval": "", + "interval": "30s", "intervalFactor": 2, "legendFormat": "oom killer invocations ", "refId": "A", From de1be4842e9e3291077a6bf80dbdf7369f74b39b Mon Sep 17 00:00:00 2001 From: Stig Telfer Date: Thu, 10 Nov 2022 11:05:00 +0000 Subject: [PATCH 141/292] Rephrase the match logic for interfaces monitored for package drops OVS bridge interfaces drop packets during normal operation. Change the regex to filter out interfaces that don't matter for packet drops. (cherry picked from commit 9c3f15a1f374d24d08a0bdac63886220247d2a4e) --- etc/kayobe/kolla/config/prometheus/ceph.rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/kolla/config/prometheus/ceph.rules b/etc/kayobe/kolla/config/prometheus/ceph.rules index 7df171501..52b9841a9 100644 --- a/etc/kayobe/kolla/config/prometheus/ceph.rules +++ b/etc/kayobe/kolla/config/prometheus/ceph.rules @@ -154,7 +154,7 @@ groups: # alert on nic packet errors and drops rates > 1 packet/s - alert: NetworkPacketsDropped - expr: irate(node_network_receive_drop_total{device=~"en.*|eth.*"}[5m]) + irate(node_network_transmit_drop_total{device=~"en.*|eth.*"}[5m]) > 1 + expr: irate(node_network_receive_drop_total{device!~"lo|br.*|.*-ovs|tap.*"}[5m]) + irate(node_network_transmit_drop_total{device!~"lo|br.*|.*-ovs|tap.*"}[5m]) > 1 labels: severity: warning annotations: From 73bf07968f62bcaf0b16813ffd4bd28cf19c3d65 Mon Sep 17 00:00:00 2001 From: Dawud M <7688823+technowhizz@users.noreply.github.com> Date: Tue, 29 Nov 2022 16:18:48 +0000 Subject: [PATCH 142/292] Add docs for SMART Monitoring --- doc/source/configuration/monitoring.rst | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/doc/source/configuration/monitoring.rst b/doc/source/configuration/monitoring.rst index f358ea084..2c8e253fb 100644 --- a/doc/source/configuration/monitoring.rst +++ b/doc/source/configuration/monitoring.rst @@ -1,6 +1,6 @@ -=========== +========== Monitoring -=========== +========== Monitoring Configuration ======================== @@ -16,7 +16,7 @@ The configuration options can be found in :language: yaml SMART Drive Monitoring -======================= +====================== StackHPC kayobe config also includes drive monitoring for spinning disks and NVME's. @@ -27,7 +27,7 @@ a cronjob, to output the metrics and we use node exporter's Textfile collector to report the metrics output by the scripts to Prometheus. These metrics can then be visualised in Grafana with the bundled dashboard. -After pulling in the latest changes into your local kayobe config, reconfigure +After pulling in the latest changes into your local kayobe config, reconfigure Prometheus and Grafana .. code-block:: console @@ -38,7 +38,7 @@ Prometheus and Grafana `this `__ bug and at present, the workaround is to go into each node running Grafana and manually restart the process with ``docker restart grafana`` and then try the reconfigure -command again.)  +command again.) Once the reconfigure has completed you can now run the custom playbook which copies over the scripts and sets up the cron jobs to start SMART monitoring @@ -50,6 +50,4 @@ on the overcloud hosts: (kayobe) [stack@node kayobe]$ kayobe playbook run ansible/smartmontools.yml SMART reporting should now be enabled along with a Prometheus alert for -unhealthy disks and a Grafana dashboard called ``Hardware Overview``.  - - +unhealthy disks and a Grafana dashboard called ``Hardware Overview``. From 2dcec472ce3db4cc5fd00df94df6dce194b1f9c3 Mon Sep 17 00:00:00 2001 From: Piotr Parczewski Date: Mon, 19 Dec 2022 10:54:21 +0100 Subject: [PATCH 143/292] Add note on enabling standard configuration --- doc/source/configuration/monitoring.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/source/configuration/monitoring.rst b/doc/source/configuration/monitoring.rst index 2c8e253fb..43ee5f4fc 100644 --- a/doc/source/configuration/monitoring.rst +++ b/doc/source/configuration/monitoring.rst @@ -15,6 +15,17 @@ The configuration options can be found in .. literalinclude:: ../../../etc/kayobe/stackhpc-monitoring.yml :language: yaml +In order to enable stock monitoring configuration within a particular +environment, create the following symbolic links: + +.. code-block:: console + + cd $KAYOBE_CONFIG_PATH + ln -s kolla/config/grafana/ environments/$KAYOBE_ENVIRONMENT/kolla/config/ + ln -s kolla/config/prometheus/ environments/$KAYOBE_ENVIRONMENT/kolla/config/ + +and commit them to the config repository. + SMART Drive Monitoring ====================== From 74b27c515a92a3b51cfe93a91b6372003c5d6ad3 Mon Sep 17 00:00:00 2001 From: Piotr Parczewski Date: Mon, 19 Dec 2022 16:09:38 +0100 Subject: [PATCH 144/292] Docs edit --- doc/source/configuration/monitoring.rst | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/doc/source/configuration/monitoring.rst b/doc/source/configuration/monitoring.rst index 43ee5f4fc..6045f3c29 100644 --- a/doc/source/configuration/monitoring.rst +++ b/doc/source/configuration/monitoring.rst @@ -5,9 +5,25 @@ Monitoring Monitoring Configuration ======================== -StackHPC kayobe config includes a reference monitoring stack based on -Prometheus. Whilst this often works out of the box, there are some tunables -which can be customised to adapt the configuration to a particular deployment. +StackHPC kayobe config includes a reference monitoring and alerting stack based +on Prometheus, Alertmanager, Grafana, Fluentd, Elasticsearch & Kibana. These +services by default come enabled and configured. Central Elasticsearch cluster +collects OpenStack logs, with an option to receive operating system logs too. +In order to enable this, execute custom playbook after deployment: + +.. code-block:: console + + cd $KAYOBE_CONFIG_PATH + kayobe playbook run ansible/rsyslog.yml + +`Prometheus `__ comes with a comprehensive set of +metrics gathered from enabled exporters; every exporter's data is visualised +by at least one `Grafana `__ dashboard. Standard set of +alerting rules is present as well. + +While the default configuration often works out of the box, there +are some tunables which can be customised to adapt the configuration to a +particular deployment's needs. The configuration options can be found in ``etc/kayobe/stackhpc-monitoring.yml``: From b5a0c2f0dcacf98fa06bcbfd48813107843438d6 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 12 Dec 2022 09:21:24 +0000 Subject: [PATCH 145/292] docs: Add current_series replacement and extlinks This allows us to reference the current release series in the documentation. --- doc/source/conf.py | 26 +++++++++++++++++++ doc/source/configuration/release-train.rst | 9 +++---- .../contributor/environments/ci-builder.rst | 2 +- doc/source/usage.rst | 5 ++-- 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index f7214fed0..aaeb990f5 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -26,12 +26,28 @@ # See the License for the specific language governing permissions and # limitations under the License. +# -- StackHPC Kayobe configuration -------------------------------------- +# Variables to override + +current_series = "xena" +branch = f"stackhpc/{current_series}" + +# Substitutions loader +rst_epilog = """ +.. |current_release| replace:: {current_release} +.. |current_release_git_branch_name| replace:: {current_release_git_branch_name} +""".format( # noqa: E501 + current_release_git_branch_name=branch, + current_release=current_series, +) + # -- General configuration ---------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ #'sphinx.ext.autodoc', + 'sphinx.ext.extlinks', #'sphinx.ext.intersphinx', 'sphinxcontrib.rsvgconverter', ] @@ -91,3 +107,13 @@ # Disable usage of xindy https://bugzilla.redhat.com/show_bug.cgi?id=1643664 latex_use_xindy = False +extlinks_projects = { + "kayobe", + "kolla", + "kolla-ansible", +} + +extlinks = { + f"{project}-doc": (f"https://docs.openstack.org/{project}/{current_series}/", "%s documentation") + for project in extlinks_projects +} diff --git a/doc/source/configuration/release-train.rst b/doc/source/configuration/release-train.rst index 04b583410..6e841b8f1 100644 --- a/doc/source/configuration/release-train.rst +++ b/doc/source/configuration/release-train.rst @@ -57,15 +57,14 @@ The distribution name for the environment should be configured as either Usage ===== -The local Pulp service will be deployed as a `Seed custom container -`__ +The local Pulp service will be deployed as a :kayobe-doc:`Seed custom container +` on next ``kayobe seed service deploy`` or ``kayobe seed service upgrade``. The following custom playbooks are provided in ``etc/kayobe/ansible/``: -See the Kayobe `custom playbook documentation -`__ -for information on how to run them. +See the Kayobe :kayobe-doc:`custom playbook documentation +` for information on how to run them. * ``pulp-repo-sync.yml``: Pull packages from Ark to the local Pulp. This will create a new repository version (snapshot) for each repository in the local diff --git a/doc/source/contributor/environments/ci-builder.rst b/doc/source/contributor/environments/ci-builder.rst index f146b2107..025c6bb65 100644 --- a/doc/source/contributor/environments/ci-builder.rst +++ b/doc/source/contributor/environments/ci-builder.rst @@ -105,7 +105,7 @@ At this point you are ready to build and push some container images. kayobe seed container image build --push kayobe overcloud container image build --push -The container images are tagged as ``xena-``. This Kayobe +The container images are tagged as |current_release|-. This Kayobe configuration includes a hook that writes the tag to ``~/kolla_tag``, since it is not always simple to determine which tag was last applied to built images. diff --git a/doc/source/usage.rst b/doc/source/usage.rst index f65b98af1..867d2983c 100644 --- a/doc/source/usage.rst +++ b/doc/source/usage.rst @@ -12,9 +12,8 @@ based on the upstream kayobe-config, with some opinionated configuration changes applied. Since this repository makes changes to the base configuration, it works best -when used with Kayobe's `multiple environments -`__ -feature. +when used with Kayobe's :kayobe-doc:`multiple environments +` feature. This configuration should be consumed using the `StackHPC Kayobe fork `__, which includes From ce21921dbe9a1d0e1d8d5071355910887e61247e Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 12 Dec 2022 16:02:46 +0000 Subject: [PATCH 146/292] docs: overview --- doc/source/index.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/source/index.rst b/doc/source/index.rst index a9906b35f..884629113 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -11,7 +11,16 @@ Welcome to StackHPC's Kayobe Config documentation! Overview ======== +This documentation covers the StackHPC Kayobe configuration. It is intended to +complement, rather than replace, the upstream :kayobe-doc:`Kayobe `, +:kolla-doc:`Kolla ` and :kolla-ansible-doc:`Kolla Ansible ` +documentation. +The configuration includes various things, such as: + +* Opinionated configuration +* Custom playbooks +* Continuous Integration (CI) workflows Contents ======== From 1b66e364832e596f448ff3474445b132a91f47e8 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 12 Dec 2022 19:55:04 +0000 Subject: [PATCH 147/292] Xena: batched release notes --- .../notes/xena-batch-bc1da4e4d0f6257e.yaml | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 releasenotes/notes/xena-batch-bc1da4e4d0f6257e.yaml diff --git a/releasenotes/notes/xena-batch-bc1da4e4d0f6257e.yaml b/releasenotes/notes/xena-batch-bc1da4e4d0f6257e.yaml new file mode 100644 index 000000000..1c9446dd5 --- /dev/null +++ b/releasenotes/notes/xena-batch-bc1da4e4d0f6257e.yaml @@ -0,0 +1,21 @@ +--- +features: + - | + Adds a custom playbook to run the `Anomoly Detection Visualiser (ADVise) + `_, ``advise-run.yml``. + - | + Adds a custom playbook to reset the RabbitMQ cluster and restart OpenStack + services that use it, ``rabbitmq-reset.yml``. + - | + Adds a custom playbook to configure swap, ``swap.yml``. + - | + Adds the `Kayobe Automation + `__ Git repository as a + submodule, and provides some basic configuration for it in an + ``.automation.conf`` directory. + - | + Adds support for deploying a Squid caching proxy as a custom container on + the seed. + - | + Enables Elasticsearch, Grafana, Kibana, Prometheus by default. Provides + standard dashboards for Grafana and alerting rules for Prometheus. From 3367a28a58847b63108117142887570403671592 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 12 Dec 2022 20:13:51 +0000 Subject: [PATCH 148/292] docs: Add release notes into main docs --- doc/requirements.txt | 2 +- doc/source/conf.py | 1 + doc/source/index.rst | 1 + doc/source/release-notes.rst | 6 ++++++ 4 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 doc/source/release-notes.rst diff --git a/doc/requirements.txt b/doc/requirements.txt index 842c709a9..650e0def0 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -2,6 +2,6 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -reno>=3.1.0 # Apache-2.0 +reno>=3.4.0 # Apache-2.0 sphinx>=4.2.0 # BSD sphinxcontrib-svg2pdfconverter>=0.1.0 # BSD diff --git a/doc/source/conf.py b/doc/source/conf.py index aaeb990f5..f52a21696 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -46,6 +46,7 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ + 'reno.sphinxext', #'sphinx.ext.autodoc', 'sphinx.ext.extlinks', #'sphinx.ext.intersphinx', diff --git a/doc/source/index.rst b/doc/source/index.rst index 884629113..a7f770bc4 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -30,4 +30,5 @@ Contents usage configuration/index + release-notes contributor/index diff --git a/doc/source/release-notes.rst b/doc/source/release-notes.rst new file mode 100644 index 000000000..e5cdefbdb --- /dev/null +++ b/doc/source/release-notes.rst @@ -0,0 +1,6 @@ +========================= +Xena Series Release Notes +========================= + +.. release-notes:: + :branch: stackhpc/xena From 80b5565e0c647e0171f4762dba3b277616fb61bf Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 12 Dec 2022 20:14:36 +0000 Subject: [PATCH 149/292] reno: Match on version-specific tag This seems to help reno collect the right notes. --- releasenotes/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/releasenotes/config.yaml b/releasenotes/config.yaml index 55f94b723..5dd9a157a 100644 --- a/releasenotes/config.yaml +++ b/releasenotes/config.yaml @@ -1,4 +1,4 @@ --- # This needs to be updated to the latest release. default_branch: stackhpc/xena -release_tag_re: stackhpc/\d+\.\d+\.\d+\.\d +release_tag_re: stackhpc/11\.\d+\.\d+\.\d From 63412e378e5fcd0016c39c22dc32c3174a9726a1 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 12 Dec 2022 16:03:05 +0000 Subject: [PATCH 150/292] docs: overcloud host image --- doc/source/configuration/host-images.rst | 66 ++++++++++++++++++++++++ doc/source/configuration/index.rst | 1 + 2 files changed, 67 insertions(+) create mode 100644 doc/source/configuration/host-images.rst diff --git a/doc/source/configuration/host-images.rst b/doc/source/configuration/host-images.rst new file mode 100644 index 000000000..cfa5fe678 --- /dev/null +++ b/doc/source/configuration/host-images.rst @@ -0,0 +1,66 @@ +.. _host-images: + +=========== +Host Images +=========== + +StackHPC Kayobe configuration provides configuration for some standard +overcloud host images, built using the :kayobe-doc:`overcloud DIB +` functionality of Kayobe. + +The overcloud DIB configuration is provided in +``etc/kayobe/stackhpc-overcloud-dib.yml``. It is not used by default, and must +be actively opted into. This can be done as follows: + +.. code-block:: yaml + :caption: ``etc/kayobe/overcloud-dib.yml`` + + overcloud_dib_build_host_images: true + + overcloud_dib_host_images: + - "{{ stackhpc_overcloud_dib_host_image }}" + +The image name is configured via ``stackhpc_overcloud_dib_name``, and is +``deployment_image`` by default. + +The list of DIB elements is configured via ``stackhpc_overcloud_dib_elements``. +The default value depends on the ``os_distribution`` variable. See the YAML +file for details. + +The DIB environment variables are configured via +``stackhpc_overcloud_dib_env_vars``. See the YAML file for details. + +A list of packages to install is configured via +``stackhpc_overcloud_dib_packages``. + +By default, a UEFI-compatible image is built that uses separate LVM volumes for +different mount points. This is done to pass Centre for Internet Security (CIS) +partition benchmarks. The block device YAML configuration is configured via +``stackhpc_overcloud_dib_block_device_config_uefi_lvm``. + +The 3 partitions are: + +* p0: EFI ESP bootloader +* p1: EFI BSP +* p2: LVM PV (``rootpv``) + +The LVM Logical Volumes are: + +============== ================== ========= +LV Mount point Size (GB) +============== ================== ========= +``lv_root`` ``/`` 5G +``lv_tmp`` ``/tmp`` 1G +``lv_var`` ``/var`` 1G +``lv_var_tmp`` ``/var/tmp`` 1G +``lv_log`` ``/var/log`` 1G +``lv_audit`` ``/var/log/audit`` 128M +``lv_home`` ``/home`` 128M +============== ================== ========= + +A compatible LVM configuration is provided, and covered in :ref:`lvm`. +The Logical Volumes in the image are defined with small sizes, with the +intention that they will be grown after provisioning. + +For RedHat family distributions, Dracut modules are configured via +``stackhpc_overcloud_dib_dracut_enabled_modules_default_config``. diff --git a/doc/source/configuration/index.rst b/doc/source/configuration/index.rst index 74d145e29..cd0a784be 100644 --- a/doc/source/configuration/index.rst +++ b/doc/source/configuration/index.rst @@ -10,5 +10,6 @@ the various features provided. walled-garden release-train + host-images cephadm monitoring From 8386215d441aff034e81b001b429aaddfe018ecf Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 19 Dec 2022 12:23:24 +0000 Subject: [PATCH 151/292] docs: LVM --- doc/source/configuration/index.rst | 1 + doc/source/configuration/lvm.rst | 98 ++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 doc/source/configuration/lvm.rst diff --git a/doc/source/configuration/index.rst b/doc/source/configuration/index.rst index cd0a784be..942591b2d 100644 --- a/doc/source/configuration/index.rst +++ b/doc/source/configuration/index.rst @@ -11,5 +11,6 @@ the various features provided. walled-garden release-train host-images + lvm cephadm monitoring diff --git a/doc/source/configuration/lvm.rst b/doc/source/configuration/lvm.rst new file mode 100644 index 000000000..65b892aa8 --- /dev/null +++ b/doc/source/configuration/lvm.rst @@ -0,0 +1,98 @@ +.. _lvm: + +=== +LVM +=== + +StackHPC Kayobe configuration provides Logical Volume Manager (LVM) +configuration that is compatible with the included :ref:`host-images` +configuration. The configuration uses the :kayobe-doc:`LVM +` host configuration functionality of +Kayobe. + +The LVM configuration is provided in +``etc/kayobe/inventory/group_vars/all/stackhpc/lvm``. This allows configuration +variables to be overridden on a per-group or per-host basis (which would not be +possible for an "extra variable" in ``etc/kayobe/*.yml``). This configuration +is not used by default, and must be actively opted into. This can be done as +follows: + +.. code-block:: yaml + + controller_lvm_groups: + - "{{ stackhpc_lvm_group_rootvg }}" + +This will configure the standard set of logical volumes for the ``rootvg`` +volume group on controller hosts. + +The disks in this volume group are configured via +``stackhpc_lvm_group_rootvg_disks``, and by default this contains a single +disk, matched by a partition label of ``root`` (as used in the standard +:ref:`host-images`). + +The size of each LV is configurable via the following variables: + +.. code-block:: yaml + + # StackHPC LVM lv_swap LV size. + stackhpc_lvm_lv_swap_size: 16g + + # StackHPC LVM lv_root LV size. + stackhpc_lvm_lv_root_size: 50g + + # StackHPC LVM lv_tmp LV size. + stackhpc_lvm_lv_tmp_size: 10g + + # StackHPC LVM lv_var LV size. + stackhpc_lvm_lv_var_size: 20g + + # StackHPC LVM lv_var_tmp LV size. + stackhpc_lvm_lv_var_tmp_size: 2g + + # StackHPC LVM lv_log LV size. + stackhpc_lvm_lv_log_size: 20g + + # StackHPC LVM lv_audit LV size. + stackhpc_lvm_lv_audit_size: 10g + + # StackHPC LVM lv_home LV size. + stackhpc_lvm_lv_home_size: 10g + +Additional LVs may be configured via ``stackhpc_lvm_group_rootvg_lvs_extra``. A +common requirement is to have ``/var/lib/docker/`` mounted on a separate LV, +so this has been made convenient to achieve: + +.. code-block:: yaml + + stackhpc_lvm_group_rootvg_lvs_extra: + - "{{ stackhpc_lvm_lv_docker }}" + + # StackHPC LVM lv_docker LV size. + stackhpc_lvm_lv_docker_size: 100%FREE + +It may be desirable to use a lower percentage of the free space, in case +another LV needs to be grown at a later date. + +Growroot playbook +================= + +A ``growroot.yml`` custom playbook is provied that can be used to grow the +partition and LVM Physical Volume (PV) of the root Volume Group (VG). This +allows for expansion of Logical Volumes (LVs) in that VG. + +The following variables may be used to configure the playbook: + +``growroot_group`` + Host pattern against which to target the playbook. Default is ``overcloud``. +``growroot_vg`` + Name of the VG containing the PV to grow. Default is ``rootvg`` to match the + standard :ref:`host image configuration `. + +This playbook may be used as a host configure pre hook, e.g. for overcloud +hosts: + +.. code-block:: console + + mkdir -p ${KAYOBE_CONFIG_PATH}/hooks/overcloud-host-configure/pre.d + cd ${KAYOBE_CONFIG_PATH}/hooks/overcloud-host-configure/pre.d + ln -s ../../../ansible/growroot.yml 30-growroot.yml From 31262bd3617f472211aeb102e66a402f01d556bc Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 20 Dec 2022 16:44:15 +0000 Subject: [PATCH 152/292] docs: swap --- doc/source/configuration/index.rst | 1 + doc/source/configuration/swap.rst | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 doc/source/configuration/swap.rst diff --git a/doc/source/configuration/index.rst b/doc/source/configuration/index.rst index 942591b2d..ebaab804e 100644 --- a/doc/source/configuration/index.rst +++ b/doc/source/configuration/index.rst @@ -12,5 +12,6 @@ the various features provided. release-train host-images lvm + swap cephadm monitoring diff --git a/doc/source/configuration/swap.rst b/doc/source/configuration/swap.rst new file mode 100644 index 000000000..20bae501b --- /dev/null +++ b/doc/source/configuration/swap.rst @@ -0,0 +1,23 @@ +==== +Swap +==== + +StackHPC Kayobe configuration provides a ``swap.yml`` custom playbook that may +be used to configure a swap device. + +The following variables may be used to configure the playbook: + +``swap_group`` + Host pattern against which to target the playbook. Default is ``overcloud``. +``swap_device`` + Name of the swap device to configure. Default is ``/dev/rootvg/lv_swap`` to + match the standard :ref:`host image configuration `. + +This playbook may be used as a host configure post hook, e.g. for overcloud +hosts: + +.. code-block:: console + + mkdir -p ${KAYOBE_CONFIG_PATH}/hooks/overcloud-host-configure/post.d + cd ${KAYOBE_CONFIG_PATH}/hooks/overcloud-host-configure/post.d + ln -s ../../../ansible/swap.yml 10-swap.yml From 65aabdec34a93176ef93a204c8d08849369b1e53 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 12 Dec 2022 16:02:28 +0000 Subject: [PATCH 153/292] docs: Current branch variable --- .../contributor/environments/ci-aio.rst | 22 +++++++++---------- .../contributor/environments/ci-builder.rst | 20 ++++++++--------- doc/source/usage.rst | 19 ++++++++-------- 3 files changed, 31 insertions(+), 30 deletions(-) diff --git a/doc/source/contributor/environments/ci-aio.rst b/doc/source/contributor/environments/ci-aio.rst index b34bad4d6..7107da6f4 100644 --- a/doc/source/contributor/environments/ci-aio.rst +++ b/doc/source/contributor/environments/ci-aio.rst @@ -20,31 +20,31 @@ Install package dependencies. On CentOS: -.. code-block:: console +.. parsed-literal:: sudo dnf install -y python3-virtualenv On Ubuntu: -.. code-block:: console +.. parsed-literal:: sudo apt update sudo apt install -y python3-virtualenv Clone the Kayobe and Kayobe configuration repositories (this one): -.. code-block:: console +.. parsed-literal:: cd mkdir -p src pushd src - git clone https://github.com/stackhpc/kayobe.git -b stackhpc/xena - git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/xena kayobe-config + git clone https://github.com/stackhpc/kayobe.git -b |current_release_git_branch_name| + git clone https://github.com/stackhpc/stackhpc-kayobe-config -b |current_release_git_branch_name| kayobe-config popd Create a virtual environment and install Kayobe: -.. code-block:: console +.. parsed-literal:: cd mkdir -p venvs @@ -57,7 +57,7 @@ Create a virtual environment and install Kayobe: Add initial network configuration: -.. code-block:: console +.. parsed-literal:: sudo ip l add breth1 type bridge sudo ip l set breth1 up @@ -75,7 +75,7 @@ Acquire the Ansible Vault password for this repository, and store a copy at The following commands install Kayobe and its dependencies, and prepare the Ansible control host. -.. code-block:: console +.. parsed-literal:: export KAYOBE_VAULT_PASSWORD=$(cat ~/vault-pw) pushd ~/venvs/kayobe @@ -90,13 +90,13 @@ Deployment Next, configure the host OS & services. -.. code-block:: console +.. parsed-literal:: kayobe overcloud host configure Finally, deploy the overcloud services. -.. code-block:: console +.. parsed-literal:: kayobe overcloud service deploy @@ -107,7 +107,7 @@ Testing Run a smoke test: -.. code-block:: console +.. parsed-literal:: cd ~/kayobe ./dev/overcloud-test-vm.sh diff --git a/doc/source/contributor/environments/ci-builder.rst b/doc/source/contributor/environments/ci-builder.rst index 025c6bb65..6cd4878c3 100644 --- a/doc/source/contributor/environments/ci-builder.rst +++ b/doc/source/contributor/environments/ci-builder.rst @@ -21,31 +21,31 @@ Install package dependencies. On CentOS: -.. code-block:: console +.. parsed-literal:: sudo dnf install -y python3-virtualenv On Ubuntu: -.. code-block:: console +.. parsed-literal:: sudo apt update sudo apt install -y python3-virtualenv Clone the Kayobe and Kayobe configuration repositories (this one): -.. code-block:: console +.. parsed-literal:: cd mkdir -p src pushd src - git clone https://github.com/stackhpc/kayobe.git -b stackhpc/xena - git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/xena kayobe-config + git clone https://github.com/stackhpc/kayobe.git -b |current_release_git_branch_name| + git clone https://github.com/stackhpc/stackhpc-kayobe-config -b |current_release_git_branch_name| kayobe-config popd Create a virtual environment and install Kayobe: -.. code-block:: console +.. parsed-literal:: cd mkdir -p venvs @@ -58,7 +58,7 @@ Create a virtual environment and install Kayobe: Add initial network configuration: -.. code-block:: console +.. parsed-literal:: sudo ip l add breth1 type bridge sudo ip l set breth1 up @@ -76,7 +76,7 @@ Acquire the Ansible Vault password for this repository, and store a copy at The following commands install Kayobe and its dependencies, and prepare the Ansible control host. -.. code-block:: console +.. parsed-literal:: export KAYOBE_VAULT_PASSWORD=$(cat ~/vault-pw) pushd ~/venvs/kayobe @@ -91,7 +91,7 @@ Deployment Next, configure the host OS & services. -.. code-block:: console +.. parsed-literal:: kayobe seed host configure @@ -100,7 +100,7 @@ Building images At this point you are ready to build and push some container images. -.. code-block:: console +.. parsed-literal:: kayobe seed container image build --push kayobe overcloud container image build --push diff --git a/doc/source/usage.rst b/doc/source/usage.rst index 867d2983c..8be28f9eb 100644 --- a/doc/source/usage.rst +++ b/doc/source/usage.rst @@ -23,35 +23,36 @@ New deployments --------------- If starting a new deployment, clone this repository as the starting point for -your configuration. +your configuration: -.. code-block:: console +.. parsed-literal:: - git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/xena + git clone https://github.com/stackhpc/stackhpc-kayobe-config -b |current_release_git_branch_name| Existing deployments -------------------- If migrating an existing deployment to StackHPC Kayobe configuration, you will -need to merge the changes in this repository into your repository. +need to merge the changes in this repository into your repository: -.. code-block:: console +.. parsed-literal:: git remote add stackhpc https://github.com/stackhpc/stackhpc-kayobe-config git fetch stackhpc - git merge stackhpc/stackhpc/xena + git merge stackhpc/|current_release_git_branch_name| Updating -------- This base configuration will be updated over time, to update repository versions, container image tags, and other configuration. Deployments may -consume these updates by merging in the changes with their local configuration. +consume these updates by merging in the changes with their local +configuration: -.. code-block:: console +.. parsed-literal:: git fetch stackhpc - git merge stackhpc/stackhpc/xena + git merge stackhpc/|current_release_git_branch_name| The intention is to avoid merge conflicts where possible, but there may be cases where this is difficult. We are open to discussion on how best to From 19b2e1aef6c6ac35bf55062cfda4917a1f481ff1 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 21 Dec 2022 10:49:26 +0000 Subject: [PATCH 154/292] docs: update ci-aio & ci-builder prerequisites --- doc/source/contributor/environments/ci-aio.rst | 2 +- doc/source/contributor/environments/ci-builder.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/contributor/environments/ci-aio.rst b/doc/source/contributor/environments/ci-aio.rst index 7107da6f4..de1d2d8fb 100644 --- a/doc/source/contributor/environments/ci-aio.rst +++ b/doc/source/contributor/environments/ci-aio.rst @@ -9,7 +9,7 @@ Prerequisites ============= * a CentOS Stream 8 or Ubuntu Focal 20.04 host -* access to the local Pulp server +* access to the Test Pulp server on SMS lab Setup ===== diff --git a/doc/source/contributor/environments/ci-builder.rst b/doc/source/contributor/environments/ci-builder.rst index 6cd4878c3..17b73ad1c 100644 --- a/doc/source/contributor/environments/ci-builder.rst +++ b/doc/source/contributor/environments/ci-builder.rst @@ -10,7 +10,7 @@ Prerequisites ============= * a CentOS Stream 8 or Ubuntu Focal 20.04 host -* access to the local Pulp server +* access to the Test Pulp server on SMS lab Setup ===== From ddc4bb0a2e71fbd3177a0ece319513b254261227 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 21 Dec 2022 13:48:01 +0000 Subject: [PATCH 155/292] README: link to rtd.io --- README.rst | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index c441e9eda..b64ef97c1 100644 --- a/README.rst +++ b/README.rst @@ -5,14 +5,9 @@ StackHPC Kayobe Configuration This repository provides a base Kayobe configuration for the Xena release of StackHPC OpenStack. -Documentation for this project is provided in the ``doc`` directory. To build -it, run: - -.. code-block:: console - - tox -e docs - -The HTML will be written to ``doc/build``. +Documentation is hosted on `readthedocs.io +`__, +and includes release notes. Resources ========= From 546c95abf6804c76de9df438bbcbbc8fc4a51b8a Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 21 Dec 2022 13:48:41 +0000 Subject: [PATCH 156/292] docs: improve release train docs --- doc/source/_static/images/release-train.svg | 1 + doc/source/configuration/release-train.rst | 117 ++++++++++++++++++-- 2 files changed, 106 insertions(+), 12 deletions(-) create mode 100644 doc/source/_static/images/release-train.svg diff --git a/doc/source/_static/images/release-train.svg b/doc/source/_static/images/release-train.svg new file mode 100644 index 000000000..aaf2f2303 --- /dev/null +++ b/doc/source/_static/images/release-train.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/doc/source/configuration/release-train.rst b/doc/source/configuration/release-train.rst index 6e841b8f1..af1e7b3f9 100644 --- a/doc/source/configuration/release-train.rst +++ b/doc/source/configuration/release-train.rst @@ -3,33 +3,52 @@ StackHPC Release Train ====================== StackHPC provides packages and container images for OpenStack via `Ark -`__. +`__. These artifacts are built and released using a +process known as the `Release Train +`__. Deployments should use a local `Pulp `__ repository -server to synchronise content from Ark and serve it locally. Access to the -repositories on Ark is controlled via user accounts issued by StackHPC. +server to synchronise content from Ark and serve it locally. This reduces +Internet bandwidth requirements for package and container downloads. Content is +synced on demand from Ark to the local Pulp, meaning that the local Pulp acts +like a pull-through cache. -This configuration is a base, and should be merged with any existing Kayobe -configuration. It currently provides the following: +Access to the repositories on Ark is controlled via user accounts issued by +StackHPC. -* Configuration to deploy a local Pulp service as a container on the seed -* Pulp repository definitions for CentOS Stream 8 -* Playbooks to synchronise a local Pulp service with Ark -* Configuration to use the local Pulp repository mirrors on control plane hosts -* Configuration to use the local Pulp container registry on control plane hosts +.. image:: /_static/images/release-train.svg + :width: 75% + +All content on Ark is versioned, meaning that a deployment may continue to use +older package repository snapshots and container images when newer content is +released. This allows for improved reliability & repeatability of deployments. This configuration defines two `Pulp distributions `__ for packages, ``development`` and ``production``. This allows packages to be updated and tested in a development or staging environment before rolling them -out to production. +out to production. Typically a given environment will always use the same +distribution, meaning that package repository configuration files do not need +to be updated on control plane hosts in order to consume a package update. Configuration ============= +This configuration provides the following: + +* Configuration to deploy a local Pulp service as a container on the seed +* Pulp repository definitions for CentOS Stream 8 and Rocky Linux 8 +* Playbooks to synchronise a local Pulp service with Ark +* Configuration to use the local Pulp repository mirrors on control plane hosts +* Configuration to use the local Pulp container registry on control plane hosts + Local Pulp server ----------------- +The Pulp container is deployed on the seed by default, but may be disabled by +setting ``seed_pulp_container_enabled`` to ``false`` in +``etc/kayobe/seed.yml``. + The URL and credentials of the local Pulp server are configured in ``etc/kayobe/pulp.yml`` via ``pulp_url``, ``pulp_username`` and ``pulp_password``. In most cases, the default values should be sufficient. @@ -38,6 +57,9 @@ An admin password must be generated and set as the value of a ``etc/kayobe/secrets.yml`` file. This password will be automatically set on Pulp startup. +If a proxy is required to access the Internet from the seed, ``pulp_proxy_url`` +may be used. + StackHPC Ark ------------ @@ -49,11 +71,49 @@ The Ark pulp credentials issued by StackHPC should be configured in stackhpc_release_pulp_username: stackhpc_release_pulp_password: +Package repositories +-------------------- + +Currently, Ark does not provide package repositories for Ubuntu - only +container images. For this reason, ``stackhpc_pulp_sync_ubuntu_focal`` in +``etc/kayobe/pulp.yml`` is set to ``false`` by default. + +CentOS Stream 8 and Rocky Linux 8 package repositories are synced based on the +value of ``os_distribution``. If you need to sync multiple distributions, +``stackhpc_pulp_sync_centos_stream8`` and ``stackhpc_pulp_sync_rocky_8`` in +``etc/kayobe/pulp.yml`` may be set to ``true``. + +On Ark, each package repository provides versioned snapshots using a datetime +stamp (e.g. ``20220817T082321``). The current set of tested versions is defined +in ``etc/kayobe/pulp-repo-versions.yml``. This file is managed by the StackHPC +Release Train and should generally not be modified by consumers of this +repository. + +Package managers +---------------- + +No configuration is provided for APT, since Ark does not currently provide +package repositories for Ubuntu - only container images. + +For CentOS and Rocky Linux based systems, package manager configuration is +provided by ``stackhpc_dnf_repos`` in ``etc/kayobe/dnf.yml``, which points to +package repositories on the local Pulp server. To use this configuration, the +``dnf_custom_repos`` variable must be set, and this is done for hosts in the +``overcloud`` group via the group_vars file +``etc/kayobe/inventory/group_vars/overcloud/stackhpc-dnf-repos``. Similar +configuration may be added for other groups, however there may be ordering +issues during initial deployment when Pulp has not yet been deployed. The distribution name for the environment should be configured as either ``development`` or ``production`` via ``stackhpc_repo_distribution`` in ``etc/kayobe/stackhpc.yml``. +Ceph container images +--------------------- + +By default, Ceph images are not synced from quay.io to the local Pulp. To sync +these images, set ``stackhpc_sync_ceph_images`` to ``true``. + Usage ===== @@ -91,6 +151,39 @@ See the Kayobe :kayobe-doc:`custom playbook documentation local Pulp. This will make synchonised container images available to cloud nodes. +Syncing content +--------------- + +A typical workflow to sync all packages and containers is as follows: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-repo-sync.yml + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-repo-publish.yml + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-container-sync.yml + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-container-publish.yml + +Once the content has been tested in a test/staging environment, it may be +promoted to production: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-repo-promote-production.yml + +Initial seed deployment +----------------------- + +During the initial seed deployment, there is an ordering issue where the +Bifrost container will not yet have been synced, but the local Pulp container +has not yet been deployed. This can be avoided with the following workflow: + +.. code-block:: console + + kayobe seed service deploy --tags seed-deploy-containers --kolla-tags none + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-container-sync.yml + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-container-publish.yml + kayobe seed service deploy + Working with pulp ================= @@ -149,7 +242,7 @@ with the push repository using the pulp CLI: .Done. HTTP Error 404: Not Found -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~ If your login credentials are incorrect, or lack the required permissions, you will see a 404 error during ``pulp-repo-sync.yml``: From d73b5630534fe4cf6e64f6d36323aa6ba300ffd2 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 21 Dec 2022 13:52:30 +0000 Subject: [PATCH 157/292] docs: add info on how to build docs --- doc/source/contributor/documentation.rst | 18 ++++++++++++++++++ doc/source/contributor/index.rst | 1 + 2 files changed, 19 insertions(+) create mode 100644 doc/source/contributor/documentation.rst diff --git a/doc/source/contributor/documentation.rst b/doc/source/contributor/documentation.rst new file mode 100644 index 000000000..99accac42 --- /dev/null +++ b/doc/source/contributor/documentation.rst @@ -0,0 +1,18 @@ +============= +Documentation +============= + +This documentation is provided in the ``doc`` directory. To build it, run: + +.. code-block:: console + + tox -e docs + +The HTML will be written to ``doc/build``, and may be viewed in a web browser. + +Documentation is hosted on readthedocs.io. Multiple maintainers can be added +for the project. GitHub integration ensures that each version is built when the +corresponding branch is pushed to. + +Each release is hosted as a separate version, and new versions will need to be +added manually. diff --git a/doc/source/contributor/index.rst b/doc/source/contributor/index.rst index d80f0c4ce..9d05da2e2 100644 --- a/doc/source/contributor/index.rst +++ b/doc/source/contributor/index.rst @@ -7,4 +7,5 @@ This guide is for contributors of the StackHPC Kayobe configuration project. .. toctree:: :maxdepth: 1 + documentation environments From 0e6c6d6d9300374180c751e0e2740373922c9247 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 21 Dec 2022 13:58:58 +0000 Subject: [PATCH 158/292] docs: move environments index --- .../{environments.rst => environments/index.rst} | 6 +++--- doc/source/contributor/index.rst | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) rename doc/source/contributor/{environments.rst => environments/index.rst} (62%) diff --git a/doc/source/contributor/environments.rst b/doc/source/contributor/environments/index.rst similarity index 62% rename from doc/source/contributor/environments.rst rename to doc/source/contributor/environments/index.rst index 9d99d7a79..7192cae6a 100644 --- a/doc/source/contributor/environments.rst +++ b/doc/source/contributor/environments/index.rst @@ -6,6 +6,6 @@ The following Kayobe environments are provided with this configuration: .. toctree:: :maxdepth: 1 - environments/ci-aio - environments/ci-builder - environments/ci-multinode + ci-aio + ci-builder + ci-multinode diff --git a/doc/source/contributor/index.rst b/doc/source/contributor/index.rst index 9d05da2e2..b6fcb982a 100644 --- a/doc/source/contributor/index.rst +++ b/doc/source/contributor/index.rst @@ -8,4 +8,4 @@ This guide is for contributors of the StackHPC Kayobe configuration project. :maxdepth: 1 documentation - environments + environments/index From 0ef2a51f0320fff36aa58671fc32f09b9c12d81a Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 21 Dec 2022 14:07:15 +0000 Subject: [PATCH 159/292] docs: add info on generating release notes --- doc/source/contributor/documentation.rst | 2 ++ doc/source/contributor/index.rst | 1 + doc/source/contributor/release-notes.rst | 38 ++++++++++++++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 doc/source/contributor/release-notes.rst diff --git a/doc/source/contributor/documentation.rst b/doc/source/contributor/documentation.rst index 99accac42..b8a263f6c 100644 --- a/doc/source/contributor/documentation.rst +++ b/doc/source/contributor/documentation.rst @@ -1,3 +1,5 @@ +.. _documentation: + ============= Documentation ============= diff --git a/doc/source/contributor/index.rst b/doc/source/contributor/index.rst index b6fcb982a..1e1bdd2d6 100644 --- a/doc/source/contributor/index.rst +++ b/doc/source/contributor/index.rst @@ -8,4 +8,5 @@ This guide is for contributors of the StackHPC Kayobe configuration project. :maxdepth: 1 documentation + release-notes environments/index diff --git a/doc/source/contributor/release-notes.rst b/doc/source/contributor/release-notes.rst new file mode 100644 index 000000000..8deb85730 --- /dev/null +++ b/doc/source/contributor/release-notes.rst @@ -0,0 +1,38 @@ +============= +Release notes +============= + +StackHPC Kayobe configuration uses the following release notes sections: + +- ``features`` --- for new features or functionality; these should ideally + refer to the blueprint being implemented; +- ``fixes`` --- for fixes closing bugs; these must refer to the bug being + closed; +- ``upgrade`` --- for notes relevant when upgrading from previous version; + these should ideally be added only between major versions; required when + the proposed change affects behaviour in a non-backwards compatible way or + generally changes something impactful; +- ``deprecations`` --- to track deprecated features; relevant changes may + consist of only the commit message and the release note; +- ``prelude`` --- filled in by the PTL before each release or RC. + +Other release note types may be applied per common sense. +Each change should include a release note unless being a ``TrivialFix`` +change or affecting only docs or CI. Such changes should `not` include +a release note to avoid confusion. +Remember release notes are mostly for end users which, in case of Kolla, +are OpenStack administrators/operators. + +To add a release note, install the ``reno`` package in a Python virtual +environment, then run the following command: + +.. code-block:: console + + reno new + +Release notes for the current release are included in the :ref:`documentation`. +Note that a note won't be included in the generated documentation until it is +tracked by ``git``. + +All release notes can be inspected by browsing ``releasenotes/notes`` +directory. From ab4da7f5856fd026935b3e21c7c11d65a75c292c Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 21 Dec 2022 13:50:15 +0000 Subject: [PATCH 160/292] cephadm: Use quay.io for cephadm_image by default The default for cephadm_image was a poor choice, because ceph images are not synced to the local Pulp by default. Use quay.io when images are not locally synced. --- etc/kayobe/cephadm.yml | 2 +- etc/kayobe/environments/ci-multinode/cephadm.yml | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/etc/kayobe/cephadm.yml b/etc/kayobe/cephadm.yml index 371336310..f326b0ae1 100644 --- a/etc/kayobe/cephadm.yml +++ b/etc/kayobe/cephadm.yml @@ -9,7 +9,7 @@ cephadm_ceph_release: "pacific" #cephadm_fsid: # Ceph container image. -cephadm_image: "{{ stackhpc_docker_registry }}/ceph/ceph:{{ cephadm_image_tag }}" +cephadm_image: "{{ stackhpc_docker_registry if stackhpc_sync_ceph_images | bool else 'quay.io' }}/ceph/ceph:{{ cephadm_image_tag }}" # Ceph container image tag. cephadm_image_tag: "v16.2.5" diff --git a/etc/kayobe/environments/ci-multinode/cephadm.yml b/etc/kayobe/environments/ci-multinode/cephadm.yml index b4cb2ca0d..addb37ddf 100644 --- a/etc/kayobe/environments/ci-multinode/cephadm.yml +++ b/etc/kayobe/environments/ci-multinode/cephadm.yml @@ -2,9 +2,6 @@ ############################################################################### # Cephadm deployment configuration. -# Ceph container image. -cephadm_image: "quay.io/ceph/ceph:v16.2.5" - # Ceph OSD specification. cephadm_osd_spec: service_type: osd From 1b5898dedf36806d5a4ea75109d0cb9f02ca67f1 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 21 Dec 2022 13:50:50 +0000 Subject: [PATCH 161/292] dnf: separate dnf_custom_repos from stackhpc_dnf_repos --- etc/kayobe/dnf.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/etc/kayobe/dnf.yml b/etc/kayobe/dnf.yml index 375fa30ff..dcbc5e8af 100644 --- a/etc/kayobe/dnf.yml +++ b/etc/kayobe/dnf.yml @@ -41,6 +41,12 @@ # file: myrepo # gpgkey: http://gpgkey # gpgcheck: yes +#dnf_custom_repos: + +# A dict of custom repositories that point to the local Pulp server. +# To use these repos, set dnf_custom_repos to the value of stackhpc_dnf_repos. +# This is done by default for hosts in the overcloud group via a group_vars +# file. stackhpc_dnf_repos: "{{ dnf_custom_repos_el8 | combine(lookup('vars', 'dnf_custom_repos_' ~ ansible_facts.distribution | lower )) }}" # Custom repositories shared between all RHEL derivatives. From bcd26d024efa79c531d0eb3b4555044d026c1160 Mon Sep 17 00:00:00 2001 From: Bartosz Bezak Date: Thu, 29 Dec 2022 10:27:45 +0100 Subject: [PATCH 162/292] add netbase and iputils-ping for ubuntu overcloud dib image netbase package is generating /etc/hosts file, among other things https://packages.debian.org/sid/netbase --- etc/kayobe/stackhpc-overcloud-dib.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etc/kayobe/stackhpc-overcloud-dib.yml b/etc/kayobe/stackhpc-overcloud-dib.yml index 84fffbc0f..a39e59608 100644 --- a/etc/kayobe/stackhpc-overcloud-dib.yml +++ b/etc/kayobe/stackhpc-overcloud-dib.yml @@ -58,6 +58,8 @@ stackhpc_overcloud_dib_env_vars: stackhpc_overcloud_dib_packages: - "logrotate" - "net-tools" + - "{% if os_distribution == 'ubuntu' %}netbase{% endif %}" + - "{% if os_distribution == 'ubuntu' %}iputils-ping{% endif %}" # StackHPC overcloud DIB image block device configuration. # This image layout conforms to the CIS partition benchmarks. From 78055cf8e7d485154a1ff06e7dd1bd8f03e7fe00 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Thu, 29 Dec 2022 12:24:03 +0000 Subject: [PATCH 163/292] Sets kolla_docker_registry_insecure to true (#297) * Sets kolla_docker_registry_insecure to true We currently don't configure TLS for the the local pulp registry. This adds the pulp server to the list of insecure-registries, so that we can pull images. * Remove kolla_docker_registry_insecure override in CI environment Based on comments in code review. * Use a better default for kolla_docker_registry_insecure Hoping this will prevent breakage for people already running pulp with TLS. --- etc/kayobe/environments/ci-aio/stackhpc-ci.yml | 5 ----- etc/kayobe/environments/ci-builder/stackhpc-ci.yml | 5 ----- etc/kayobe/environments/ci-multinode/stackhpc-ci.yml | 5 ----- etc/kayobe/kolla.yml | 2 +- 4 files changed, 1 insertion(+), 16 deletions(-) diff --git a/etc/kayobe/environments/ci-aio/stackhpc-ci.yml b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml index a9827d82e..a874b9d7d 100644 --- a/etc/kayobe/environments/ci-aio/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml @@ -5,11 +5,6 @@ # Docker namespace to use for Kolla images. Default is 'kolla'. kolla_docker_namespace: stackhpc-dev -# Whether docker should be configured to use an insecure registry for Kolla -# images. Default is false, unless docker_registry_enabled is true and -# docker_registry_enable_tls is false. -kolla_docker_registry_insecure: "{{ 'https' not in stackhpc_repo_mirror_url }}" - ############################################################################### # Network configuration. diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml index 4a9e50f1c..0fce89039 100644 --- a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml @@ -5,11 +5,6 @@ # Docker namespace to use for Kolla images. Default is 'kolla'. kolla_docker_namespace: stackhpc-dev -# Whether docker should be configured to use an insecure registry for Kolla -# images. Default is false, unless docker_registry_enabled is true and -# docker_registry_enable_tls is false. -kolla_docker_registry_insecure: "{{ 'https' not in stackhpc_repo_mirror_url }}" - # Kolla feature flag configuration. kolla_enable_barbican: true kolla_enable_central_logging: true diff --git a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml index 9cfccab43..3a1bedd31 100644 --- a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml @@ -5,11 +5,6 @@ # Docker namespace to use for Kolla images. Default is 'kolla'. kolla_docker_namespace: stackhpc-dev -# Whether docker should be configured to use an insecure registry for Kolla -# # images. Default is false, unless docker_registry_enabled is true and -# # docker_registry_enable_tls is false. -kolla_docker_registry_insecure: "{{ 'https' not in stackhpc_repo_mirror_url }}" - ############################################################################### # Network configuration. diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 920eb4caa..f98dd5a9a 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -84,7 +84,7 @@ kolla_docker_namespace: stackhpc # Whether docker should be configured to use an insecure registry for Kolla # images. Default is false, unless docker_registry_enabled is true and # docker_registry_enable_tls is false. -#kolla_docker_registry_insecure: +kolla_docker_registry_insecure: "{{ 'https' not in stackhpc_repo_mirror_url }}" # Username to use to access a docker registry. Default is not set, in which # case the registry will be used without authentication. From de21305e89afb6099efcc7a3d706cf90478b13dc Mon Sep 17 00:00:00 2001 From: Dawud M <7688823+technowhizz@users.noreply.github.com> Date: Fri, 30 Dec 2022 19:49:05 +0000 Subject: [PATCH 164/292] Fix bug where old metrics wouldn't be overwritten The mv command wasnt overwriting the textcollector file. mv -f is now being used Additionally, serial numbers are now all lowercase --- etc/kayobe/ansible/scripts/smartmon.sh | 2 +- etc/kayobe/ansible/smartmon-tools.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/ansible/scripts/smartmon.sh b/etc/kayobe/ansible/scripts/smartmon.sh index bcac8b8b3..32b3ea81c 100644 --- a/etc/kayobe/ansible/scripts/smartmon.sh +++ b/etc/kayobe/ansible/scripts/smartmon.sh @@ -115,7 +115,7 @@ parse_smartctl_info() { case "${info_type}" in Model_Family) model_family="${info_value}" ;; Device_Model) device_model="${info_value}" ;; - Serial_Number) serial_number="${info_value}" ;; + Serial_Number) serial_number="$(echo ${info_value} | tr '[:upper:]' '[:lower:]')" ;; Firmware_Version) fw_version="${info_value}" ;; Vendor) vendor="${info_value}" ;; Product) product="${info_value}" ;; diff --git a/etc/kayobe/ansible/smartmon-tools.yml b/etc/kayobe/ansible/smartmon-tools.yml index 6b275c264..e23e32396 100644 --- a/etc/kayobe/ansible/smartmon-tools.yml +++ b/etc/kayobe/ansible/smartmon-tools.yml @@ -36,7 +36,7 @@ name: "SMART metrics for drive monitoring using {{ item }}" user: root minute: "*/5" - job: "/usr/local/bin/{{ item }}.sh > /var/lib/docker/volumes/textfile/_data/{{ item }}.prom.temp && mv /var/lib/docker/volumes/textfile/_data/{{ item }}.prom.temp /var/lib/docker/volumes/textfile/_data/{{ item }}.prom" + job: "/usr/local/bin/{{ item }}.sh > /var/lib/docker/volumes/textfile/_data/{{ item }}.prom.temp && mv -f /var/lib/docker/volumes/textfile/_data/{{ item }}.prom.temp /var/lib/docker/volumes/textfile/_data/{{ item }}.prom" loop: - smartmon - nvmemon From b0bc77f0fed90bdced0146b1261cfdd9d881b7fa Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 4 Jan 2023 17:00:07 +0000 Subject: [PATCH 165/292] Ubuntu: bump OVN & Open vSwitch packages The Open vSwitch and OVN packages in Ubuntu Xena UCA repository are quite old - 2.15 and 20.12 respectively. Pull in these packages from the Yoga UCA, which are 2.17 and 22.03, to more closely match the CentOS packages. This change uses newly built containers with these packages. --- etc/kayobe/kolla/globals.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 7d15d542d..85d72d700 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -11,6 +11,13 @@ enable_docker_repo: {% raw %}"{{ 'overcloud' not in group_names or ansible_facts bifrost_tag: xena-20221128T101757 {% else %} bifrost_tag: xena-20221213T224057 +kolla_toolbox_tag: xena-20230104T145414 +neutron_tag: xena-20230104T145414 +neutron_tls_proxy_tag: "{% raw %}{{ openstack_tag }}{% endraw %}" +nova_tag: xena-20230104T145414 +octavia_tag: xena-20230104T145414 +openvswitch_tag: xena-20230104T145414 +ovn_tag: xena-20230104T145414 {% endif %} ############################################################################# From 1862a3ae476666636d88b0481b9d8fe37977c798 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Fri, 6 Jan 2023 14:31:18 +0000 Subject: [PATCH 166/292] Add docs for rolling out changes to RabbitMQ --- doc/source/operations/rabbitmq.rst | 73 ++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 doc/source/operations/rabbitmq.rst diff --git a/doc/source/operations/rabbitmq.rst b/doc/source/operations/rabbitmq.rst new file mode 100644 index 000000000..f4c00f6dd --- /dev/null +++ b/doc/source/operations/rabbitmq.rst @@ -0,0 +1,73 @@ +Rolling out changes to RabbitMQ +================================ + +Prior to upgrading, some changes need to be rolled out for RabbitMQ. +This should make RabbitMQ more stable during the upgrade(s). These changes +are: + +* Enable the high availability setting via Kolla-Ansible + ``om_enable_rabbitmq_high_availability``. +* Update RabbitMQ to version 3.9.22 + +**NOTE:** There is guaranteed to be downtime during this procedure, as it +requires restarting RabbitMQ and all the OpenStack services that use it. The +state of RabbitMQ may also need to be reset. + +Instructions +------------ + +If you are upgrading from Wallaby to Xena, you will need to enable the HA +config option in +``/home/ubuntu/kayobe/config/src/kolla-ansible/ansible/group_vars/all.yml``. +This will default to ``true`` from Xena onwards. + +``om_enable_rabbitmq_high_availability: true`` + +Synchronise the Pulp containers. + +``kayobe playbook run etc/kayobe/ansible/pulp-container-sync.yml pulp-container-publish.yml -e stackhpc_pulp_images_kolla_filter=rabbitmq`` + +Generate the new config files for the overcloud services. + +``kayobe overcloud servuce configuration generate`` + +Pull the RabbitMQ container image. + +``kayobe overcloud container image pull -kt rabbitmq`` + +Stop all the OpenStack services which use RabbitMQ. + +``kayobe overcloud host command run --command 'docker ps -a | egrep '(barbican|blazar|ceilometer|cinder|cloudkitty|designate|heat|ironic|keystone|magnum|manila|masakari|neutron|nova|octavia)' | awk '{ print $NF }' | xargs docker stop'`` + +Upgrade RabbitMQ. + +``kayobe overcloud service upgrade -kt rabbitmq --skip-prechecks`` + +Check to see if RabbitMQ is functioning as expected. + +``kayobe overcloud service upgrade -kt rabbitmq --skip-prechecks`` + +``kayobe overcloud host command run --command 'docker exec rabbitmq rabbitmqctl cluster_status'`` + +``kayobe overcloud host command run --command 'docker exec rabbitmq rabbitmqctl list_queues name durable'`` + +The cluster status should list all controllers. The queues listed should be +durable if their names do not start with the following: + +* amq. +* .\*\_fanout\_ +* reply\_ + +At this stage, you may find that the above is not correct. If this is the case, +running the rabbitmq hammer playbook will reset the state of the RabbitMQ +nodes, and then bring the services which use RabbitMQ back up. + +``kayobe playbook run stackhpc-kayobe-config/etc/kayobe/ansible/rabbitmq-reset.yml`` + +If you do not need to use the playbook here, then you will need to bring the +services back up yourself: + +``kayobe overcloud host command run --command 'docker ps -a | egrep '(barbican|blazar|ceilometer|cinder|cloudkitty|designate|heat|ironic|keystone|magnum|manila|masakari|neutron|nova|octavia)' | awk '{ print $NF }' | xargs docker start'`` + +If there are issues with the services after this, you may still need to run the +hammer playbook. From 136afdab7919996ae2141a925ddf69ce36a0ad02 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Fri, 6 Jan 2023 14:54:47 +0000 Subject: [PATCH 167/292] Changes to work with Sphinx docs --- doc/source/index.rst | 1 + doc/source/operations/index.rst | 10 +++++++ doc/source/operations/rabbitmq.rst | 43 +++++++++++++++++++++--------- 3 files changed, 41 insertions(+), 13 deletions(-) create mode 100644 doc/source/operations/index.rst diff --git a/doc/source/index.rst b/doc/source/index.rst index a7f770bc4..ac47cf87d 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -32,3 +32,4 @@ Contents configuration/index release-notes contributor/index + operations/index diff --git a/doc/source/operations/index.rst b/doc/source/operations/index.rst new file mode 100644 index 000000000..9ce3b29a0 --- /dev/null +++ b/doc/source/operations/index.rst @@ -0,0 +1,10 @@ +================= +Operator Guide +================= + +This guide is for operators of the StackHPC Kayobe configuration project. + +.. toctree:: + :maxdepth: 1 + + rabbitmq diff --git a/doc/source/operations/rabbitmq.rst b/doc/source/operations/rabbitmq.rst index f4c00f6dd..14b4f0b73 100644 --- a/doc/source/operations/rabbitmq.rst +++ b/doc/source/operations/rabbitmq.rst @@ -1,5 +1,6 @@ +=============================== Rolling out changes to RabbitMQ -================================ +=============================== Prior to upgrading, some changes need to be rolled out for RabbitMQ. This should make RabbitMQ more stable during the upgrade(s). These changes @@ -21,35 +22,47 @@ config option in ``/home/ubuntu/kayobe/config/src/kolla-ansible/ansible/group_vars/all.yml``. This will default to ``true`` from Xena onwards. -``om_enable_rabbitmq_high_availability: true`` +.. code-block:: console + + om_enable_rabbitmq_high_availability: true Synchronise the Pulp containers. -``kayobe playbook run etc/kayobe/ansible/pulp-container-sync.yml pulp-container-publish.yml -e stackhpc_pulp_images_kolla_filter=rabbitmq`` +.. code-block:: console + + kayobe playbook run etc/kayobe/ansible/pulp-container-sync.yml pulp-container-publish.yml -e stackhpc_pulp_images_kolla_filter=rabbitmq Generate the new config files for the overcloud services. -``kayobe overcloud servuce configuration generate`` +.. code-block:: console + + kayobe overcloud servuce configuration generate Pull the RabbitMQ container image. -``kayobe overcloud container image pull -kt rabbitmq`` +.. code-block:: console + + kayobe overcloud container image pull -kt rabbitmq Stop all the OpenStack services which use RabbitMQ. -``kayobe overcloud host command run --command 'docker ps -a | egrep '(barbican|blazar|ceilometer|cinder|cloudkitty|designate|heat|ironic|keystone|magnum|manila|masakari|neutron|nova|octavia)' | awk '{ print $NF }' | xargs docker stop'`` +.. code-block:: console + + kayobe overcloud host command run --command 'docker ps -a | egrep '(barbican|blazar|ceilometer|cinder|cloudkitty|designate|heat|ironic|keystone|magnum|manila|masakari|neutron|nova|octavia)' | awk '{ print $NF }' | xargs docker stop' Upgrade RabbitMQ. -``kayobe overcloud service upgrade -kt rabbitmq --skip-prechecks`` +.. code-block:: console -Check to see if RabbitMQ is functioning as expected. + kayobe overcloud service upgrade -kt rabbitmq --skip-prechecks -``kayobe overcloud service upgrade -kt rabbitmq --skip-prechecks`` +Check to see if RabbitMQ is functioning as expected. -``kayobe overcloud host command run --command 'docker exec rabbitmq rabbitmqctl cluster_status'`` +.. code-block:: console -``kayobe overcloud host command run --command 'docker exec rabbitmq rabbitmqctl list_queues name durable'`` + kayobe overcloud service upgrade -kt rabbitmq --skip-prechecks + kayobe overcloud host command run --command 'docker exec rabbitmq rabbitmqctl cluster_status' + kayobe overcloud host command run --command 'docker exec rabbitmq rabbitmqctl list_queues name durable' The cluster status should list all controllers. The queues listed should be durable if their names do not start with the following: @@ -62,12 +75,16 @@ At this stage, you may find that the above is not correct. If this is the case, running the rabbitmq hammer playbook will reset the state of the RabbitMQ nodes, and then bring the services which use RabbitMQ back up. -``kayobe playbook run stackhpc-kayobe-config/etc/kayobe/ansible/rabbitmq-reset.yml`` +.. code-block:: console + + kayobe playbook run stackhpc-kayobe-config/etc/kayobe/ansible/rabbitmq-reset.yml If you do not need to use the playbook here, then you will need to bring the services back up yourself: -``kayobe overcloud host command run --command 'docker ps -a | egrep '(barbican|blazar|ceilometer|cinder|cloudkitty|designate|heat|ironic|keystone|magnum|manila|masakari|neutron|nova|octavia)' | awk '{ print $NF }' | xargs docker start'`` +.. code-block:: console + + kayobe overcloud host command run --command 'docker ps -a | egrep '(barbican|blazar|ceilometer|cinder|cloudkitty|designate|heat|ironic|keystone|magnum|manila|masakari|neutron|nova|octavia)' | awk '{ print $NF }' | xargs docker start' If there are issues with the services after this, you may still need to run the hammer playbook. From 89501ff3c0e48cdc759fa7fbc371c747dc23e97d Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Mon, 9 Jan 2023 14:57:14 +0000 Subject: [PATCH 168/292] Add cephadm_commands placeholder variable Support for this feature was added in #290. This adds a variable and some documentation to raise awareness of its existence. --- doc/source/configuration/cephadm.rst | 15 +++++++++++++++ etc/kayobe/cephadm.yml | 4 ++++ 2 files changed, 19 insertions(+) diff --git a/doc/source/configuration/cephadm.rst b/doc/source/configuration/cephadm.rst index e207dd5b7..39ea14882 100644 --- a/doc/source/configuration/cephadm.rst +++ b/doc/source/configuration/cephadm.rst @@ -239,6 +239,21 @@ for Cinder, Cinder backup, Glance, and Nova in Kolla Ansible. mgr: "profile rbd pool=images" state: present +Ceph Commands +~~~~~~~~~~~~~ + +It is possible to run an arbitrary list of commands against the cluster after deployment +by setting the ``ceph_commands`` variable. ``ceph_commands`` should be a list of commands +to pass to ``cephadm shell -- ceph``. For example: + +.. code:: yaml + + # A list of commands to pass to cephadm shell -- ceph. See stackhpc.cephadm.commands + # for format. + cephadm_commands: + - "fs new cephfs cephfs_metadata cephfs_data" + - "orch apply mds cephfs --placement 3" + Deployment ========== diff --git a/etc/kayobe/cephadm.yml b/etc/kayobe/cephadm.yml index f326b0ae1..4f5f0ed7c 100644 --- a/etc/kayobe/cephadm.yml +++ b/etc/kayobe/cephadm.yml @@ -60,6 +60,10 @@ cephadm_cluster_network: "{{ storage_mgmt_net_name | net_cidr }}" # List of Cephx keys. See stackhpc.cephadm.keys role for format. #cephadm_keys: +# A list of commands to pass to cephadm shell -- ceph. See stackhpc.cephadm.commands +# for format. +#cephadm_commands: + ############################################################################### # Kolla Ceph auto-configuration. From 01875620846a14b54017f77fd5149cf0bced9463 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Mon, 9 Jan 2023 15:59:00 +0000 Subject: [PATCH 169/292] Update based on review --- doc/source/operations/rabbitmq.rst | 81 +++++++++++++++++------------- 1 file changed, 47 insertions(+), 34 deletions(-) diff --git a/doc/source/operations/rabbitmq.rst b/doc/source/operations/rabbitmq.rst index 14b4f0b73..0ff5c3b69 100644 --- a/doc/source/operations/rabbitmq.rst +++ b/doc/source/operations/rabbitmq.rst @@ -1,26 +1,47 @@ -=============================== -Rolling out changes to RabbitMQ -=============================== +======== +RabbitMQ +======== -Prior to upgrading, some changes need to be rolled out for RabbitMQ. -This should make RabbitMQ more stable during the upgrade(s). These changes -are: +High Availability +================= +In order to improve the stability of RabbitMQ, some changes need to be rolled, +out. These changes are: + +* Update RabbitMQ to version 3.9.22, if you are running old images on Wallaby + or Xena, by synchronising and then pulling a new RabbitMQ container from + Pulp. * Enable the high availability setting via Kolla-Ansible ``om_enable_rabbitmq_high_availability``. -* Update RabbitMQ to version 3.9.22 + +By default in Kolla-Ansible, two key options for the high availability of +RabbitMQ are disabled. These are durable queues, where messages are persisted +to disk; and classic queue mirroring, where messages are replicated across +multiple exchanges. Without these, a deployment will be a poor state for +managing any updates for RabbitMQ, or any outages that cause it to be stopped. +Messages held in RabbitMQ nodes that are stopped will be lost, which causes +knock-on effects to the OpenStack services which either sent or were expecting +to receive them. The Kolla-Ansible flag +``om_enable_rabbitmq_high_availability`` can be used to enable both of these +options. This will default to ``true`` from Xena onwards. + +While the `RabbitMQ docs `_ do +say "throughput and latency of a queue is not affected by whether a queue is +durable or not in most cases", it should be mentioned that there could be a +potential performance hit from replicating all messages to the disk within +large deployments. These changes would therefore be a tradeoff of performance +for stability. **NOTE:** There is guaranteed to be downtime during this procedure, as it requires restarting RabbitMQ and all the OpenStack services that use it. The -state of RabbitMQ may also need to be reset. +state of RabbitMQ will also be reset. Instructions ------------ -If you are upgrading from Wallaby to Xena, you will need to enable the HA -config option in -``/home/ubuntu/kayobe/config/src/kolla-ansible/ansible/group_vars/all.yml``. -This will default to ``true`` from Xena onwards. +If you are upgrading from Wallaby to Xena, or if you're on Wallaby and want +RabbitMQ to be more stable, you will need to enable the HA config option in +``etc/kayobe/kolla/globals.yml``. .. code-block:: console @@ -36,7 +57,7 @@ Generate the new config files for the overcloud services. .. code-block:: console - kayobe overcloud servuce configuration generate + kayobe overcloud service configuration generate Pull the RabbitMQ container image. @@ -48,7 +69,7 @@ Stop all the OpenStack services which use RabbitMQ. .. code-block:: console - kayobe overcloud host command run --command 'docker ps -a | egrep '(barbican|blazar|ceilometer|cinder|cloudkitty|designate|heat|ironic|keystone|magnum|manila|masakari|neutron|nova|octavia)' | awk '{ print $NF }' | xargs docker stop' + kayobe overcloud host command run --command "docker ps -a | egrep '(barbican|blazar|ceilometer|cinder|cloudkitty|designate|heat|ironic|keystone|magnum|manila|masakari|neutron|nova|octavia)' | awk '{ print $NF }' | xargs docker stop" Upgrade RabbitMQ. @@ -56,13 +77,20 @@ Upgrade RabbitMQ. kayobe overcloud service upgrade -kt rabbitmq --skip-prechecks +In order to convert the queues to be durable, you will need to reset the state +of RabbitMQ, and restart the services which use it. This can be done with the +RabbitMQ hammer playbook: + +.. code-block:: console + + kayobe playbook run stackhpc-kayobe-config/etc/kayobe/ansible/rabbitmq-reset.yml + Check to see if RabbitMQ is functioning as expected. .. code-block:: console - kayobe overcloud service upgrade -kt rabbitmq --skip-prechecks - kayobe overcloud host command run --command 'docker exec rabbitmq rabbitmqctl cluster_status' - kayobe overcloud host command run --command 'docker exec rabbitmq rabbitmqctl list_queues name durable' + kayobe overcloud host command run --show-output --command 'docker exec rabbitmq rabbitmqctl cluster_status' + kayobe overcloud host command run --show-output --command 'docker exec rabbitmq rabbitmqctl list_queues name durable' The cluster status should list all controllers. The queues listed should be durable if their names do not start with the following: @@ -71,20 +99,5 @@ durable if their names do not start with the following: * .\*\_fanout\_ * reply\_ -At this stage, you may find that the above is not correct. If this is the case, -running the rabbitmq hammer playbook will reset the state of the RabbitMQ -nodes, and then bring the services which use RabbitMQ back up. - -.. code-block:: console - - kayobe playbook run stackhpc-kayobe-config/etc/kayobe/ansible/rabbitmq-reset.yml - -If you do not need to use the playbook here, then you will need to bring the -services back up yourself: - -.. code-block:: console - - kayobe overcloud host command run --command 'docker ps -a | egrep '(barbican|blazar|ceilometer|cinder|cloudkitty|designate|heat|ironic|keystone|magnum|manila|masakari|neutron|nova|octavia)' | awk '{ print $NF }' | xargs docker start' - -If there are issues with the services after this, you may still need to run the -hammer playbook. +If there are issues with the services after this, particularly during upgrades, +you may find it useful to reuse the hammer playbook. From 2fd1590eb8ac739a07ad9cccbefc7725ea1a3855 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Tue, 10 Jan 2023 09:34:00 +0000 Subject: [PATCH 170/292] Enable RabbitMQ HA flag and add new rabbitmq_tags --- etc/kayobe/kolla/globals.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 85d72d700..358538cfa 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -9,6 +9,7 @@ enable_docker_repo: {% raw %}"{{ 'overcloud' not in group_names or ansible_facts {% if kolla_base_distro == 'centos' %} bifrost_tag: xena-20221128T101757 +rabbitmq_tag: xena-20230105T141714 {% else %} bifrost_tag: xena-20221213T224057 kolla_toolbox_tag: xena-20230104T145414 @@ -18,8 +19,11 @@ nova_tag: xena-20230104T145414 octavia_tag: xena-20230104T145414 openvswitch_tag: xena-20230104T145414 ovn_tag: xena-20230104T145414 +rabbitmq_tag: xena-20230105T145825 {% endif %} +om_enable_rabbitmq_high_availability: true + ############################################################################# # Monitoring and alerting related settings From cbbfbd67d7c9dbc43083403240732abd41e95eb9 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Mon, 9 Jan 2023 16:28:35 +0100 Subject: [PATCH 171/292] Improve description of Blackbox SSL alerts --- etc/kayobe/kolla/config/prometheus/blackbox.rules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/kolla/config/prometheus/blackbox.rules b/etc/kayobe/kolla/config/prometheus/blackbox.rules index 59e144669..0f0f72012 100644 --- a/etc/kayobe/kolla/config/prometheus/blackbox.rules +++ b/etc/kayobe/kolla/config/prometheus/blackbox.rules @@ -39,7 +39,7 @@ groups: severity: warning annotations: summary: Blackbox SSL certificate will expire soon (instance {{ $labels.instance }}) - description: "SSL certificate expires in 30 days" + description: "SSL certificate expires in less than 30 days" - alert: BlackboxSslCertificateWillExpireSoon expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 3 @@ -48,7 +48,7 @@ groups: severity: critical annotations: summary: Blackbox SSL certificate will expire soon (instance {{ $labels.instance }}) - description: "SSL certificate expires in 3 days" + description: "SSL certificate expires in less than 3 days" - alert: BlackboxSslCertificateExpired expr: probe_ssl_earliest_cert_expiry - time() <= 0 From c4956835b910b440efa28dc8a8b5d1fc63c38716 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Tue, 10 Jan 2023 09:51:35 +0000 Subject: [PATCH 172/292] Add release note --- ...-and-update-rabbitmq-version-8eaa68b2a9486320.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 releasenotes/notes/enable-rabbitmq-ha-and-update-rabbitmq-version-8eaa68b2a9486320.yaml diff --git a/releasenotes/notes/enable-rabbitmq-ha-and-update-rabbitmq-version-8eaa68b2a9486320.yaml b/releasenotes/notes/enable-rabbitmq-ha-and-update-rabbitmq-version-8eaa68b2a9486320.yaml new file mode 100644 index 000000000..8a6bc5023 --- /dev/null +++ b/releasenotes/notes/enable-rabbitmq-ha-and-update-rabbitmq-version-8eaa68b2a9486320.yaml @@ -0,0 +1,11 @@ +--- +features: + - | + The flag ``om_enable_rabbitmq_high-availability`` is now set to ``true``. + Adds tags for new RabbitMQ containers to update to RabbitMQ version 3.9.22. +upgrade: + - | + The flag ``om_enable_rabbitmq_high-availability`` is now set to ``true``. + As this enables durable queues, RabbitMQ will need to be reset, and the + services which use it restarted. + Tags are added to update the RabbitMQ containers to version 3.9.22. From f7993eb05cee5bd2dca7d4b84702e92d941cb630 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Tue, 10 Jan 2023 14:40:31 +0000 Subject: [PATCH 173/292] Apply suggestions from code review Co-authored-by: Mark Goddard --- doc/source/operations/rabbitmq.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/doc/source/operations/rabbitmq.rst b/doc/source/operations/rabbitmq.rst index 0ff5c3b69..ac250f573 100644 --- a/doc/source/operations/rabbitmq.rst +++ b/doc/source/operations/rabbitmq.rst @@ -17,13 +17,13 @@ out. These changes are: By default in Kolla-Ansible, two key options for the high availability of RabbitMQ are disabled. These are durable queues, where messages are persisted to disk; and classic queue mirroring, where messages are replicated across -multiple exchanges. Without these, a deployment will be a poor state for -managing any updates for RabbitMQ, or any outages that cause it to be stopped. +multiple exchanges. Without these, a deployment has a higher risk of experiencing +issues when updating RabbitMQ, or recovering from network outages. Messages held in RabbitMQ nodes that are stopped will be lost, which causes knock-on effects to the OpenStack services which either sent or were expecting to receive them. The Kolla-Ansible flag ``om_enable_rabbitmq_high_availability`` can be used to enable both of these -options. This will default to ``true`` from Xena onwards. +options. The default will be overridden to ``true`` from Xena onwards in StackHPC Kayobe configuration. While the `RabbitMQ docs `_ do say "throughput and latency of a queue is not affected by whether a queue is @@ -38,16 +38,15 @@ state of RabbitMQ will also be reset. Instructions ------------ - -If you are upgrading from Wallaby to Xena, or if you're on Wallaby and want -RabbitMQ to be more stable, you will need to enable the HA config option in +If you are planning to perform an upgrade, it is recommended to first roll out these changes. +If you are currently running Wallaby, you will need to enable the HA config option in ``etc/kayobe/kolla/globals.yml``. .. code-block:: console om_enable_rabbitmq_high_availability: true -Synchronise the Pulp containers. +If you are running Wallaby or Xena, synchronise the Pulp containers. .. code-block:: console From 02a47164e2bf0a6054d004a5aed0d6a93bcf0fc4 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Tue, 10 Jan 2023 15:00:43 +0000 Subject: [PATCH 174/292] Alter instructions, reorder index --- doc/source/index.rst | 2 +- doc/source/operations/rabbitmq.rst | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/source/index.rst b/doc/source/index.rst index ac47cf87d..be1320948 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -30,6 +30,6 @@ Contents usage configuration/index + operations/index release-notes contributor/index - operations/index diff --git a/doc/source/operations/rabbitmq.rst b/doc/source/operations/rabbitmq.rst index ac250f573..39751c5e4 100644 --- a/doc/source/operations/rabbitmq.rst +++ b/doc/source/operations/rabbitmq.rst @@ -84,6 +84,13 @@ RabbitMQ hammer playbook: kayobe playbook run stackhpc-kayobe-config/etc/kayobe/ansible/rabbitmq-reset.yml +The hammer playbook only targets the services which are known to have issues +when RabbitMQ breaks. You will still need to start the remaining services: + +.. code-block:: console + + kayobe overcloud host command run --command "docker ps -a | egrep '(barbican|blazar|ceilometer|cloudkitty|designate|manila|masakari|octavia)' | awk '{ print $NF }' | xargs docker start" + Check to see if RabbitMQ is functioning as expected. .. code-block:: console From 50cf847edd562208ab607cf8ec88ee965b6796dc Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Wed, 11 Jan 2023 11:23:08 +0100 Subject: [PATCH 175/292] Update magnum_tag --- etc/kayobe/kolla/globals.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 358538cfa..c2c7ce5c6 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -9,10 +9,12 @@ enable_docker_repo: {% raw %}"{{ 'overcloud' not in group_names or ansible_facts {% if kolla_base_distro == 'centos' %} bifrost_tag: xena-20221128T101757 +magnum_tag: xena-20230111T093652 rabbitmq_tag: xena-20230105T141714 {% else %} bifrost_tag: xena-20221213T224057 kolla_toolbox_tag: xena-20230104T145414 +magnum_tag: xena-20230111T095751 neutron_tag: xena-20230104T145414 neutron_tls_proxy_tag: "{% raw %}{{ openstack_tag }}{% endraw %}" nova_tag: xena-20230104T145414 From b47457817ef33871dcea8a31f8a8d01c4a7b34e5 Mon Sep 17 00:00:00 2001 From: Matt Anson Date: Tue, 10 Jan 2023 15:46:49 +0000 Subject: [PATCH 176/292] Add releasenote (cherry picked from commit ee30588f999fb60bf77cc35c63f48d015242139a) --- .../notes/magnum-bump-tags-7841edcbd3da4b9c.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 releasenotes/notes/magnum-bump-tags-7841edcbd3da4b9c.yaml diff --git a/releasenotes/notes/magnum-bump-tags-7841edcbd3da4b9c.yaml b/releasenotes/notes/magnum-bump-tags-7841edcbd3da4b9c.yaml new file mode 100644 index 000000000..28b922e76 --- /dev/null +++ b/releasenotes/notes/magnum-bump-tags-7841edcbd3da4b9c.yaml @@ -0,0 +1,10 @@ +--- +fixes: + - | + Fixes CoreDNS for Magnum clusters crashing on startup. + - | + Allows cinder-csi nodeplugin to start on the same Magnum cluster host as + cinder-csi controllerplugin. + - | + Corrects ClusterRole rules for Magnum cluster-autoscaler, and sets + cluster-autoscaler pods to use hostNetwork. From 280b2152c46a3ba981f87bf7fa653d919b92302a Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 11 Jan 2023 14:33:38 +0000 Subject: [PATCH 177/292] Update doc/source/configuration/cephadm.rst Co-authored-by: Mark Goddard --- doc/source/configuration/cephadm.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/configuration/cephadm.rst b/doc/source/configuration/cephadm.rst index 39ea14882..f20674a8e 100644 --- a/doc/source/configuration/cephadm.rst +++ b/doc/source/configuration/cephadm.rst @@ -243,7 +243,7 @@ Ceph Commands ~~~~~~~~~~~~~ It is possible to run an arbitrary list of commands against the cluster after deployment -by setting the ``ceph_commands`` variable. ``ceph_commands`` should be a list of commands +by setting the ``cephadm_commands`` variable. ``cephadm_commands`` should be a list of commands to pass to ``cephadm shell -- ceph``. For example: .. code:: yaml From 4ce07df3c99b9a08d0861119fc4356d5aeecf8e2 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 11 Jan 2023 16:16:31 +0000 Subject: [PATCH 178/292] Update example Updating the example based on feedback from the code review. The previous example applied configuration that would have been better applied using the cephadm role. We don't want to encourage such usage. --- doc/source/configuration/cephadm.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/source/configuration/cephadm.rst b/doc/source/configuration/cephadm.rst index f20674a8e..0322482c0 100644 --- a/doc/source/configuration/cephadm.rst +++ b/doc/source/configuration/cephadm.rst @@ -251,8 +251,9 @@ to pass to ``cephadm shell -- ceph``. For example: # A list of commands to pass to cephadm shell -- ceph. See stackhpc.cephadm.commands # for format. cephadm_commands: - - "fs new cephfs cephfs_metadata cephfs_data" - - "orch apply mds cephfs --placement 3" + # Configure Prometheus exporter to listen on a specific interface. The default + # is to listen on all interfaces. + - "config set mgr mgr/prometheus/server_addr 10.0.0.1" Deployment ========== From 382a13710b3fceb92aa7324829dc2ed07fe45167 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 11 Jan 2023 17:01:52 +0000 Subject: [PATCH 179/292] Remove Wallaby RabbitMQ release note We have an equivalent already --- .../notes/update-rabbitmq-version-90fe988f516e090a.yaml | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 releasenotes/notes/update-rabbitmq-version-90fe988f516e090a.yaml diff --git a/releasenotes/notes/update-rabbitmq-version-90fe988f516e090a.yaml b/releasenotes/notes/update-rabbitmq-version-90fe988f516e090a.yaml deleted file mode 100644 index 040ee58df..000000000 --- a/releasenotes/notes/update-rabbitmq-version-90fe988f516e090a.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -features: - - | - Adds tags for new RabbitMQ containers to update to RabbitMQ version 3.9.22. -upgrade: - - | - Tags are added to update the RabbitMQ containers to version 3.9.22. \ No newline at end of file From c9376b64c0b1139b970de0e6be485cfe4f10eb96 Mon Sep 17 00:00:00 2001 From: Rafal Lewandowski Date: Wed, 11 Jan 2023 16:42:55 +0100 Subject: [PATCH 180/292] fix pep8 syntax check (cherry picked from commit 7af94d9984d416f81f851a0459cfdd235a510135) --- etc/kayobe/ansible/rsyslog.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/etc/kayobe/ansible/rsyslog.yml b/etc/kayobe/ansible/rsyslog.yml index acc567eb7..3df575dec 100644 --- a/etc/kayobe/ansible/rsyslog.yml +++ b/etc/kayobe/ansible/rsyslog.yml @@ -11,26 +11,26 @@ become: yes tasks: - - name: Ensure rsyslog is installed - package: + - name: Ensure rsyslog is installed + package: name: rsyslog state: present - - name: Ensure rsyslog is started and enabled - systemd: + - name: Ensure rsyslog is started and enabled + systemd: state: started enabled: yes name: rsyslog - - name: Update rsyslog configuration - lineinfile: + - name: Update rsyslog configuration + lineinfile: path: /etc/rsyslog.conf insertafter: "^#*.* @@remote-host:514" line: "*.* @{{ internal_net_name | net_ip }}:5140" - register: rsyslog_config + register: rsyslog_config - - name: Restart rsyslog - systemd: + - name: Restart rsyslog + systemd: state: restarted name: rsyslog - when: rsyslog_config.changed + when: rsyslog_config.changed From 47c2fc5f23b9e44411ac909e2a9fbaefbcfb1bc2 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 11 Jan 2023 17:04:28 +0000 Subject: [PATCH 181/292] RabbitMQ: fix HA rollout docs --- doc/source/operations/rabbitmq.rst | 74 ++++++++++++++++++++++++------ 1 file changed, 59 insertions(+), 15 deletions(-) diff --git a/doc/source/operations/rabbitmq.rst b/doc/source/operations/rabbitmq.rst index 39751c5e4..a4540e8d0 100644 --- a/doc/source/operations/rabbitmq.rst +++ b/doc/source/operations/rabbitmq.rst @@ -39,6 +39,29 @@ state of RabbitMQ will also be reset. Instructions ------------ If you are planning to perform an upgrade, it is recommended to first roll out these changes. + +The configuration should be merged with StackHPC Kayobe configuration. If +bringing in the latest changes is not possible for some reason, you may cherry +pick the following changes: + +RabbitMQ hammer playbook (all releases): + +* ``3933e4520ba512b5bf095a28b791c0bac12c5dd0`` +* ``d83cceb2c41c18c2406032dac36cf90e57f37107`` +* ``097c98565dd6bd0eb16d49b87e4da7e2f2be3a5c`` + +RabbitMQ tags (Wallaby): + +* ``69c245dc91a2eb4d34590624760c32064c3ac07b`` + +RabbitMQ tags & HA flag (Xena): + +* ``2fd1590eb8ac739a07ad9cccbefc7725ea1a3855`` + +RabbitMQ HA flag (Yoga): + +* ``31406648544372187352e129d2a3b4f48498267c`` + If you are currently running Wallaby, you will need to enable the HA config option in ``etc/kayobe/kolla/globals.yml``. @@ -52,11 +75,18 @@ If you are running Wallaby or Xena, synchronise the Pulp containers. kayobe playbook run etc/kayobe/ansible/pulp-container-sync.yml pulp-container-publish.yml -e stackhpc_pulp_images_kolla_filter=rabbitmq -Generate the new config files for the overcloud services. +Ensure that Kolla Ansible is up to date. .. code-block:: console - kayobe overcloud service configuration generate + kayobe control host bootstrap + +Generate the new config files for the overcloud services. This ensures that +queues are created as durable. + +.. code-block:: console + + kayobe overcloud service configuration generate --node-config-dir /etc/kolla Pull the RabbitMQ container image. @@ -68,7 +98,7 @@ Stop all the OpenStack services which use RabbitMQ. .. code-block:: console - kayobe overcloud host command run --command "docker ps -a | egrep '(barbican|blazar|ceilometer|cinder|cloudkitty|designate|heat|ironic|keystone|magnum|manila|masakari|neutron|nova|octavia)' | awk '{ print $NF }' | xargs docker stop" + kayobe overcloud host command run --command "docker ps -a | egrep '(barbican|blazar|ceilometer|cinder|cloudkitty|designate|heat|ironic|keystone|magnum|manila|masakari|neutron|nova|octavia)' | awk '{ print \$NF }' | xargs docker stop" Upgrade RabbitMQ. @@ -77,33 +107,47 @@ Upgrade RabbitMQ. kayobe overcloud service upgrade -kt rabbitmq --skip-prechecks In order to convert the queues to be durable, you will need to reset the state -of RabbitMQ, and restart the services which use it. This can be done with the -RabbitMQ hammer playbook: +of RabbitMQ. This can be done with the RabbitMQ hammer playbook: .. code-block:: console - kayobe playbook run stackhpc-kayobe-config/etc/kayobe/ansible/rabbitmq-reset.yml + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/rabbitmq-reset.yml --skip-tags restart-openstack -The hammer playbook only targets the services which are known to have issues -when RabbitMQ breaks. You will still need to start the remaining services: +Check to see if RabbitMQ is functioning as expected. .. code-block:: console - kayobe overcloud host command run --command "docker ps -a | egrep '(barbican|blazar|ceilometer|cloudkitty|designate|manila|masakari|octavia)' | awk '{ print $NF }' | xargs docker start" + kayobe overcloud host command run --limit controllers --show-output --command 'docker exec rabbitmq rabbitmqctl cluster_status' -Check to see if RabbitMQ is functioning as expected. +The cluster status should list all controllers. + +Check to see if all OpenStack queues and exchanges have been removed from the RabbitMQ cluster. + +.. code-block:: console + + kayobe overcloud host command run --limit controllers --show-output --command 'docker exec rabbitmq rabbitmqctl list_queues name' + kayobe overcloud host command run --limit controllers --show-output --command 'docker exec rabbitmq rabbitmqctl list_exchanges name' + +Start the OpenStack services which use RabbitMQ. Note that this will start all +matching services, even if they weren't running prior to starting this +procedure. + +.. code-block:: console + + kayobe overcloud host command run --command "docker ps -a | egrep '(barbican|blazar|ceilometer|cinder|cloudkitty|designate|heat|ironic|keystone|magnum|manila|masakari|neutron|nova|octavia)' | awk '{ print \$NF }' | xargs docker start" + +Check to see if the expected queues are durable. .. code-block:: console - kayobe overcloud host command run --show-output --command 'docker exec rabbitmq rabbitmqctl cluster_status' - kayobe overcloud host command run --show-output --command 'docker exec rabbitmq rabbitmqctl list_queues name durable' + kayobe overcloud host command run --limit controllers --show-output --command 'docker exec rabbitmq rabbitmqctl list_queues name durable' -The cluster status should list all controllers. The queues listed should be -durable if their names do not start with the following: +The queues listed should be durable if their names do not start with the +following: * amq. * .\*\_fanout\_ * reply\_ If there are issues with the services after this, particularly during upgrades, -you may find it useful to reuse the hammer playbook. +you may find it useful to reuse the hammer playbook, ``rabbitmq-reset.yml``. From a5aee951689e24f37f67d4f452ffddd0af999b26 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 12 Jan 2023 14:04:22 +0000 Subject: [PATCH 182/292] docs: RabbitMQ HA: add known issues section --- doc/source/operations/rabbitmq.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/doc/source/operations/rabbitmq.rst b/doc/source/operations/rabbitmq.rst index a4540e8d0..f24231946 100644 --- a/doc/source/operations/rabbitmq.rst +++ b/doc/source/operations/rabbitmq.rst @@ -151,3 +151,25 @@ following: If there are issues with the services after this, particularly during upgrades, you may find it useful to reuse the hammer playbook, ``rabbitmq-reset.yml``. + +Known issues +------------ + +If there are any OpenStack services running without durable queues enabled +while the RabbitMQ cluster is reset, they are likely to create non-durable +queues before the other OpenStack services start. This leads to an error +such as the following when other OpenStack services start:: + + Unable to connect to AMQP server on :5672 after inf tries: + Exchange.declare: (406) PRECONDITION_FAILED - inequivalent arg 'durable' + for exchange 'neutron' in vhost '/': received 'true' but current is + 'false': amqp.exceptions.PreconditionFailed: Exchange.declare: (406) + PRECONDITION_FAILED - inequivalent arg 'durable' for exchange 'neutron' in + vhost '/': received 'true' but current is 'false' + +This may happen if a host is not in the inventory, leading to them not being +targeted by the ``docker stop`` command. If this does happen, look for the +hostname of the offending node in the queues created after the RabbitMQ reset. + +Once the rogue services have been stopped, reset the RabbitMQ cluster again to +clear the queues. From a0c0ab8397ba77e9ef85428e2de32639b621fee6 Mon Sep 17 00:00:00 2001 From: Matt Anson Date: Thu, 19 Jan 2023 15:41:17 +0000 Subject: [PATCH 183/292] Configure TLS on local Pulp --- etc/kayobe/containers/pulp/pre.yml | 21 +++++++++++++++++++++ etc/kayobe/pulp.yml | 16 ++++++++++++++-- etc/kayobe/seed.yml | 2 +- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/etc/kayobe/containers/pulp/pre.yml b/etc/kayobe/containers/pulp/pre.yml index 22d999023..bd40b2e12 100644 --- a/etc/kayobe/containers/pulp/pre.yml +++ b/etc/kayobe/containers/pulp/pre.yml @@ -19,3 +19,24 @@ dest: /opt/kayobe/containers/pulp/settings.py mode: 0644 become: true + +- name: Configure TLS for local Pulp + when: pulp_enable_tls + become: true + block: + - name: Ensure /opt/kayobe/containers/pulp/certs exists + file: + path: "/opt/kayobe/containers/pulp/certs" + state: directory + + - name: Copy TLS cert and key into container directory + template: + src: "{{ item.src }}" + dest: "/opt/kayobe/containers/pulp/certs/{{ item.dest }}" + mode: 0644 + become: true + loop: + - src: "{{ pulp_cert_path }}" + dest: 'pulp_webserver.crt' + - src: "{{ pulp_key_path }}" + dest: 'pulp_webserver.key' diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index f2a3d39da..028a079d2 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -1,10 +1,22 @@ --- ############################################################################### -# Local Pulp access credentials +# Local Pulp server configuration # Base URL of the local Pulp service. # Default uses the seed node's IP on the admin network. -pulp_url: "http://{{ admin_oc_net_name | net_ip(groups['seed'][0]) }}:80" +pulp_url: "{{ 'https' if pulp_enable_tls else 'http' }}://{{ admin_oc_net_name | net_ip(groups['seed'][0]) }}:80" + +# Whether to enable TLS for Pulp. +pulp_enable_tls: false + +# Path to a TLS certificate to use when TLS is enabled. +#pulp_cert_path: + +# Path to a TLS key to use when TLS is enabled. +#pulp_key_path: + +############################################################################### +# Local Pulp access credentials # Credentials used to access the local Pulp REST API. pulp_username: admin diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index 2d0784136..6d23b4383 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -96,7 +96,7 @@ seed_pulp_container: image: pulp/pulp pre: "{{ kayobe_config_path }}/containers/pulp/pre.yml" post: "{{ kayobe_config_path }}/containers/pulp/post.yml" - tag: "3.16" + tag: "{{ '3.16-https' if pulp_enable_tls else '3.16' }}" network_mode: host volumes: - /opt/kayobe/containers/pulp:/etc/pulp From 0ab0acb877595cf4516316c4f00ce0cd9d9b821d Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Thu, 29 Dec 2022 12:24:03 +0000 Subject: [PATCH 184/292] Sets kolla_docker_registry_insecure to true (#297) * Sets kolla_docker_registry_insecure to true We currently don't configure TLS for the the local pulp registry. This adds the pulp server to the list of insecure-registries, so that we can pull images. * Remove kolla_docker_registry_insecure override in CI environment Based on comments in code review. * Use a better default for kolla_docker_registry_insecure Hoping this will prevent breakage for people already running pulp with TLS. (cherry picked from commit 78055cf8e7d485154a1ff06e7dd1bd8f03e7fe00) --- etc/kayobe/kolla.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index bb649872b..9db98fab2 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -85,6 +85,11 @@ kolla_install_type: source # Docker namespace to use for Kolla images. Default is 'kolla'. kolla_docker_namespace: stackhpc +# Whether docker should be configured to use an insecure registry for Kolla +# images. Default is false, unless docker_registry_enabled is true and +# docker_registry_enable_tls is false. +kolla_docker_registry_insecure: "{{ 'https' not in stackhpc_repo_mirror_url }}" + # Username to use to access a docker registry. Default is not set, in which # case the registry will be used without authentication. kolla_docker_registry_username: "{{ stackhpc_docker_registry_username }}" From dc7c0e21c147c1ddfa1ff2bced35b15e4083fc4b Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Thu, 19 Jan 2023 16:22:00 +0000 Subject: [PATCH 185/292] Small fix to rabbit docs --- doc/source/operations/rabbitmq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/operations/rabbitmq.rst b/doc/source/operations/rabbitmq.rst index f24231946..e678fbed9 100644 --- a/doc/source/operations/rabbitmq.rst +++ b/doc/source/operations/rabbitmq.rst @@ -73,7 +73,7 @@ If you are running Wallaby or Xena, synchronise the Pulp containers. .. code-block:: console - kayobe playbook run etc/kayobe/ansible/pulp-container-sync.yml pulp-container-publish.yml -e stackhpc_pulp_images_kolla_filter=rabbitmq + kayobe playbook run etc/kayobe/ansible/pulp-container-sync.yml etc/kayobe/ansible/pulp-container-publish.yml -e stackhpc_pulp_images_kolla_filter=rabbitmq Ensure that Kolla Ansible is up to date. From 7189ed26d7dd955526665b03c7511ed0305d7737 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 20 Jan 2023 14:06:11 +0100 Subject: [PATCH 186/292] Bump nova tag for Xena --- etc/kayobe/kolla/globals.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index c2c7ce5c6..3cb843778 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -10,6 +10,7 @@ enable_docker_repo: {% raw %}"{{ 'overcloud' not in group_names or ansible_facts {% if kolla_base_distro == 'centos' %} bifrost_tag: xena-20221128T101757 magnum_tag: xena-20230111T093652 +nova_tag: xena-20230120T115956 rabbitmq_tag: xena-20230105T141714 {% else %} bifrost_tag: xena-20221213T224057 From 11531602e897cbe663d79fea23e53510d88ec23a Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Tue, 24 Jan 2023 12:16:47 +0000 Subject: [PATCH 187/292] adding alert for nf_conntrack --- etc/kayobe/kolla/config/prometheus/system.rules | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/etc/kayobe/kolla/config/prometheus/system.rules b/etc/kayobe/kolla/config/prometheus/system.rules index ffc7d25a3..532710fd3 100644 --- a/etc/kayobe/kolla/config/prometheus/system.rules +++ b/etc/kayobe/kolla/config/prometheus/system.rules @@ -78,4 +78,12 @@ groups: summary: "Host EDAC Uncorrectable Errors detected (instance {{ $labels.instance }})" description: "{{ $labels.instance }} has had {{ printf \"%.0f\" $value }} uncorrectable memory errors reported by EDAC in the last 5 minutes." + - alert: HostConntrackLimit + expr: node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8 + for: 5m + labels: + severity: warning + annotations: + summary: Host conntrack limit (instance {{ $labels.instance }}) + description: "The number of conntrack is approaching limit" {% endraw %} From cd3208d366d1de08d4ff0227b9ddcb03c249cad1 Mon Sep 17 00:00:00 2001 From: Matt Anson Date: Tue, 24 Jan 2023 14:37:43 +0000 Subject: [PATCH 188/292] Cast pulp_enable_tls to bool everywhere --- etc/kayobe/containers/pulp/pre.yml | 2 +- etc/kayobe/seed.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/containers/pulp/pre.yml b/etc/kayobe/containers/pulp/pre.yml index bd40b2e12..85af37916 100644 --- a/etc/kayobe/containers/pulp/pre.yml +++ b/etc/kayobe/containers/pulp/pre.yml @@ -21,7 +21,7 @@ become: true - name: Configure TLS for local Pulp - when: pulp_enable_tls + when: pulp_enable_tls | bool become: true block: - name: Ensure /opt/kayobe/containers/pulp/certs exists diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index 6d23b4383..46d20fc7e 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -96,7 +96,7 @@ seed_pulp_container: image: pulp/pulp pre: "{{ kayobe_config_path }}/containers/pulp/pre.yml" post: "{{ kayobe_config_path }}/containers/pulp/post.yml" - tag: "{{ '3.16-https' if pulp_enable_tls else '3.16' }}" + tag: "{{ '3.16-https' if pulp_enable_tls | bool else '3.16' }}" network_mode: host volumes: - /opt/kayobe/containers/pulp:/etc/pulp From 6903947c151f9b1054387d426666a74c6fc1dd6d Mon Sep 17 00:00:00 2001 From: Matt Anson Date: Tue, 24 Jan 2023 14:38:08 +0000 Subject: [PATCH 189/292] Expose pulp_port variable --- etc/kayobe/pulp.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index 028a079d2..4f786f8ba 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -4,7 +4,11 @@ # Base URL of the local Pulp service. # Default uses the seed node's IP on the admin network. -pulp_url: "{{ 'https' if pulp_enable_tls else 'http' }}://{{ admin_oc_net_name | net_ip(groups['seed'][0]) }}:80" +pulp_url: "{{ 'https' if pulp_enable_tls | bool else 'http' }}://{{ admin_oc_net_name | net_ip(groups['seed'][0]) }}:{{ pulp_port }}" + +# Port on the seed node's interface on the admin network that the Pulp service +# listens on. +pulp_port: "{{ '443' if pulp_enable_tls | bool else '80' }}" # Whether to enable TLS for Pulp. pulp_enable_tls: false From 7d6a6cb614b37534798863671608d33d2e115500 Mon Sep 17 00:00:00 2001 From: Matt Anson Date: Tue, 24 Jan 2023 14:47:48 +0000 Subject: [PATCH 190/292] Add releasenote --- releasenotes/notes/local-pulp-tls-a8e7464d8cb0d114.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 releasenotes/notes/local-pulp-tls-a8e7464d8cb0d114.yaml diff --git a/releasenotes/notes/local-pulp-tls-a8e7464d8cb0d114.yaml b/releasenotes/notes/local-pulp-tls-a8e7464d8cb0d114.yaml new file mode 100644 index 000000000..a6340bca7 --- /dev/null +++ b/releasenotes/notes/local-pulp-tls-a8e7464d8cb0d114.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Enable TLS for the Seed Pulp service. Set ``pulp_enable_tls: true`` and + provide paths to a TLS certificate and key using ``pulp_cert_path`` and + ``pulp_key_path`` respectively. From 8e89ddbd772c03ebab721fecbeb8072ed9aaf11a Mon Sep 17 00:00:00 2001 From: Matt Anson Date: Wed, 25 Jan 2023 18:57:59 +0000 Subject: [PATCH 191/292] Add timing information to tasks --- etc/kayobe/ansible.cfg | 2 ++ .../notes/ansible-profile-tasks-3d341727a39dadcb.yaml | 5 +++++ 2 files changed, 7 insertions(+) create mode 100644 releasenotes/notes/ansible-profile-tasks-3d341727a39dadcb.yaml diff --git a/etc/kayobe/ansible.cfg b/etc/kayobe/ansible.cfg index 696ed923a..310c79499 100644 --- a/etc/kayobe/ansible.cfg +++ b/etc/kayobe/ansible.cfg @@ -6,6 +6,8 @@ stdout_callback = yaml bin_ansible_callbacks = True # Disable fact variable injection to improve performance. inject_facts_as_vars = False +# Add timing information to output +callback_whitelist = ansible.posix.profile_tasks [ssh_connection] pipelining = True diff --git a/releasenotes/notes/ansible-profile-tasks-3d341727a39dadcb.yaml b/releasenotes/notes/ansible-profile-tasks-3d341727a39dadcb.yaml new file mode 100644 index 000000000..7caa578cc --- /dev/null +++ b/releasenotes/notes/ansible-profile-tasks-3d341727a39dadcb.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Adds time information to tasks using the `ansible.posix.profile_tasks` + callback. From e4f7154c9743c0a6624e549e23b55c2a6e470cda Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Wed, 25 Jan 2023 22:28:57 +0100 Subject: [PATCH 192/292] Fix name of RabbitMQ HA flag in release note --- ...bitmq-ha-and-update-rabbitmq-version-8eaa68b2a9486320.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/releasenotes/notes/enable-rabbitmq-ha-and-update-rabbitmq-version-8eaa68b2a9486320.yaml b/releasenotes/notes/enable-rabbitmq-ha-and-update-rabbitmq-version-8eaa68b2a9486320.yaml index 8a6bc5023..bced4f825 100644 --- a/releasenotes/notes/enable-rabbitmq-ha-and-update-rabbitmq-version-8eaa68b2a9486320.yaml +++ b/releasenotes/notes/enable-rabbitmq-ha-and-update-rabbitmq-version-8eaa68b2a9486320.yaml @@ -1,11 +1,11 @@ --- features: - | - The flag ``om_enable_rabbitmq_high-availability`` is now set to ``true``. + The flag ``om_enable_rabbitmq_high_availability`` is now set to ``true``. Adds tags for new RabbitMQ containers to update to RabbitMQ version 3.9.22. upgrade: - | - The flag ``om_enable_rabbitmq_high-availability`` is now set to ``true``. + The flag ``om_enable_rabbitmq_high_availability`` is now set to ``true``. As this enables durable queues, RabbitMQ will need to be reset, and the services which use it restarted. Tags are added to update the RabbitMQ containers to version 3.9.22. From f59de341d976dda62d2d92eafb163060216c9a93 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Wed, 25 Jan 2023 22:31:42 +0100 Subject: [PATCH 193/292] Fix typo in release note --- releasenotes/notes/xena-batch-bc1da4e4d0f6257e.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/releasenotes/notes/xena-batch-bc1da4e4d0f6257e.yaml b/releasenotes/notes/xena-batch-bc1da4e4d0f6257e.yaml index 1c9446dd5..144f03a55 100644 --- a/releasenotes/notes/xena-batch-bc1da4e4d0f6257e.yaml +++ b/releasenotes/notes/xena-batch-bc1da4e4d0f6257e.yaml @@ -1,7 +1,7 @@ --- features: - | - Adds a custom playbook to run the `Anomoly Detection Visualiser (ADVise) + Adds a custom playbook to run the `Anomaly Detection Visualiser (ADVise) `_, ``advise-run.yml``. - | Adds a custom playbook to reset the RabbitMQ cluster and restart OpenStack From 76f580350879e343f6c006be19636cb880506e98 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 31 Jan 2023 14:14:52 +0000 Subject: [PATCH 194/292] Fix growpart.yml custom playbook in Ansible check mode --- etc/kayobe/ansible/growroot.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etc/kayobe/ansible/growroot.yml b/etc/kayobe/ansible/growroot.yml index f1367df4c..1e9f4d983 100644 --- a/etc/kayobe/ansible/growroot.yml +++ b/etc/kayobe/ansible/growroot.yml @@ -34,6 +34,7 @@ cmd: type growpart changed_when: false failed_when: false + check_mode: false register: growpart_check become: true @@ -51,6 +52,7 @@ register: pvs become: True changed_when: False + check_mode: false - name: Fail if root PV device not found fail: From bd9118632a0e783cade487099b0dd8e84a7de25c Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 31 Jan 2023 15:15:19 +0000 Subject: [PATCH 195/292] Multinode xena (#248) * feat: copy `ci-multinode` from PR `#246` * feat: copy `ansible` from PR `#246` * revert: configure-aio-resources.yml to `da05599` * feat: actually copy contents of `ci-multinode` For some reason the previous commit does not contain the important changes to the configuration. This commit fixes that. * feat: include `.automation.conf` from PR `#246` * revert: `stackhpc-ci` back to `stackhpc/xena` * feat: update address of `pulp-server` * feat: bump `stackhpc.cephadm` version * feat: `haproxy` group has been replaced with `loadbalancer` * fix: ensure `cephadm_host_labels` is defined * feat: reduce concurrency and remove load list * feat: increase `max_microversion` * fix: restore order to `requirements.yml` * fix: README has been moved * fix: remove unused collection * fix: add missing new line * feat: disable some unwanted service within ci-multinode * fix: remove extra blank lines * feat: revert `TEMPEST_CONCURRENCY` to two * feat: load `ci-multinode` tempest settings conditionally Firstly, the skip-list has been renamed from `tempest-full` to `ci-multinode` as this list would be environment specific. Secondly the `config.sh` script now contains a conditional for the `ci-multinode` environment allowing for the test list and concurrency settings to be loaded without impacting other environments. * fix: use `HTTP` dashboard URL over `HTTPS` * fix: reduce multinode tempest concurrency * feat: bump `stackhpc.cephadm` collection version * feat: remove unecessary comment --- .automation.conf/config.sh | 4 + .../tempest/load-lists/tempest-full | 1601 +++++++++++++++++ .../tempest/skip-lists/ci-multinode | 1 + .../tempest-ci-multinode.overrides.conf | 30 + etc/kayobe/ansible/configure-vxlan.yml | 9 +- etc/kayobe/ansible/fix-hostname.yml | 23 + etc/kayobe/ansible/fix-networking.yml | 21 + etc/kayobe/ansible/requirements.yml | 2 +- .../environments/ci-multinode/compute.yml | 1 + .../environments/ci-multinode/controllers.yml | 1 + .../environments/ci-multinode/growroot.yml | 2 + .../inventory/group_vars/all/main.yml | 2 + .../inventory/group_vars/all/vxlan.yml | 18 + .../inventory/group_vars/compute/lvm.yml | 34 + .../group_vars/compute/network-interfaces | 13 +- .../inventory/group_vars/controllers/lvm.yml | 34 + .../group_vars/controllers/network-interfaces | 13 +- .../inventory/group_vars/seed/lvm.yml | 34 + .../group_vars/seed/network-interfaces | 14 +- .../group_vars/storage/{lvm => lvm.yml} | 11 + .../group_vars/storage/network-interfaces | 13 +- .../ci-multinode/inventory/kayobe-automation | 4 +- .../environments/ci-multinode/kolla.yml | 12 + .../kolla/config/keepalived/keepalived.conf | 48 + .../ci-multinode/kolla/globals.yml | 12 + .../environments/ci-multinode/networks.yml | 86 +- .../environments/ci-multinode/neutron.yml | 3 + etc/kayobe/environments/ci-multinode/seed.yml | 10 + .../environments/ci-multinode/storage.yml | 1 + .../notes/cephadm-bump-294148e93121da48.yaml | 4 + 30 files changed, 2000 insertions(+), 61 deletions(-) create mode 100644 .automation.conf/tempest/load-lists/tempest-full create mode 100644 .automation.conf/tempest/skip-lists/ci-multinode create mode 100644 .automation.conf/tempest/tempest-ci-multinode.overrides.conf create mode 100644 etc/kayobe/ansible/fix-hostname.yml create mode 100644 etc/kayobe/ansible/fix-networking.yml create mode 100644 etc/kayobe/environments/ci-multinode/growroot.yml create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/all/main.yml create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/all/vxlan.yml create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/lvm.yml create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/lvm.yml create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/lvm.yml rename etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/{lvm => lvm.yml} (60%) create mode 100644 etc/kayobe/environments/ci-multinode/kolla/config/keepalived/keepalived.conf create mode 100644 etc/kayobe/environments/ci-multinode/neutron.yml create mode 100644 etc/kayobe/environments/ci-multinode/seed.yml create mode 100644 releasenotes/notes/cephadm-bump-294148e93121da48.yaml diff --git a/.automation.conf/config.sh b/.automation.conf/config.sh index d5766fce9..e8b88728b 100644 --- a/.automation.conf/config.sh +++ b/.automation.conf/config.sh @@ -19,6 +19,10 @@ if [ ! -z ${KAYOBE_ENVIRONMENT:+x} ]; then # Seem to get servers failing to spawn with higher concurrency export TEMPEST_CONCURRENCY=1 fi + if [[ "$KAYOBE_ENVIRONMENT" =~ "ci-multinode" ]]; then + export KAYOBE_AUTOMATION_TEMPEST_LOADLIST=tempest-full + export KAYOBE_AUTOMATION_TEMPEST_SKIPLIST=ci-multinode + fi fi if [[ -z "${KAYOBE_AUTOMATION_TEMPEST_CONF_OVERRIDES:+x}" ]] || [[ ! -e "${KAYOBE_AUTOMATION_TEMPEST_CONF_OVERRIDES}" ]]; then diff --git a/.automation.conf/tempest/load-lists/tempest-full b/.automation.conf/tempest/load-lists/tempest-full new file mode 100644 index 000000000..c063d803e --- /dev/null +++ b/.automation.conf/tempest/load-lists/tempest-full @@ -0,0 +1,1601 @@ +tempest.api.object_storage.test_object_slo.ObjectSloTest.test_upload_manifest[id-2c3f24a6-36e8-4711-9aa2-800ee1fc7b5b] +tempest.api.compute.servers.test_server_rescue_negative.ServerRescueNegativeTestJSON.test_rescued_vm_reboot[id-db22b618-f157-4566-a317-1b6d467a8094,negative] +tempest.api.network.test_networks.NetworksTest.test_list_networks_fields[id-6ae6d24f-9194-4869-9c85-c313cb20e080] +tempest.api.compute.admin.test_flavors_extra_specs.FlavorsExtraSpecsTestJSON.test_flavor_set_get_update_show_unset_keys[id-0b2f9d4b-1ca2-4b99-bb40-165d4bb94208] +tempest.api.compute.servers.test_servers.ServerShowV247Test.test_show_server[id-88b0bdb2-494c-11e7-a919-92ebcb67fe33] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_assign_user_role_for_non_existent_tenant[id-b2285aaa-9e76-4704-93a9-7a8acd0a6c8f,negative] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_sort_name_asc[id-d58b5fed-0c37-42d3-8c5d-39014ac13c00] +tempest.api.identity.admin.v3.test_users.UsersV3TestJSON.test_user_update[id-b537d090-afb9-4519-b95d-270b0708e87e] +tempest.api.compute.admin.test_quotas_negative.QuotasAdminNegativeTest.test_update_quota_normal_user[id-733abfe8-166e-47bb-8363-23dbd7ff3476,negative] +tempest.api.image.v2.test_images.ListSharedImagesTest.test_list_images_param_member_status[id-3fa50be4-8e38-4c02-a8db-7811bb780122] +tempest.api.compute.volumes.test_attach_volume.AttachVolumeMultiAttachTest.test_snapshot_volume_backed_multiattach[id-885ac48a-2d7a-40c5-ae8b-1993882d724c,image] +tempest.scenario.test_network_v6.TestGettingAddress.test_dualnet_multi_prefix_dhcpv6_stateless[compute,id-cf1c4425-766b-45b8-be35-e2959728eb00,network,slow] +tempest.api.network.test_networks_negative.NetworksNegativeTestJSON.test_update_non_existent_port[id-cf8eef21-4351-4f53-adcd-cc5cb1e76b92,negative] +tempest.api.volume.admin.test_volume_types_negative.VolumeTypesNegativeTest.test_create_with_repeated_name[id-969b10c7-3d77-4e1b-a4f2-2d265980f7e5,negative] +tempest.api.volume.admin.test_volume_type_access.VolumeTypesAccessTest.test_volume_type_access_list[id-5220eb28-a435-43ce-baaf-ed46f0e95159] +tempest.api.identity.admin.v2.test_users.UsersTestJSON.test_update_user_password[id-1aeb25ac-6ec5-4d8b-97cb-7ac3567a989f] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_list_roles[id-f5654bcc-08c4-4f71-88fe-05d64e06de94] +tempest.api.network.test_security_groups.SecGroupIPv6Test.test_create_security_group_rule_with_remote_group_id[id-c2ed2deb-7a0c-44d8-8b4c-a5825b5c310b] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_list_all_implied_roles[id-3748c316-c18f-4b08-997b-c60567bc6235] +tempest.api.compute.admin.test_aggregates.AggregatesAdminTestJSON.test_aggregate_add_remove_host[id-c8e85064-e79b-4906-9931-c11c24294d02] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_get_details_as_user[id-557cad12-34c9-4ff4-95f0-22f0dfbaf7dc,negative] +tempest.api.network.test_routers.RoutersIpV6Test.test_update_delete_extra_route[id-c86ac3a8-50bd-4b00-a6b8-62af84a0765c] +tempest.api.image.v2.test_images.ImportImagesTest.test_remote_delete[id-44d60544-1524-42f7-8899-315301105dd8] +tempest.scenario.test_shelve_instance.TestShelveInstance.test_cold_migrate_unshelved_instance[compute,id-1295fd9e-193a-4cf8-b211-55358e021bae,image,network,slow] +tempest.api.volume.admin.test_qos.QosSpecsTestJSON.test_associate_disassociate_qos[id-1dd93c76-6420-485d-a771-874044c416ac] +tempest.api.object_storage.test_container_services.ContainerTest.test_create_container_with_remove_metadata_value[id-8a21ebad-a5c7-4e29-b428-384edc8cd156] +tempest.api.network.admin.test_routers_dvr.RoutersTestDVR.test_centralized_router_update_to_dvr[id-acd43596-c1fb-439d-ada8-31ad48ae3c2e] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_get_users_by_unauthorized_user[id-284192ce-fb7c-4909-a63b-9a502e0ddd11,negative] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_container_format[id-9959ca1d-1aa7-4b7a-a1ea-0fff0499b37e] +tempest.api.network.test_networks.NetworksIpV6Test.test_list_subnets_fields[id-842589e3-9663-46b0-85e4-7f01273b0412] +tempest.api.object_storage.test_container_services_negative.ContainerNegativeTest.test_create_container_metadata_value_exceeds_max_length[id-81e36922-326b-4b7c-8155-3bbceecd7a82,negative] +tempest.api.identity.v3.test_ec2_credentials.EC2CredentialsTest.test_list_ec2_credentials[id-897813f0-160c-4fdc-aabc-24ee635ce4a9] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcp_stateful[id-4ab211a0-276f-4552-9070-51e27f58fecf] +tempest.api.compute.admin.test_quotas.QuotasAdminTestJSON.test_update_all_quota_resources_for_tenant[id-55fbe2bf-21a9-435b-bbd2-4162b0ed799a] +tempest.api.object_storage.test_container_services_negative.ContainerNegativeTest.test_delete_non_empty_container[id-42da116e-1e8c-4c96-9e06-2f13884ed2b1,negative] +tempest.api.object_storage.test_object_temp_url_negative.ObjectTempUrlNegativeTest.test_get_object_after_expiration_time[id-5a583aca-c804-41ba-9d9a-e7be132bdf0b,negative] +tempest.api.object_storage.test_object_services.ObjectTest.test_get_object_with_range[id-05a1890e-7db9-4a6c-90a8-ce998a2bddfa] +tempest.api.identity.v3.test_tokens.TokensV3Test.test_validate_token[id-a9512ac3-3909-48a4-b395-11f438e16260] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_create_update_delete_port[id-c72c1c0c-2193-4aca-aaa4-b1442640f51c,smoke] +tempest.api.compute.servers.test_instance_actions_negative.InstanceActionsNegativeTestJSON.test_get_instance_action_invalid_request[id-0269f40a-6f18-456c-b336-c03623c897f1,negative] +tempest.api.network.test_ports.PortsTestJSON.test_list_ports_fields[id-ff7f117f-f034-4e0e-abff-ccef05c454b4] +tempest.scenario.test_network_v6.TestGettingAddress.test_dualnet_multi_prefix_slaac[compute,id-9178ad42-10e4-47e9-8987-e02b170cc5cd,network,slow] +tempest.api.compute.servers.test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_metadata_items_limit[id-d8c0a210-a5c3-4664-be04-69d96746b547,negative] +tempest.api.network.admin.test_metering_extensions.MeteringIpV6TestJSON.test_create_delete_metering_label_rule_with_filters[id-f4d547cd-3aee-408f-bf36-454f8825e045] +tempest.api.compute.admin.test_live_migration.LiveAutoBlockMigrationV225Test.test_live_block_migration_paused[id-1e107f21-61b2-4988-8f22-b196e938ab88] +tempest.api.volume.test_versions.VersionsTest.test_show_version[id-7f755ae2-caa9-4049-988c-331d8f7a579f] +tempest.api.compute.admin.test_quotas.QuotasAdminTestJSON.test_get_updated_quotas[id-ce9e0815-8091-4abd-8345-7fe5b85faa1d] +tempest.api.volume.test_snapshot_metadata.SnapshotMetadataTestJSON.test_crud_snapshot_metadata[id-a2f20f99-e363-4584-be97-bc33afb1a56c] +tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_port[id-48037ff2-e889-4c3b-b86a-8e3f34d2d060,smoke] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_end_marker[id-5ca164e4-7bde-43fa-bafb-913b53b9e786] +tempest.api.identity.admin.v3.test_domain_configuration.DomainConfigurationTestJSON.test_create_domain_config_and_show_config_groups_and_options[id-9e3ff13c-f597-4f01-9377-d6c06c2a1477] +tempest.api.network.admin.test_ports.PortsAdminExtendedAttrsTestJSON.test_show_port_binding_ext_attr[id-b54ac0ff-35fc-4c79-9ca3-c7dbd4ea4f13] +tempest.api.object_storage.test_container_services.ContainerTest.test_update_container_metadata_with_create_metadata[id-2ae5f295-4bf1-4e04-bfad-21e54b62cec5] +tempest.api.compute.security_groups.test_security_group_rules.SecurityGroupRulesTestJSON.test_security_group_rules_create[id-850795d7-d4d3-4e55-b527-a774c0123d3a,smoke] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_assignments_for_implied_roles_create_delete[id-c8828027-df48-4021-95df-b65b92c7429e] +tempest.api.volume.admin.test_encrypted_volumes_extend.EncryptedVolumesExtendAttachedTest.test_extend_attached_encrypted_volume_luksv1[compute,id-e93243ec-7c37-4b5b-a099-ebf052c13216] +tempest.api.compute.keypairs.test_keypairs_negative.KeyPairsNegativeTestJSON.test_create_keypair_with_empty_public_key[id-dade320e-69ca-42a9-ba4a-345300f127e0,negative] +tempest.api.image.v1.test_images.ListImagesTest.test_index_container_format[id-2143655d-96d9-4bec-9188-8674206b4b3b] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_delete_users_by_unauthorized_user[id-d45195d5-33ed-41b9-a452-7d0d6a00f6e9,negative] +tempest.api.identity.admin.v3.test_domains_negative.DomainsNegativeTestJSON.test_create_domain_with_name_length_over_64[id-37b1bbf2-d664-4785-9a11-333438586eae,negative] +tempest.api.identity.admin.v3.test_services.ServicesTestJSON.test_list_services[id-e55908e8-360e-439e-8719-c3230a3e179e] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents[id-312ff6bd-5290-497f-bda1-7c5fec6697ab,smoke] +tempest.api.compute.admin.test_simple_tenant_usage_negative.TenantUsagesNegativeTestJSON.test_get_usage_tenant_with_empty_tenant_id[id-8b21e135-d94b-4991-b6e9-87059609c8ed,negative] +tempest.api.identity.admin.v2.test_roles.RolesTestJSON.test_remove_user_role[id-f0b9292c-d3ba-4082-aa6c-440489beef69] +tempest.api.network.test_routers.RoutersIpV6Test.test_create_show_list_update_delete_router[id-f64403e2-8483-4b34-8ccd-b09a87bcc68c,smoke] +tempest.api.compute.volumes.test_volumes_list.VolumesTestJSON.test_volume_list_with_detail_param_offset_and_limit[id-06b6abc4-3f10-48e9-a7a1-3facc98f03e5] +tempest.api.object_storage.test_object_version.ContainerTest.test_versioned_container[id-a151e158-dcbf-4a1f-a1e7-46cd65895a6f] +tempest.api.compute.admin.test_auto_allocate_network.AutoAllocateNetworkTest.test_server_multi_create_auto_allocate[id-2e6cf129-9e28-4e8a-aaaa-045ea826b2a6] +tempest.api.object_storage.test_account_quotas_negative.AccountQuotasNegativeTest.test_user_modify_quota[id-d1dc5076-555e-4e6d-9697-28f1fe976324,negative] +tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_resize[compute,id-719eb59d-2f42-4b66-b8b1-bb1254473967,network,slow] +tempest.api.network.admin.test_metering_extensions.MeteringTestJSON.test_list_metering_labels[id-e2fb2f8c-45bf-429a-9f17-171c70444612] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_implied_roles_create_check_show_delete[id-c90c316c-d706-4728-bcba-eb1912081b69] +tempest.api.object_storage.test_container_acl_negative.ObjectACLsNegativeTest.test_read_object_without_rights[id-9ed01334-01e9-41ea-87ea-e6f465582823,negative] +tempest.api.compute.images.test_images_oneserver.ImagesOneServerTestJSON.test_create_delete_image[id-3731d080-d4c5-4872-b41a-64d0d0021314] +tempest.api.compute.servers.test_create_server.ServersTestBootFromVolume.test_list_servers_with_detail[id-585e934c-448e-43c4-acbf-d06a9b899997] +tempest.api.compute.volumes.test_volumes_negative.VolumesNegativeTest.test_volume_delete_nonexistent_volume_id[id-54a34226-d910-4b00-9ef8-8683e6c55846,negative] +tempest.api.volume.admin.test_volume_quota_classes.VolumeQuotaClassesTest.test_update_default_quota[id-a7644c63-2669-467a-b00e-452dd5c5397b] +tempest.api.volume.test_volume_metadata.VolumesMetadataTest.test_update_show_volume_metadata_item[id-862261c5-8df4-475a-8c21-946e50e36a20] +tempest.api.network.test_routers_negative.RoutersNegativeIpV6Test.test_delete_non_existent_router_returns_404[id-c7edc5ad-d09d-41e6-a344-5c0c31e2e3e4,negative] +tempest.api.volume.admin.test_snapshots_actions.SnapshotsActionsTest.test_reset_snapshot_status[id-3e13ca2f-48ea-49f3-ae1a-488e9180d535] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_delete_with_invalid_id[id-c74f4bf1-4708-4ff2-95a0-f49eaca951bd,negative] +tempest.api.image.v1.test_images.CreateRegisterImagesTest.test_register_remote_image[id-69da74d9-68a9-404b-9664-ff7164ccb0f5] +tempest.api.network.test_routers.RoutersIpV6Test.test_add_remove_router_interface_with_subnet_id[id-b42e6e39-2e37-49cc-a6f4-8467e940900a,smoke] +tempest.api.volume.admin.test_volume_services_negative.VolumeServicesNegativeTest.test_disable_service_with_invalid_binary[id-c571f179-c6e6-4c50-a0ab-368b628a8ac1,negative] +tempest.api.network.test_networks.NetworksIpV6Test.test_show_network_fields[id-867819bb-c4b6-45f7-acf9-90edcf70aa5e] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_delete_non_existent_user[id-7cc82f7e-9998-4f89-abae-23df36495867,negative] +tempest.api.image.v2.test_images_metadefs_schema.MetadataSchemaTest.test_get_metadata_objects_schema[id-dee4a891-b38b-3bf0-a3b2-e03ee67b3a3c] +tempest.api.object_storage.test_container_acl_negative.ObjectACLsNegativeTest.test_delete_object_without_write_rights[id-b4e366f8-f185-47ab-b789-df4416f9ecdb,negative] +tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_network[id-d4f9024d-1e28-4fc1-a6b1-25dbc6fa11e2,smoke] +tempest.api.object_storage.test_object_temp_url.ObjectTempUrlTest.test_get_object_using_temp_url_with_inline_query_parameter[id-9d9cfd90-708b-465d-802c-e4a8090b823d] +tempest.api.volume.test_volumes_clone.VolumesCloneTest.test_create_from_bootable_volume[id-cbbcd7c6-5a6c-481a-97ac-ca55ab715d16,image] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers[id-3499406a-ae53-4f8c-b43a-133d4dc6fe3f,smoke] +tempest.api.compute.admin.test_aggregates.AggregatesAdminTestJSON.test_aggregate_create_verify_entry_in_list[id-68089c38-04b1-4758-bdf0-cf0daec4defd] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_port_list_filter_by_ip_substr[id-79895408-85d5-460d-94e7-9531c5fd9123] +tempest.api.volume.admin.test_group_snapshots.GroupSnapshotsTest.test_group_snapshot_create_show_list_delete[id-1298e537-f1f0-47a3-a1dd-8adec8168897] +tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_update_metadata_empty_body[id-0f58d402-e34a-481d-8af8-b392b17426d9] +tempest.api.object_storage.test_crossdomain.CrossdomainTest.test_get_crossdomain_policy[id-d1b8b031-b622-4010-82f9-ff78a9e915c7] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_create_with_existent_aggregate_name[id-9c23a291-b0b1-487b-b464-132e061151b3,negative] +tempest.api.volume.admin.test_volume_quotas.VolumeQuotasAdminTestJSON.test_quota_usage[id-ae8b6091-48ad-4bfa-a188-bbf5cc02115f] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_suspend_non_existent_server[id-d1f032d5-7b6e-48aa-b252-d5f16dd994ca,negative] +tempest.api.identity.v3.test_access_rules.AccessRulesV3Test.test_delete_access_rule[id-278757e9-e193-4bf8-adf2-0b0a229a17d0] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_volume_extend_with_size_smaller_than_original_size[id-e0c75c74-ee34-41a9-9288-2a2051452854,negative] +tempest.api.compute.admin.test_hosts.HostsAdminTestJSON.test_list_hosts_with_zone[id-5dc06f5b-d887-47a2-bb2a-67762ef3c6de] +tempest.api.identity.v3.test_users.IdentityV3UsersTest.test_user_account_lockout[id-a7ad8bbf-2cff-4520-8c1d-96332e151658] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_create_with_invalid_group_description[id-777b6f14-aca9-4758-9e84-38783cfa58bc,negative] +tempest.api.identity.v2.test_api_discovery.TestApiDiscovery.test_api_version_statuses[id-77fd6be0-8801-48e6-b9bf-38cdd2f253ec,smoke] +tempest.api.compute.admin.test_simple_tenant_usage_negative.TenantUsagesNegativeTestJSON.test_get_usage_tenant_with_invalid_date[id-4079dd2a-9e8d-479f-869d-6fa985ce45b6,negative] +tempest.api.volume.admin.test_volume_types_negative.VolumeTypesNegativeTest.test_delete_nonexistent_type_id[id-6b3926d2-7d73-4896-bc3d-e42dfd11a9f6,negative] +tempest.api.volume.test_volumes_snapshots.VolumesSnapshotTestJSON.test_snapshot_create_offline_delete_online[compute,id-5210a1de-85a0-11e6-bb21-641c676a5d61] +tempest.api.image.v2.admin.test_images_metadefs_namespace_tags.MetadataNamespaceTagsTest.test_create_update_delete_tag[id-a2a3765e-1a2c-3f6d-a3a7-3cc3466ab875] +tempest.api.object_storage.test_object_services.ObjectTest.test_create_object_with_expect_continue[id-84dafe57-9666-4f6d-84c8-0814d37923b8] +tempest.api.network.test_networks_negative.NetworksNegativeTestJSON.test_create_port_on_non_existent_network[id-13d3b106-47e6-4b9b-8d53-dae947f092fe,negative] +tempest.api.compute.servers.test_instance_actions.InstanceActionsTestJSON.test_list_instance_actions[id-77ca5cc5-9990-45e0-ab98-1de8fead201a] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents_with_format_json[id-196f5034-6ab0-4032-9da9-a937bbb9fba9] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_delete_security_group_without_passing_id[id-1438f330-8fa4-4aeb-8a94-37c250106d7f,negative] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_delete_deleted_image[id-e57fc127-7ba0-4693-92d7-1d8a05ebcba9,negative] +tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_subnet[id-8936533b-c0aa-4f29-8e53-6cc873aec489,smoke] +tempest.api.network.admin.test_routers.RoutersIpV6AdminTest.test_create_router_with_default_snat_value[id-847257cc-6afd-4154-b8fb-af49f5670ce8] +tempest.api.volume.admin.test_groups.GroupsV320Test.test_reset_group_status[id-b20c696b-0cbc-49a5-8b3a-b1fb9338f45c] +tempest.api.network.admin.test_quotas.QuotasTest.test_quotas[id-2390f766-836d-40ef-9aeb-e810d78207fb] +tempest.api.identity.admin.v3.test_projects_negative.ProjectsNegativeStaticTestJSON.test_create_project_by_unauthorized_user[id-8fba9de2-3e1f-4e77-812a-60cb68f8df13,negative] +tempest.api.identity.admin.v2.test_tenant_negative.TenantsNegativeTestJSON.test_tenant_update_request_without_token[id-7a421573-72c7-4c22-a98e-ce539219c657,negative] +tempest.api.volume.admin.test_volume_services.VolumesServicesTestJSON.test_get_service_by_volume_host_name[id-67ec6902-f91d-4dec-91fa-338523208bbc] +tempest.api.network.test_allowed_address_pair.AllowedAddressPairTestJSON.test_update_port_with_cidr_address_pair[id-4d6d178f-34f6-4bff-a01c-0a2f8fe909e4] +tempest.api.volume.admin.test_volume_services.VolumesServicesTestJSON.test_list_services[id-e0218299-0a59-4f43-8b2b-f1c035b3d26d] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_create_user_with_name_length_over_255[id-7704b4f3-3b75-4b82-87cc-931d41c8f780,negative] +tempest.api.volume.test_volumes_backup.VolumesBackupsTest.test_backup_create_attached_volume[compute,id-07af8f6d-80af-44c9-a5dc-c8427b1b62e6] +tempest.api.identity.admin.v3.test_domains.DomainsTestJSON.test_create_domain_with_disabled_status[id-036df86e-bb5d-42c0-a7c2-66b9db3a6046] +tempest.api.identity.admin.v3.test_trusts.TrustsV3TestJSON.test_get_trusts_all[id-4773ebd5-ecbf-4255-b8d8-b63e6f72b65d,smoke] +tempest.api.identity.admin.v3.test_list_projects.ListProjectsTestJSON.test_list_projects_with_parent[id-6edc66f5-2941-4a17-9526-4073311c1fac] +tempest.api.compute.admin.test_fixed_ips.FixedIPsTestJson.test_set_unreserve[id-7476e322-b9ff-4710-bf82-49d51bac6e2e] +tempest.api.image.v1.test_images.CreateRegisterImagesTest.test_register_http_image[id-6d0e13a7-515b-460c-b91f-9f4793f09816] +tempest.api.network.test_floating_ips.FloatingIPTestJSON.test_create_list_show_update_delete_floating_ip[id-62595970-ab1c-4b7f-8fcc-fddfe55e8718,smoke] +tempest.api.compute.servers.test_server_personality.ServerPersonalityTestJSON.test_rebuild_server_with_personality[id-128966d8-71fc-443c-8cab-08e24114ecc9,slow] +tempest.api.compute.servers.test_attach_interfaces.AttachInterfacesTestJSON.test_create_list_show_delete_interfaces_by_network_port[id-73fe8f02-590d-4bf1-b184-e9ca81065051,network] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_get_details_with_invalid_id[id-3c916244-2c46-49a4-9b55-b20bb0ae512c,negative] +tempest.api.image.v2.admin.test_images_metadefs_namespace_tags.MetadataNamespaceTagsTest.test_create_list_delete_namespace_tags[id-a2a3765e-3a6d-4f6d-a3a7-3cc3476aa876] +tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_set_server_metadata[id-211021f6-21de-4657-a68f-908878cfe251] +tempest.api.identity.admin.v3.test_domains.DomainsTestJSON.test_domain_delete_cascades_content[id-d8d318b7-d1b3-4c37-94c5-3c5ba0b121ea] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_greater_than_actual_count[id-d47c17fb-eebd-4287-8e95-f20a7e627b18,negative] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcpv6_invalid_options[id-81f18ef6-95b5-4584-9966-10d480b7496a] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_invalid_size[id-1ed83a8a-682d-4dfb-a30e-ee63ffd6c049,negative] +tempest.api.image.v2.test_images_member.ImagesMemberTest.test_image_share_reject[id-d9e83e5f-3524-4b38-a900-22abcb26e90e] +tempest.api.network.admin.test_routers.RoutersAdminTest.test_update_router_set_gateway_without_snat[id-96536bc7-8262-4fb2-9967-5c46940fa279] +tempest.api.compute.images.test_list_images.ListImagesTestJSON.test_get_image[id-490d0898-e12a-463f-aef0-c50156b9f789] +tempest.scenario.test_minbw_allocation_placement.MinBwAllocationPlacementTest.test_qos_min_bw_allocation_update_with_multiple_ports[compute,id-756ced7f-6f1a-43e7-a851-2fcfc16f3dd7,network] +tempest.api.volume.test_volumes_extend.VolumesExtendTest.test_volume_extend_when_volume_has_snapshot[id-86be1cba-2640-11e5-9c82-635fb964c912] +tempest.api.compute.admin.test_hosts.HostsAdminTestJSON.test_list_hosts_with_nonexistent_zone[id-c6ddbadb-c94e-4500-b12f-8ffc43843ff8] +tempest.api.compute.admin.test_networks.NetworksTest.test_get_network[id-d206d211-8912-486f-86e2-a9d090d1f416] +tempest.api.network.admin.test_metering_extensions.MeteringTestJSON.test_create_delete_metering_label_with_filters[id-ec8e15ff-95d0-433b-b8a6-b466bddb1e50] +tempest.api.object_storage.test_container_services_negative.ContainerNegativeTest.test_list_all_container_objects_with_nonexistent_container[id-14331d21-1e81-420a-beea-19cb5e5207f5,negative] +tempest.api.volume.test_volumes_backup.VolumesBackupsV39Test.test_update_backup[id-9b374cbc-be5f-4d37-8848-7efb8a873dcc] +tempest.api.network.test_ports.PortsTestJSON.test_update_port_with_two_security_groups_and_extra_attributes[id-edf6766d-3d40-4621-bc6e-2521a44c257d] +tempest.api.image.v2.admin.test_images.BasicOperationsImagesAdminTest.test_create_image_owner_param[id-646a6eaa-135f-4493-a0af-12583021224e] +tempest.api.object_storage.test_object_services.ObjectTest.test_list_no_object_metadata[id-170fb90e-f5c3-4b1f-ae1b-a18810821172] +tempest.api.image.v2.test_images.MultiStoresImportImagesTest.test_glance_direct_import_image_to_all_stores[id-bf04ff00-3182-47cb-833a-f1c6767b47fd] +tempest.api.network.test_networks.NetworksTest.test_show_subnet[id-bd635d81-6030-4dd1-b3b9-31ba0cfdf6cc,smoke] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_server_name[id-f9eb2b70-735f-416c-b260-9914ac6181e4] +tempest.api.volume.test_volumes_get.VolumesGetTest.test_volume_create_get_update_delete_from_image[id-54a01030-c7fc-447c-86ee-c1182beae638,image,smoke] +tempest.api.network.test_networks_negative.NetworksNegativeTestJSON.test_update_non_existent_network[id-98bfe4e3-574e-4012-8b17-b2647063de87,negative] +tempest.api.object_storage.test_object_temp_url.ObjectTempUrlTest.test_head_object_using_temp_url[id-249a0111-5ad3-4534-86a7-1993d55f9185] +tempest.api.volume.admin.test_groups.GroupsV314Test.test_create_group_from_group[id-2424af8c-7851-4888-986a-794b10c3210e] +tempest.api.volume.admin.test_user_messages.UserMessagesTest.test_list_show_messages[id-50f29e6e-f363-42e1-8ad1-f67ae7fd4d5a] +tempest.api.compute.servers.test_novnc.NoVNCConsoleTestJSON.test_novnc[id-c640fdff-8ab4-45a4-a5d8-7e6146cbd0dc] +tempest.api.volume.admin.test_volume_types_extra_specs_negative.ExtraSpecsNegativeTest.test_get_nonexistent_extra_spec_name[id-c881797d-12ff-4f1a-b09d-9f6212159753,negative] +tempest.api.volume.admin.test_volume_types_extra_specs_negative.ExtraSpecsNegativeTest.test_create_invalid_body[id-bc772c71-1ed4-4716-b945-8b5ed0f15e87,negative] +tempest.api.compute.keypairs.test_keypairs_negative.KeyPairsNegativeTestJSON.test_create_keypair_with_duplicate_name[id-0359a7f1-f002-4682-8073-0c91e4011b7c,negative] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_resize_volume_backed_server_confirm[id-e6c28180-7454-4b59-b188-0257af08a63b,volume] +tempest.api.compute.servers.test_device_tagging.TaggedAttachmentsTest.test_tagged_attachment[id-3e41c782-2a89-4922-a9d2-9a188c4e7c7c,image,network,volume] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_shelve_paused_server[id-8cf9f450-a871-42cf-9bef-77eba189c0b0] +tempest.api.network.test_floating_ips_negative.FloatingIPNegativeTestJSON.test_associate_floatingip_port_ext_net_unreachable[id-6b3b8797-6d43-4191-985c-c48b773eb429,negative] +tempest.api.compute.volumes.test_attach_volume.AttachVolumeTestJSON.test_list_get_volume_attachments[id-7fa563fe-f0f7-43eb-9e22-a1ece036b513] +tempest.api.compute.volumes.test_attach_volume.AttachVolumeTestJSON.test_attach_detach_volume[id-52e9045a-e90d-4c0d-9087-79d657faffff,slow] +tempest.api.compute.volumes.test_attach_volume_negative.AttachVolumeNegativeTest.test_delete_attached_volume[id-a313b5cd-fbd0-49cc-94de-870e99f763c7,negative] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filtered_by_name_wildcard[id-e9f624ee-92af-4562-8bec-437945a18dcb] +tempest.api.compute.security_groups.test_security_group_rules.SecurityGroupRulesTestJSON.test_security_group_rules_create_with_optional_group_id[id-7f5d2899-7705-4d4b-8458-4505188ffab6] +tempest.api.network.test_security_groups.SecGroupIPv6Test.test_create_show_delete_security_group_rule[id-cfb99e0e-7410-4a3d-8a0c-959a63ee77e9,smoke] +tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON.test_list_flavors_detailed_filter_by_min_disk[id-3df2743e-3034-4e57-a4cb-b6527f6eac79] +tempest.api.compute.admin.test_migrations.MigrationsAdminTest.test_resize_server_revert_deleted_flavor[id-33f1fec3-ba18-4470-8e4e-1d888e7c3593] +tempest.api.compute.security_groups.test_security_group_rules_negative.SecurityGroupRulesNegativeTestJSON.test_create_security_group_rule_with_invalid_to_port[id-ff88804d-144f-45d1-bf59-dd155838a43a,negative] +tempest.api.image.v2.test_images_metadefs_schema.MetadataSchemaTest.test_get_metadata_tag_schema[id-dde4a891-b38b-3bf0-a3b2-e03ee67b3a3e] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_volume_extend_without_passing_volume_id[id-aff8ba64-6d6f-4f2e-bc33-41a08ee9f115,negative] +tempest.api.volume.admin.test_volume_quotas.VolumeQuotasAdminTestJSON.test_list_default_quotas[id-2be020a2-5fdd-423d-8d35-a7ffbc36e9f7] +tempest.api.identity.admin.v2.test_tenant_negative.TenantsNegativeTestJSON.test_create_tenants_name_length_over_64[id-2ff18d1e-dfe3-4359-9dc3-abf582c196b9,negative] +tempest.api.network.test_networks_negative.NetworksNegativeTestJSON.test_show_non_existent_port[id-a954861d-cbfd-44e8-b0a9-7fab111f235d,negative] +tempest.api.compute.admin.test_services_negative.ServicesAdminNegativeV253TestJSON.test_enable_service_with_invalid_service_id[id-508671aa-c929-4479-bd10-8680d40dd0a6,negative] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_remove_user_role_non_existent_role[id-ab32d759-cd16-41f1-a86e-44405fa9f6d2,negative] +tempest.api.network.test_networks.NetworksIpV6Test.test_create_list_subnet_with_no_gw64_one_network[id-a9653883-b2a4-469b-8c3c-4518430a7e55] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshots_list_with_params[id-59f41f43-aebf-48a9-ab5d-d76340fab32b] +tempest.api.network.test_agent_management_negative.AgentManagementNegativeTest.test_list_agents_non_admin[id-e335be47-b9a1-46fd-be30-0874c0b751e6,negative] +tempest.api.volume.admin.test_volume_retype.VolumeRetypeWithMigrationTest.test_available_volume_retype_with_migration[id-a1a41f3f-9dad-493e-9f09-3ff197d477cd] +tempest.api.identity.admin.v3.test_list_projects.ListProjectsStaticTestJSON.test_list_projects_with_domains[id-fab13f3c-f6a6-4b9f-829b-d32fd44fdf10] +tempest.api.compute.admin.test_simple_tenant_usage.TenantUsagesTestJSON.test_list_usage_all_tenants[id-062c8ae9-9912-4249-8b51-e38d664e926e] +tempest.api.compute.servers.test_server_rescue.ServerBootFromVolumeStableRescueTest.test_stable_device_rescue_bfv_blank_volume[id-48f123cb-922a-4065-8db6-b9a9074a556b,slow] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_security_group_with_invalid_sg_id[id-00579617-fe04-4e1c-9d08-ca7467d2e34b,negative] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_create_aggregate_name_length_less_than_1[id-3b8a1929-3793-4e92-bcb4-dfa572ee6c1d,negative] +tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_with_existing_server_name[id-8fea6be7-065e-47cf-89b8-496e6f96c699] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcpv6_two_subnets[id-4544adf7-bb5f-4bdc-b769-b3e77026cef2] +tempest.api.identity.admin.v3.test_policies.PoliciesTestJSON.test_create_update_delete_policy[id-e544703a-2f03-4cf2-9b0f-350782fdb0d3,smoke] +tempest.api.network.test_networks.NetworksIpV6Test.test_create_delete_subnet_all_attributes[id-a4d9ec4c-0306-4111-a75c-db01a709030b] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_remote_ip_prefix[id-5f8daf69-3c5f-4aaa-88c9-db1d66f68679,negative] +tempest.api.image.v1.test_images.UpdateImageMetaTest.test_update_image_metadata[id-d6d7649c-08ce-440d-9ea7-e3dda552f33c] +tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_deactivate_reactivate_image[id-951ebe01-969f-4ea9-9898-8a3f1f442ab0] +tempest.api.compute.admin.test_services_negative.ServicesAdminNegativeV253TestJSON.test_disable_log_reason_with_invalid_service_id[id-f46a9d91-1e85-4b96-8e7a-db7706fa2e9a,negative] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_duplicate_security_group_rule_fails[id-8fde898f-ce88-493b-adc9-4e4692879fc5,negative] +tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_specify_keypair[id-f9e15296-d7f9-4e62-b53f-a04e89160833] +tempest.scenario.test_minbw_allocation_placement.MinBwAllocationPlacementTest.test_resize_with_qos_min_bw_allocation[compute,id-c29e7fd3-035d-4993-880f-70819847683f,network] +tempest.api.network.test_routers_negative.RoutersNegativeIpV6Test.test_add_router_interfaces_on_overlapping_subnets_returns_400[id-957751a3-3c68-4fa2-93b6-eb52ea10db6e,negative] +tempest.api.volume.test_volumes_actions.VolumesActionsTest.test_volume_readonly_update[id-fff74e1e-5bd3-4b33-9ea9-24c103bc3f59] +tempest.api.object_storage.test_object_services.ObjectTest.test_update_object_metadata_with_x_object_manifest[id-08854588-6449-4bb7-8cca-f2e1040f5e6f] +tempest.api.image.v2.test_images_tags_negative.ImagesTagsNegativeTest.test_update_tags_for_non_existing_image[id-8cd30f82-6f9a-4c6e-8034-c1b51fba43d9,negative] +tempest.api.volume.admin.test_qos.QosSpecsTestJSON.test_list_qos[id-75e04226-bcf7-4595-a34b-fdf0736f38fc] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_list_roles_by_unauthorized_user[id-d5d5f1df-f8ca-4de0-b2ef-259c1cc67025,negative] +tempest.api.volume.admin.test_volume_hosts.VolumeHostsAdminTestsJSON.test_show_host[id-21168d57-b373-4b71-a3ac-f2c88f0c5d31] +tempest.api.compute.admin.test_hypervisor_negative.HypervisorAdminNegativeTestJSON.test_show_nonexistent_hypervisor[id-c136086a-0f67-4b2b-bc61-8482bd68989f,negative] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_stop_start_server[id-af8eafd4-38a7-4a4b-bdbc-75145a580560] +tempest.scenario.test_minbw_allocation_placement.MinBwAllocationPlacementTest.test_qos_min_bw_allocation_update_policy[compute,id-79fdaa1c-df62-4738-a0f0-1cff9dc415f6,network] +tempest.api.network.admin.test_external_network_extension.ExternalNetworksTestJSON.test_create_external_network[id-462be770-b310-4df9-9c42-773217e4c8b1] +tempest.scenario.test_shelve_instance.TestShelveInstance.test_shelve_volume_backed_instance[compute,id-c1b6318c-b9da-490b-9c67-9339b627271f,image,network,slow,volume] +tempest.api.compute.admin.test_hosts.HostsAdminTestJSON.test_list_hosts[id-9bfaf98d-e2cb-44b0-a07e-2558b2821e4f] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_get_console_output[id-4b8867e6-fffa-4d54-b1d1-6fdda57be2f3] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list[id-0b6ddd39-b948-471f-8038-4787978747c4,smoke] +tempest.api.identity.admin.v3.test_inherits.InheritsV3TestJSON.test_inherit_assign_list_check_revoke_roles_on_domains_group[id-c7a8dda2-be50-4fb4-9a9c-e830771078b1] +tempest.api.volume.test_volumes_actions.VolumesActionsTest.test_reserve_unreserve_volume[id-92c4ef64-51b2-40c0-9f7e-4749fbaaba33] +tempest.api.compute.admin.test_services_negative.ServicesAdminNegativeTestJSON.test_get_service_by_invalid_params[id-d0884a69-f693-4e79-a9af-232d15643bf7,negative] +tempest.api.image.v2.admin.test_images_metadefs_namespaces.MetadataNamespacesTest.test_basic_metadata_definition_namespaces[id-319b765e-7f3d-4b3d-8b37-3ca3876ee768] +tempest.api.image.v2.test_images.ImportImagesTest.test_remote_import[id-e04761a1-22af-42c2-b8bc-a34a3f12b585] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_with_detail_param_display_name_and_status[id-856ab8ca-6009-4c37-b691-be1065528ad4] +tempest.api.network.test_networks.BulkNetworkOpsTest.test_bulk_create_delete_subnet[id-8936533b-c0aa-4f29-8e53-6cc873aec489,smoke] +tempest.api.image.v2.test_images_metadefs_schema.MetadataSchemaTest.test_get_metadata_property_schema[id-dae4a891-b38b-3bf0-a3b2-e03ee67b3a3d] +tempest.api.network.test_networks.NetworksTest.test_delete_network_with_subnet[id-f04f61a9-b7f3-4194-90b2-9bcf660d1bfe] +tempest.api.compute.admin.test_servers_on_multinodes.ServersOnMultiNodesTest.test_create_servers_on_same_host[id-26a9d5df-6890-45f2-abc4-a659290cb130] +tempest.api.identity.admin.v3.test_groups.GroupsV3TestJSON.test_list_groups[id-cc9a57a5-a9ed-4f2d-a29f-4f979a06ec71] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_get_console_output_server_id_in_shutoff_status[id-5b65d4e7-4ecd-437c-83c0-d6b79d927568] +tempest.api.identity.admin.v3.test_users.UsersV3TestJSON.test_get_user[id-c10dcd90-461d-4b16-8e23-4eb836c00644] +tempest.api.identity.v3.test_tokens.TokensV3Test.test_token_auth_creation_existence_deletion[id-0f9f5a5f-d5cd-4a86-8a5b-c5ded151f212] +tempest.api.network.admin.test_external_network_extension.ExternalNetworksTestJSON.test_delete_external_networks_with_floating_ip[id-82068503-2cf2-4ed4-b3be-ecb89432e4bb] +tempest.api.network.admin.test_routers.RoutersAdminTest.test_create_router_set_gateway_with_fixed_ip[id-cbe42f84-04c2-11e7-8adb-fa163e4fa634] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_sort_name_desc[id-96ba6f4d-1f18-47e1-b4bc-76edc6c21250] +tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_associate_nonexistent_floating_ip[id-595fa616-1a71-4670-9614-46564ac49a4c,negative] +tempest.api.volume.admin.test_group_snapshots.GroupSnapshotsV319Test.test_reset_group_snapshot_status[id-3b42c9b9-c984-4444-816e-ca2e1ed30b40] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents_with_format_xml[id-655a53ca-4d15-408c-a377-f4c6dbd0a1fa] +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_host_routes_and_dns_nameservers[id-d830de0a-be47-468f-8f02-1fd996118289] +tempest.api.volume.admin.test_volumes_actions.VolumesActionsTest.test_volume_force_delete_when_volume_is_creating[id-21737d5a-92f2-46d7-b009-a0cc0ee7a570] +tempest.api.compute.servers.test_server_group.ServerGroupTestJSON.test_show_server_group[id-b3545034-dd78-48f0-bdc2-a4adfa6d0ead] +tempest.api.object_storage.test_object_formpost_negative.ObjectFormPostNegativeTest.test_post_object_using_form_invalid_signature[id-b277257f-113c-4499-b8d1-5fead79f7360,negative] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_detach_volumes_with_invalid_volume_id[id-9f9c24e4-011d-46b5-b992-952140ce237a,negative] +tempest.api.network.admin.test_dhcp_agent_scheduler.DHCPAgentSchedulersTestJSON.test_list_dhcp_agent_hosting_network[id-5032b1fe-eb42-4a64-8f3b-6e189d8b5c7d] +tempest.api.network.test_networks.NetworksIpV6TestAttrs.test_create_delete_subnet_with_v6_attributes_slaac[id-176b030f-a923-4040-a755-9dc94329e60c] +tempest.api.compute.admin.test_simple_tenant_usage.TenantUsagesTestJSON.test_get_usage_tenant_with_non_admin_user[id-9d00a412-b40e-4fd9-8eba-97b496316116] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_list_ports[id-cf95b358-3e92-4a29-a148-52445e1ac50e,smoke] +tempest.api.compute.admin.test_volume_swap.TestMultiAttachVolumeSwap.test_volume_swap_with_multiattach[id-e8f8f9d1-d7b7-4cd2-8213-ab85ef697b6e,slow,volume] +tempest.api.identity.admin.v3.test_endpoint_groups.EndPointGroupsTest.test_update_endpoint_group[id-51c8fc38-fa84-4e76-b5b6-6fc37770fb26] +tempest.scenario.test_volume_boot_pattern.TestVolumeBootPattern.test_image_defined_boot_from_volume[compute,id-36c34c67-7b54-4b59-b188-02a2f458a63b,image,volume] +tempest.scenario.test_network_v6.TestGettingAddress.test_multi_prefix_slaac[compute,id-dec222b1-180c-4098-b8c5-cc1b8342d611,network,slow] +tempest.api.compute.admin.test_flavors.FlavorsAdminTestJSON.test_list_non_public_flavor[id-be6cc18c-7c5d-48c0-ac16-17eaf03c54eb] +tempest.api.identity.admin.v3.test_list_projects.ListProjectsStaticTestJSON.test_list_projects_with_name[id-fa178524-4e6d-4925-907c-7ab9f42c7e26] +tempest.api.object_storage.test_object_services.ObjectTest.test_get_object_with_if_modified_since[id-be133639-e5d2-4313-9b1f-2d59fc054a16] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_shelve_unshelve_server[id-77eba8e0-036e-4635-944b-f7a8f3b78dc9,image] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_sort_key_dir[id-9faaa0c2-c3a5-43e1-8f61-61c54b409a49] +tempest.api.volume.test_volumes_snapshots_negative.VolumesSnapshotNegativeTestJSON.test_create_snapshot_with_nonexistent_volume_id[id-e3e466af-70ab-4f4b-a967-ab04e3532ea7,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_pause_paused_server[id-d1417e7f-a509-41b5-a102-d5eed8613369,negative] +tempest.api.compute.admin.test_server_diagnostics.ServerDiagnosticsTest.test_get_server_diagnostics[id-31ff3486-b8a0-4f56-a6c0-aab460531db3] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_create_image_reserved_property[id-e3fb7df8-2742-4143-8976-f1b26870f0a0,negative] +tempest.api.volume.admin.test_volume_types_negative.VolumeTypesNegativeTest.test_create_volume_with_private_volume_type[id-8c09f849-f225-4d78-ba87-bffd9a5e0c6f,negative] +tempest.api.volume.test_volumes_actions.VolumesActionsTest.test_get_volume_attachment[compute,id-9516a2c8-9135-488c-8dd6-5677a7e5f371] +tempest.api.identity.admin.v2.test_users.UsersTestJSON.test_get_users[id-a149c02e-e5e0-4b89-809e-7e8faf33ccda] +tempest.api.compute.admin.test_fixed_ips_negative.FixedIPsNegativeTestJson.test_set_reserve_with_non_admin_user[id-ce60042c-fa60-4836-8d43-1c8e3359dc47,negative] +tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON.test_list_images_filter_by_status[id-a3f5b513-aeb3-42a9-b18e-f091ef73254d] +tempest.api.network.test_security_groups.SecGroupIPv6Test.test_create_list_update_show_delete_security_group[id-bfd128e5-3c92-44b6-9d66-7fe29d22c802,smoke] +tempest.api.compute.admin.test_volumes_negative.VolumesAdminNegativeTest.test_update_attached_volume_with_nonexistent_volume_in_uri[id-309b5ecd-0585-4a7e-a36f-d2b2bf55259d,negative] +tempest.api.network.admin.test_routers.RoutersIpV6AdminTest.test_update_router_unset_gateway[id-ad81b7ee-4f81-407b-a19c-17e623f763e8] +tempest.api.volume.test_extensions.ExtensionsTestJSON.test_list_extensions[id-94607eb0-43a5-47ca-82aa-736b41bd2e2c] +tempest.api.image.v1.test_images.ListImagesTest.test_index_disk_format[id-f1755589-63d6-4468-b098-589820eb4031] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents_with_delimiter[id-fe323a32-57b9-4704-a996-2e68f83b09bc] +tempest.api.compute.admin.test_hypervisor.HypervisorAdminUnderV252Test.test_get_hypervisor_show_servers[id-e81bba3f-6215-4e39-a286-d52d2f906862] +tempest.api.volume.test_volumes_clone.VolumesCloneTest.test_create_from_volume[id-9adae371-a257-43a5-9555-dc7c88e66e0e] +tempest.api.network.test_security_groups.SecGroupIPv6Test.test_create_security_group_rule_with_icmp_type_code[id-c9463db8-b44d-4f52-b6c0-8dbda99f26ce] +tempest.api.compute.test_versions.TestVersions.test_get_version_details[id-b953a29e-929c-4a8e-81be-ec3a7e03cb76,smoke] +tempest.api.network.test_networks_negative.NetworksNegativeTestJSON.test_delete_non_existent_port[id-49ec2bbd-ac2e-46fd-8054-798e679ff894,negative] +tempest.api.object_storage.test_account_services.AccountTest.test_list_account_metadata[id-4894c312-6056-4587-8d6f-86ffbf861f80,smoke] +tempest.api.compute.images.test_images_negative.ImagesDeleteNegativeTestJSON.test_delete_image_with_id_over_character_limit[id-b340030d-82cd-4066-a314-c72fb7c59277,negative] +tempest.api.network.test_routers.RoutersTest.test_update_router_admin_state[id-a8902683-c788-4246-95c7-ad9c6d63a4d9] +tempest.api.compute.admin.test_live_migration.LiveMigrationTest.test_live_block_migration_paused[id-1e107f21-61b2-4988-8f22-b196e938ab88] +tempest.api.object_storage.test_object_services.ObjectTest.test_copy_object_across_containers[id-aa467252-44f3-472a-b5ae-5b57c3c9c147] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_reserve_volume_with_negative_volume_status[id-449c4ed2-ecdd-47bb-98dc-072aeccf158c,negative] +tempest.api.compute.volumes.test_volumes_negative.VolumesNegativeTest.test_delete_invalid_volume_id[id-62972737-124b-4513-b6cf-2f019f178494,negative] +tempest.api.identity.v3.test_application_credentials.ApplicationCredentialsV3Test.test_create_application_credential_access_rules[id-529936eb-aa5d-463d-9f79-01c113d3b88f] +tempest.api.compute.servers.test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_server_create_metadata_key_too_long[id-fe114a8f-3a57-4eff-9ee2-4e14628df049,negative] +tempest.api.compute.servers.test_create_server.ServersTestJSON.test_verify_created_server_vcpus[id-cbc0f52f-05aa-492b-bdc1-84b575ca294b] +tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_identity_v3_existence[id-79aec9ae-710f-4c54-a4fc-3aa25b4feac3] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_remove_nonexistent_host[id-95d6a6fa-8da9-4426-84d0-eec0329f2e4d,negative] +tempest.api.compute.images.test_image_metadata_negative.ImagesMetadataNegativeTestJSON.test_update_nonexistent_image_metadata[id-a403ef9e-9f95-427c-b70a-3ce3388796f1,negative] +tempest.api.compute.test_versions.TestVersions.test_list_api_versions[id-6c0a0990-43b6-4529-9b61-5fd8daf7c55c,smoke] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_remove_host_as_user[id-7a53af20-137a-4e44-a4ae-e19260e626d9,negative] +tempest.api.compute.servers.test_delete_server.DeleteServersTestJSON.test_delete_server_while_in_verify_resize_state[id-ab0c38b4-cdd8-49d3-9b92-0cb898723c01] +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_gw[id-9393b468-186d-496d-aa36-732348cd76e7] +tempest.api.identity.admin.v3.test_domains_negative.DomainsNegativeTestJSON.test_delete_non_existent_domain[id-43781c07-764f-4cf2-a405-953c1916f605,negative] +tempest.api.compute.servers.test_server_rescue.ServerBootFromVolumeStableRescueTest.test_stable_device_rescue_bfv_image_volume[id-e4636333-c928-40fc-98b7-70a23eef4224,slow] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_image_param_owner[id-e9a44b91-31c8-4b40-a332-e0a39ffb4dbb] +tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON.test_list_images_filter_by_name[id-33163b73-79f5-4d07-a7ea-9213bcc468ff] +tempest.api.compute.images.test_image_metadata.ImagesMetadataTestJSON.test_list_image_metadata[id-37ec6edd-cf30-4c53-bd45-ae74db6b0531] +tempest.api.identity.admin.v3.test_trusts.TrustsV3TestJSON.test_trust_expire_invalid[id-3e48f95d-e660-4fa9-85e0-5a3d85594384] +tempest.api.identity.admin.v3.test_policies.PoliciesTestJSON.test_list_policies[id-1a0ad286-2d06-4123-ab0d-728893a76201] +tempest.api.network.test_routers.RoutersIpV6Test.test_update_router_admin_state[id-a8902683-c788-4246-95c7-ad9c6d63a4d9] +tempest.api.compute.servers.test_server_addresses_negative.ServerAddressesNegativeTestJSON.test_list_server_addresses_invalid_server_id[id-02c3f645-2d2e-4417-8525-68c0407d001b,negative,network] +tempest.scenario.test_security_groups_basic_ops.TestSecurityGroupsBasicOps.test_in_tenant_traffic[compute,id-63163892-bbf6-4249-aa12-d5ea1f8f421b,network] +tempest.api.compute.admin.test_flavors.FlavorsAdminTestJSON.test_is_public_string_variations[id-fb9cbde6-3a0e-41f2-a983-bdb0a823c44e] +tempest.api.network.test_ports.PortsTestJSON.test_show_port_fields[id-45fcdaf2-dab0-4c13-ac6c-fcddfb579dbd] +tempest.api.compute.security_groups.test_security_group_rules_negative.SecurityGroupRulesNegativeTestJSON.test_create_security_group_rule_with_non_existent_id[id-1d507e98-7951-469b-82c3-23f1e6b8c254,negative] +tempest.api.identity.admin.v3.test_services.ServicesTestJSON.test_create_service_without_description[id-d1dcb1a1-2b6b-4da8-bbb8-5532ef6e8269] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_add_existent_host[id-19dd44e1-c435-4ee1-a402-88c4f90b5950,negative] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_grant_list_revoke_role_to_user_on_project[id-c6b80012-fe4a-498b-9ce8-eb391c05169f] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_domain_roles_create_delete[id-d92a41d2-5501-497a-84bb-6e294330e8f8] +tempest.api.network.test_networks_negative.NetworksNegativeTestJSON.test_show_non_existent_network[id-9293e937-824d-42d2-8d5b-e985ea67002a,negative] +tempest.scenario.test_volume_migrate_attached.TestVolumeMigrateRetypeAttached.test_volume_retype_attached[compute,id-deadd2c2-beef-4dce-98be-f86765ff311b,slow,volume] +tempest.api.compute.servers.test_instance_actions.InstanceActionsTestJSON.test_get_instance_action[id-aacc71ca-1d70-4aa5-bbf6-0ff71470e43c] +tempest.api.identity.admin.v3.test_projects.ProjectsTestJSON.test_project_create_not_enabled[id-78f96a9c-e0e0-4ee6-a3ba-fbf6dfd03207] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_delete_non_existing_image[id-6fe40f1c-57bd-4918-89cc-8500f850f3de,negative] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_format_json[id-1c7efa35-e8a2-4b0b-b5ff-862c7fd83704] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_status[id-7fc9e369-0f58-4d05-9aa5-0969e2d59d15] +tempest.api.volume.admin.test_groups.GroupsTest.test_group_create_show_list_delete[id-4b111d28-b73d-4908-9bd2-03dc2992e4d4] +tempest.api.compute.servers.test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_update_metadata_non_existent_server[id-904b13dc-0ef2-4e4c-91cd-3b4a0f2f49d8,negative] +tempest.api.object_storage.test_account_services.AccountTest.test_update_account_metadata_with_create_and_delete_metadata[id-8e5fc073-59b9-42ee-984a-29ed11b2c749] +tempest.api.volume.admin.test_volumes_backup.VolumesBackupsAdminTest.test_volume_backup_export_import[id-a99c54a1-dd80-4724-8a13-13bf58d4068d] +tempest.api.network.admin.test_metering_extensions.MeteringIpV6TestJSON.test_create_delete_metering_label_with_filters[id-ec8e15ff-95d0-433b-b8a6-b466bddb1e50] +tempest.api.object_storage.test_account_services.AccountTest.test_list_extensions[id-6eb04a6a-4860-4e31-ba91-ea3347d76b58] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_marker_and_end_marker[id-ac8502c2-d4e4-4f68-85a6-40befea2ef5e] +tempest.api.identity.v2.test_extension.ExtensionTestJSON.test_list_extensions[id-85f3f661-f54c-4d48-b563-72ae952b9383] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_add_non_exist_host[id-0ef07828-12b4-45ba-87cc-41425faf5711,negative] +tempest.api.compute.security_groups.test_security_group_rules_negative.SecurityGroupRulesNegativeTestJSON.test_create_security_group_rule_with_invalid_from_port[id-12bbc875-1045-4f7a-be46-751277baedb9,negative] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_create_security_group_rule_with_bad_protocol[id-981bdc22-ce48-41ed-900a-73148b583958,negative] +tempest.api.volume.admin.test_volume_types_extra_specs_negative.ExtraSpecsNegativeTest.test_create_nonexistent_type_id[id-49d5472c-a53d-4eab-a4d3-450c4db1c545,negative] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_volume_extend_with_non_number_size[id-5d0b480d-e833-439f-8a5a-96ad2ed6f22f,negative] +tempest.api.identity.admin.v2.test_roles.RolesTestJSON.test_list_roles[id-75d9593f-50b7-4fcf-bd64-e3fb4a278e23] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_image[id-ff01387d-c7ad-47b4-ae9e-64fa214638fe,negative] +tempest.api.image.v1.test_image_members.ImageMembersTest.test_add_image_member[id-1d6ef640-3a20-4c84-8710-d95828fdb6ad] +tempest.api.compute.admin.test_fixed_ips.FixedIPsTestJson.test_set_reserve[id-5485077b-7e46-4cec-b402-91dc3173433b] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_resize_server_confirm[id-1499262a-9328-4eda-9068-db1ac57498d2] +tempest.api.object_storage.test_object_services.ObjectTest.test_get_object_with_x_newest[id-94587078-475f-48f9-a40f-389c246e31cd] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_sort_id_asc[id-c5513ada-64c1-4d28-83b9-af3307ec1388] +tempest.api.identity.admin.v2.test_roles.RolesTestJSON.test_role_create_delete[id-c62d909d-6c21-48c0-ae40-0a0760e6db5e] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_nonexistent_volume_type[id-10254ed8-3849-454e-862e-3ab8e6aa01d2,negative] +tempest.api.network.test_allowed_address_pair.AllowedAddressPairIpV6TestJSON.test_update_port_with_cidr_address_pair[id-4d6d178f-34f6-4bff-a01c-0a2f8fe909e4] +tempest.api.compute.servers.test_device_tagging.TaggedBootDevicesTest.test_tagged_boot_devices[id-a2e65a6c-66f1-4442-aaa8-498c31778d96,image,network,slow,volume] +tempest.api.identity.admin.v3.test_domain_configuration.DomainConfigurationTestJSON.test_create_update_and_delete_domain_config_groups_and_opts[id-c7510fa2-6661-4170-9c6b-4783a80651e9] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_rebuild_server_with_volume_attached[id-b68bd8d6-855d-4212-b59b-2e704044dace,slow,volume] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_without_passing_size[id-9387686f-334f-4d31-a439-33494b9e2683,negative] +tempest.api.identity.admin.v3.test_endpoints.EndPointsTestJSON.test_update_endpoint[id-37e8f15e-ee7c-4657-a1e7-f6b61e375eff,smoke] +tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON.test_list_images_filter_by_server_id[id-9f238683-c763-45aa-b848-232ec3ce3105] +tempest.api.compute.admin.test_services.ServicesAdminTestJSON.test_get_service_by_host_name[id-affb42d5-5b4b-43c8-8b0b-6dca054abcca] +tempest.api.compute.servers.test_instance_actions.InstanceActionsV221TestJSON.test_get_list_deleted_instance_actions[id-0a0f85d4-10fa-41f6-bf80-a54fb4aa2ae1] +tempest.api.compute.admin.test_hypervisor_negative.HypervisorAdminNegativeTestJSON.test_get_hypervisor_list_details_with_non_admin_user[id-dc02db05-e801-4c5f-bc8e-d915290ab345,negative] +tempest.api.network.test_routers_negative.RoutersNegativeTest.test_router_add_gateway_invalid_network_returns_404[id-37a94fc0-a834-45b9-bd23-9a81d2fd1e22,negative] +tempest.api.identity.v2.test_ec2_credentials.EC2CredentialsTest.test_create_ec2_credential[id-b580fab9-7ae9-46e8-8138-417260cb6f9f] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_change_server_password[id-6158df09-4b82-4ab3-af6d-29cf36af858d] +tempest.api.object_storage.test_container_services_negative.ContainerNegativeTest.test_create_container_name_exceeds_max_length[id-30686921-4bed-4764-a038-40d741ed4e78,negative] +tempest.api.compute.servers.test_server_password.ServerPasswordTestJSON.test_get_server_password[id-f83b582f-62a8-4f22-85b0-0dee50ff783a] +tempest.api.object_storage.test_container_acl_negative.ObjectACLsNegativeTest.test_delete_object_with_non_authorized_user[id-7343ac3d-cfed-4198-9bb0-00149741a492,negative] +tempest.api.network.admin.test_routers_dvr.RoutersTestDVR.test_centralized_router_creation[id-8a0a72b4-7290-4677-afeb-b4ffe37bc352] +tempest.api.compute.admin.test_servers.ServersAdminTestJSON.test_reset_state_server[id-ee8ae470-db70-474d-b752-690b7892cab1] +tempest.api.image.v2.test_images_metadefs_schema.MetadataSchemaTest.test_get_metadata_properties_schema[id-dce4a891-b38b-3bf0-a3b2-e03ee67b3a3e] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_implied_domain_roles[id-eb1e1c24-1bc4-4d47-9748-e127a1852c82] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_server_status[id-de2612ab-b7dd-4044-b0b1-d2539601911f] +tempest.api.volume.admin.test_volume_pools.VolumePoolsAdminTestsJSON.test_get_pools_with_details[id-d4bb61f7-762d-4437-b8a4-5785759a0ced] +tempest.api.compute.servers.test_server_rescue.ServerStableDeviceRescueTest.test_stable_device_rescue_disk_virtio[id-16865750-1417-4854-bcf7-496e6753c01e] +tempest.api.object_storage.test_account_services.AccountTest.test_update_account_metadata_with_create_metadata[id-e2a08b5f-3115-4768-a3ee-d4287acd6c08] +tempest.api.identity.admin.v3.test_trusts.TrustsV3TestJSON.test_trust_impersonate[id-5a0a91a4-baef-4a14-baba-59bf4d7fcace] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_unpause_server_invalid_state[id-c8e639a7-ece8-42dd-a2e0-49615917ba4f,negative] +tempest.api.image.v1.test_images.CreateRegisterImagesTest.test_register_image_with_min_ram[id-05b19d55-140c-40d0-b36b-fafd774d421b] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_name_of_non_existent_server[id-aa8eed43-e2cb-4ebf-930b-da14f6a21d81,negative] +tempest.api.compute.servers.test_server_rescue.ServerRescueTestJSON.test_rescue_unrescue_instance[id-fd032140-714c-42e4-a8fd-adcd8df06be6] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_min_max_size[id-4ad8c157-971a-4ba8-aa84-ed61154b1e7f] +tempest.api.volume.test_volumes_extend.VolumesExtendAttachedTest.test_extend_attached_volume[compute,id-301f5a30-1c6f-4ea0-be1a-91fd28d44354] +tempest.api.image.v2.admin.test_images_metadefs_namespace_objects.MetadataNamespaceObjectsTest.test_show_meta_namespace_objects[id-b1a3674e-3b4c-3f6a-a3b4-1ba3573ca768] +tempest.api.compute.admin.test_keypairs_v210.KeyPairsV210TestJSON.test_admin_manage_keypairs_for_other_users[id-3c8484af-cfb3-48f6-b8ba-d5d58bbf3eac] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents_with_limit[id-297ec38b-2b61-4ff4-bcd1-7fa055e97b61] +tempest.api.compute.volumes.test_volume_snapshots.VolumesSnapshotsTestJSON.test_volume_snapshot_create_get_list_delete[id-cd4ec87d-7825-450d-8040-6e2068f2da8f] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filtered_by_ip_regex[id-a905e287-c35e-42f2-b132-d02b09f3654a] +tempest.api.volume.admin.test_volumes_actions.VolumesActionsTest.test_volume_force_delete_when_volume_is_attaching[id-db8d607a-aa2e-4beb-b51d-d4005c232011] +tempest.api.compute.servers.test_server_tags.ServerTagsTestJSON.test_check_tag_existence[id-81279a66-61c3-4759-b830-a2dbe64cbe08] +tempest.scenario.test_minbw_allocation_placement.MinBwAllocationPlacementTest.test_qos_min_bw_allocation_update_policy_to_zero[compute,id-a9725a70-1d28-4e3b-ae0e-450abc235962,network] +tempest.api.identity.admin.v3.test_endpoints_negative.EndpointsNegativeTestJSON.test_update_with_enabled_False[id-65e41f32-5eb7-498f-a92a-a6ccacf7439a,negative] +tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_set_server_metadata_item[id-58c02d4f-5c67-40be-8744-d3fa5982eb1c] +tempest.api.compute.volumes.test_volumes_negative.VolumesNegativeTest.test_get_volume_without_passing_volume_id[id-62bab09a-4c03-4617-8cca-8572bc94af9b,negative] +tempest.api.compute.admin.test_quotas_negative.QuotasAdminNegativeTest.test_create_server_when_memory_quota_is_full[id-6fdd7012-584d-4327-a61c-49122e0d5864,negative] +tempest.api.compute.servers.test_disk_config.ServerDiskConfigTestJSON.test_update_server_from_auto_to_manual[id-5ef18867-358d-4de9-b3c9-94d4ba35742f] +tempest.api.compute.images.test_images_negative.ImagesDeleteNegativeTestJSON.test_delete_image_blank_id[id-e6e41425-af5c-4fe6-a4b5-7b7b963ffda5,negative] +tempest.api.identity.admin.v2.test_tokens.TokensTestJSON.test_create_check_get_delete_token[id-453ad4d5-e486-4b2f-be72-cffc8149e586] +tempest.api.image.v2.test_images_metadefs_schema.MetadataSchemaTest.test_get_metadata_object_schema[id-dff4a891-b38b-3bf0-a3b2-e03ee67b3a3b] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_with_details[id-adcbb5a7-5ad8-4b61-bd10-5380e111a877] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_marker[id-05489dde-44bc-4961-a1f5-3ce7ee7824f7] +tempest.api.identity.admin.v3.test_projects_negative.ProjectsNegativeStaticTestJSON.test_delete_non_existent_project[id-7965b581-60c1-43b7-8169-95d4ab7fc6fb,negative] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_delete_protected_image[id-ab980a34-8410-40eb-872b-f264752f46e5,negative] +tempest.api.volume.test_availability_zone.AvailabilityZoneTestJSON.test_get_availability_zone_list[id-01f1ae88-eba9-4c6b-a011-6f7ace06b725] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_update_user_for_non_existent_user[id-3d07e294-27a0-4144-b780-a2a1bf6fee19,negative] +tempest.api.volume.admin.test_group_snapshots.GroupSnapshotsTest.test_delete_group_snapshots_following_updated_volumes[id-7d7fc000-0b4c-4376-a372-544116d2e127] +tempest.api.identity.admin.v2.test_tenant_negative.TenantsNegativeTestJSON.test_tenant_update_by_unauthorized_user[id-41704dc5-c5f7-4f79-abfa-76e6fedc570b,negative] +tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON.test_list_flavors_filter_by_min_ram[id-935cf550-e7c8-4da6-8002-00f92d5edfaa] +tempest.api.network.test_allowed_address_pair.AllowedAddressPairTestJSON.test_create_list_port_with_address_pair[id-86c3529b-1231-40de-803c-00e40882f043] +tempest.api.volume.admin.test_volumes_backup.VolumesBackupsAdminTest.test_volume_backup_reset_status[id-47a35425-a891-4e13-961c-c45deea21e94] +tempest.api.compute.admin.test_hypervisor_negative.HypervisorAdminNegativeTestJSON.test_show_hypervisor_with_non_admin_user[id-51e663d0-6b89-4817-a465-20aca0667d03,negative] +tempest.scenario.test_aggregates_basic_ops.TestAggregatesBasicOps.test_aggregate_basic_ops[compute,id-cb2b4c4f-0c7c-4164-bdde-6285b302a081,slow] +tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_host_name_is_same_as_server_name[id-ac1ad47f-984b-4441-9274-c9079b7a0666] +tempest.api.compute.servers.test_server_addresses.ServerAddressesTestJSON.test_list_server_addresses_by_network[id-87bbc374-5538-4f64-b673-2b0e4443cc30,network,smoke] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_show_non_existent_security_group_rule[id-4c094c09-000b-4e41-8100-9617600c02a6,negative] +tempest.api.compute.admin.test_fixed_ips_negative.FixedIPsNegativeTestJson.test_set_reserve_with_invalid_ip[id-f51cf464-7fc5-4352-bc3e-e75cfa2cb717,negative] +tempest.api.identity.admin.v2.test_tenants.TenantsTestJSON.test_tenant_create_with_description[id-d25e9f24-1310-4d29-b61b-d91299c21d6d] +tempest.api.network.test_extra_dhcp_options.ExtraDHCPOptionsIpV6TestJSON.test_update_show_port_with_extra_dhcp_options[id-9a6aebf4-86ee-4f47-b07a-7f7232c55607] +tempest.api.object_storage.test_container_services.ContainerTest.test_delete_container[id-95d3a249-b702-4082-a2c4-14bb860cf06a] +tempest.api.identity.admin.v2.test_endpoints.EndPointsTestJSON.test_list_endpoints[id-11f590eb-59d8-4067-8b2b-980c7f387f51] +tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON.test_get_flavor[id-1f12046b-753d-40d2-abb6-d8eb8b30cb2f,smoke] +tempest.api.compute.images.test_image_metadata_negative.ImagesMetadataNegativeTestJSON.test_delete_nonexistent_image_metadata_item[id-848e157f-6bcf-4b2e-a5dd-5124025a8518,negative] +tempest.api.compute.images.test_images_oneserver_negative.ImagesOneServerNegativeTestJSON.test_create_image_specify_invalid_metadata[id-55d1d38c-dd66-4933-9c8e-7d92aeb60ddc,negative] +tempest.api.compute.images.test_image_metadata_negative.ImagesMetadataNegativeTestJSON.test_set_nonexistent_image_metadata[id-dc64f2ce-77e8-45b0-88c8-e15041d08eaf,negative] +tempest.api.compute.admin.test_live_migration.LiveAutoBlockMigrationV225Test.test_live_block_migration_with_attached_volume[id-e19c0cc6-6720-4ed8-be83-b6603ed5c812,volume] +tempest.api.compute.admin.test_availability_zone.AZAdminV2TestJSON.test_get_availability_zone_list[id-d3431479-8a09-4f76-aa2d-26dc580cb27c] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_update_nonexistent_host[id-23c92146-2100-4d68-b2d6-c7ade970c9c1,negative] +tempest.api.object_storage.test_object_services.ObjectTest.test_get_object_with_if_match[id-c05b4013-e4de-47af-be84-e598062b16fc] +tempest.api.volume.admin.test_volume_types_extra_specs.VolumeTypesExtraSpecsTest.test_volume_type_extra_specs_update[id-0806db36-b4a0-47a1-b6f3-c2e7f194d017] +tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_protocol_integer_value[id-0a307599-6655-4220-bebc-fd70c64f2290] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_from_deactivated_image[id-d15e7f35-2cfc-48c8-9418-c8223a89bcbb,image,negative] +tempest.api.compute.admin.test_servers.ServersAdminTestJSON.test_rebuild_server_in_error_state[id-682cb127-e5bb-4f53-87ce-cb9003604442] +tempest.api.network.test_networks_negative.NetworksNegativeTestJSON.test_delete_non_existent_network[id-03795047-4a94-4120-a0a1-bd376e36fd4e,negative] +tempest.api.compute.volumes.test_volumes_get.VolumesGetTestJSON.test_volume_create_get_delete[id-f10f25eb-9775-4d9d-9cbe-1cf54dae9d5f] +tempest.api.volume.admin.test_qos.QosSpecsTestJSON.test_create_delete_qos_with_back_end_consumer[id-b115cded-8f58-4ee4-aab5-9192cfada08f] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_offset[id-ca96d551-17c6-4e11-b0e8-52d3bb8a63c7] +tempest.api.identity.v3.test_users.IdentityV3UsersTest.test_user_update_own_password[id-ad71bd23-12ad-426b-bb8b-195d2b635f27] +tempest.api.compute.certificates.test_certificates.CertificatesV2TestJSON.test_get_root_certificate[id-3ac273d0-92d2-4632-bdfc-afbc21d4606c] +tempest.api.compute.images.test_image_metadata.ImagesMetadataTestJSON.test_delete_image_metadata_item[id-a013796c-ba37-4bb5-8602-d944511def14] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_with_detail_param_metadata[id-1ca92d3c-4a8e-4b43-93f5-e4c7fb3b291d] +tempest.api.identity.admin.v3.test_projects.ProjectsTestJSON.test_project_get_equals_list[id-d1db68b6-aebe-4fa0-b79d-d724d2e21162] +tempest.scenario.test_volume_boot_pattern.TestVolumeBootPattern.test_create_server_from_volume_snapshot[compute,id-05795fb2-b2a7-4c9f-8fac-ff25aedb1489,image,slow,volume] +tempest.api.volume.test_volumes_backup.VolumesBackupsTest.test_volume_backup_create_get_detailed_list_restore_delete[id-a66eb488-8ee1-47d4-8e9f-575a095728c6] +tempest.api.compute.servers.test_disk_config.ServerDiskConfigTestJSON.test_rebuild_server_with_auto_disk_config[id-9c9fae77-4feb-402f-8450-bf1c8b609713] +tempest.api.volume.test_volumes_actions.VolumesActionsTest.test_volume_bootable[id-63e21b4c-0a0c-41f6-bfc3-7c2816815599] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_get_console_output_of_non_existent_server[id-7dd919e7-413f-4198-bebb-35e2a01b13e9,negative] +tempest.api.identity.admin.v3.test_trusts.TrustsV3TestJSON.test_trust_noimpersonate[id-ed2a8779-a7ac-49dc-afd7-30f32f936ed2] +tempest.scenario.test_security_groups_basic_ops.TestSecurityGroupsBasicOps.test_multiple_security_groups[compute,id-d2f77418-fcc4-439d-b935-72eca704e293,network,slow] +tempest.api.volume.admin.test_volume_services_negative.VolumeServicesNegativeTest.test_freeze_host_with_invalid_host[id-712bfab8-1f44-4eb5-a632-fa70bf78f05e,negative] +tempest.api.network.test_ports.PortsTestJSON.test_list_ports[id-cf95b358-3e92-4a29-a148-52445e1ac50e,smoke] +tempest.api.volume.admin.test_volume_types_extra_specs_negative.ExtraSpecsNegativeTest.test_get_nonexistent_volume_type_id[id-9f402cbd-1838-4eb4-9554-126a6b1908c9,negative] +tempest.api.compute.admin.test_hypervisor.HypervisorAdminV228Test.test_get_list_hypervisor_details[id-d46bab64-0fbe-4eb8-9133-e6ee56188cc5] +tempest.api.image.v1.test_image_members_negative.ImageMembersNegativeTest.test_delete_member_with_non_existing_image[id-e1559f05-b667-4f1b-a7af-518b52dc0c0f,negative] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_image[id-b3304c3b-97df-46d2-8cd3-e2b6659724e7] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_lock_unlock_server[id-80a8094c-211e-440a-ab88-9e59d556c7ee] +tempest.api.network.admin.test_ports.PortsAdminExtendedAttrsIpV6TestJSON.test_list_ports_binding_ext_attr[compute,id-1c82a44a-6c6e-48ff-89e1-abe7eaf8f9f8] +tempest.api.network.test_versions.NetworksApiDiscovery.test_api_version_resources[id-cac8a836-c2e0-4304-b556-cd299c7281d1,smoke] +tempest.api.compute.images.test_images_oneserver_negative.ImagesOneServerNegativeTestJSON.test_create_image_specify_name_over_character_limit[id-084f0cbc-500a-4963-8a4e-312905862581,negative] +tempest.api.image.v2.test_images_member_negative.ImagesMemberNegativeTest.test_image_share_invalid_status[id-b79efb37-820d-4cf0-b54c-308b00cf842c,negative] +tempest.api.compute.volumes.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_size_zero[id-8cce995e-0a83-479a-b94d-e1e40b8a09d1,negative] +tempest.api.compute.admin.test_live_migration.LiveAutoBlockMigrationV225Test.test_live_block_migration[id-1dce86b8-eb04-4c03-a9d8-9c1dc3ee0c7b] +tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_update_server_metadata[id-344d981e-0c33-4997-8a5d-6c1d803e4134] +tempest.api.compute.admin.test_flavors_extra_specs_negative.FlavorsExtraSpecsNegativeTestJSON.test_flavor_update_more_key[id-f5889590-bf66-41cc-b4b1-6e6370cfd93f,negative] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_active_status[id-ca78e20e-fddb-4ce6-b7f7-bcbf8605e66e] +tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsTestJSON.test_allocate_floating_ip[id-f7bfb946-297e-41b8-9e8c-aba8e9bb5194] +tempest.api.identity.admin.v3.test_users.UsersV3TestJSON.test_password_history_not_enforced_in_admin_reset[id-568cd46c-ee6c-4ab4-a33a-d3791931979e] +tempest.api.network.admin.test_routers.RoutersIpV6AdminTest.test_update_router_set_gateway_without_snat[id-96536bc7-8262-4fb2-9967-5c46940fa279] +tempest.api.identity.v3.test_access_rules.AccessRulesV3Test.test_show_access_rule[id-795dd507-ca1e-40e9-ba90-ff0a08689ba4] +tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON.test_list_images_filter_by_server_ref[id-05a377b8-28cf-4734-a1e6-2ab5c38bf606] +tempest.api.identity.admin.v2.test_roles.RolesTestJSON.test_list_user_roles[id-262e1e3e-ed71-4edd-a0e5-d64e83d66d05] +tempest.api.volume.admin.test_volume_quotas_negative.VolumeQuotasNegativeTestJSON.test_volume_extend_gigabytes_quota_deviation[id-d321dc21-d8c6-401f-95fe-49f4845f1a6d,negative] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_create_update_port_with_second_ip[id-63aeadd4-3b49-427f-a3b1-19ca81f06270] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents_with_prefix[id-77e742c7-caf2-4ec9-8aa4-f7d509a3344c] +tempest.api.compute.volumes.test_volumes_negative.VolumesNegativeTest.test_delete_volume_without_passing_volume_id[id-0d1417c5-4ae8-4c2c-adc5-5f0b864253e5,negative] +tempest.api.compute.servers.test_device_tagging.TaggedBootDevicesTest_v242.test_tagged_boot_devices[id-a2e65a6c-66f1-4442-aaa8-498c31778d96,image,network,slow,volume] +tempest.api.compute.admin.test_quotas.QuotaClassesAdminTestJSON.test_update_default_quotas[id-7932ab0f-5136-4075-b201-c0e2338df51a] +tempest.api.network.test_security_groups.SecGroupIPv6Test.test_create_security_group_rule_with_protocol_integer_value[id-0a307599-6655-4220-bebc-fd70c64f2290] +tempest.api.network.test_service_providers.ServiceProvidersTest.test_service_providers_list[id-2cbbeea9-f010-40f6-8df5-4eaa0c918ea6] +tempest.api.compute.admin.test_fixed_ips_negative.FixedIPsNegativeTestJson.test_set_unreserve_with_non_admin_user[id-f1f7a35b-0390-48c5-9803-5f27461439db,negative] +tempest.api.compute.servers.test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_create_server_metadata_blank_key[id-92431555-4d8b-467c-b95b-b17daa5e57ff,negative] +tempest.api.compute.keypairs.test_keypairs_v22.KeyPairsV22TestJSON.test_keypairs_create_list_delete[id-1d1dbedb-d7a0-432a-9d09-83f543c3c19b] +tempest.api.compute.servers.test_attach_interfaces.AttachInterfacesV270Test.test_create_get_list_interfaces[id-2853f095-8277-4067-92bd-9f10bd4f8e0c,network] +tempest.api.compute.admin.test_servers_on_multinodes.ServersOnMultiNodesTest.test_create_server_with_scheduler_hint_group_affinity[id-9d2e924a-baf4-11e7-b856-fa163e65f5ce] +tempest.scenario.test_shelve_instance.TestShelveInstance.test_shelve_instance[compute,id-1164e700-0af0-4a4c-8792-35909a88743c,image,network,slow] +tempest.api.compute.admin.test_agents.AgentsAdminTestJSON.test_delete_agent[id-470e0b89-386f-407b-91fd-819737d0b335] +tempest.api.object_storage.test_object_expiry.ObjectExpiryTest.test_get_object_at_expiry_time[id-e592f18d-679c-48fe-9e36-4be5f47102c5] +tempest.api.compute.servers.test_multiple_create_negative.MultipleCreateNegativeTestJSON.test_max_count_non_integer[id-9c5698d1-d7af-4c80-b971-9d403135eea2,negative] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_with_detail_param_marker[id-46eff077-100b-427f-914e-3db2abcdb7e2] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_pass_string[id-679bc053-5e70-4514-9800-3dfab1a380a6,negative] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_metadata[id-96e68f0e-19ec-4aa2-86f3-adc6a45e14dd,smoke] +tempest.api.compute.images.test_images_negative.ImagesNegativeTestJSON.test_create_image_specify_uuid_35_characters_or_less[id-ec176029-73dc-4037-8d72-2e4ff60cf538,negative] +tempest.api.network.admin.test_external_networks_negative.ExternalNetworksAdminNegativeTestJSON.test_create_port_with_precreated_floatingip_as_fixed_ip[id-d402ae6c-0be0-4d8e-833b-a738895d98d0,negative] +tempest.api.compute.flavors.test_flavors_negative.FlavorsV2NegativeTest.test_boot_with_low_ram[id-90f0d93a-91c1-450c-91e6-07d18172cefe,image,negative] +tempest.api.compute.images.test_images_negative.ImagesDeleteNegativeTestJSON.test_delete_image_non_hex_string_id[id-924540c3-f1f1-444c-8f58-718958b6724e,negative] +tempest.api.compute.admin.test_volume_swap.TestVolumeSwap.test_volume_swap[id-1769f00d-a693-4d67-a631-6a3496773813,slow,volume] +tempest.api.compute.servers.test_delete_server.DeleteServersTestJSON.test_delete_server_while_in_attached_volume[id-d0f3f0d6-d9b6-4a32-8da4-23015dcab23c,volume] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filtered_by_ip[id-43a1242e-7b31-48d1-88f2-3f72aa9f2077] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_unreserve_volume_with_nonexistent_volume_id[id-eb467654-3dc1-4a72-9b46-47c29d22654c,negative] +tempest.api.object_storage.test_object_services.ObjectTest.test_list_object_metadata[id-9a447cf6-de06-48de-8226-a8c6ed31caf2,smoke] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_show_host_detail_with_non_admin_user[id-19ebe09c-bfd4-4b7c-81a2-e2e0710f59cc,negative] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_no_container_metadata[id-a2faf936-6b13-4f8d-92a2-c2278355821e] +tempest.api.compute.admin.test_hypervisor_negative.HypervisorAdminNegativeTestJSON.test_get_hypervisor_list_with_non_admin_user[id-51b3d536-9b14-409c-9bce-c6f7c794994e,negative] +tempest.api.compute.admin.test_flavors.FlavorsAdminTestJSON.test_create_flavor_with_int_id[id-8b4330e1-12c4-4554-9390-e6639971f086] +tempest.api.object_storage.test_container_services_negative.ContainerNegativeTest.test_list_all_container_objects_on_deleted_container[id-86b2ab08-92d5-493d-acd2-85f0c848819e,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_invalid_ip_v6_address[id-5226dd80-1e9c-4d8a-b5f9-b26ca4763fd0,negative] +tempest.api.image.v2.test_images_metadefs_schema.MetadataSchemaTest.test_get_metadata_resources_types_schema[id-dfe4a891-b38b-3bf0-a3b2-e03ee67b3a3a] +tempest.api.identity.admin.v2.test_tenant_negative.TenantsNegativeTestJSON.test_list_tenant_request_without_token[id-df33926c-1c96-4d8d-a762-79cc6b0c3cf4,negative] +tempest.api.network.admin.test_routers.RoutersAdminTest.test_create_router_setting_project_id[id-e54dd3a3-4352-4921-b09d-44369ae17397] +tempest.api.image.v1.test_image_members_negative.ImageMembersNegativeTest.test_delete_member_with_non_existing_tenant[id-f5720333-dd69-4194-bb76-d2f048addd56,negative] +tempest.api.compute.admin.test_auto_allocate_network.AutoAllocateNetworkTest.test_server_create_no_allocate[id-5eb7b8fa-9c23-47a2-9d7d-02ed5809dd34] +tempest.api.compute.admin.test_aggregates.AggregatesAdminTestJSON.test_aggregate_create_delete[id-0d148aa3-d54c-4317-aa8d-42040a475e20] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_remove_user_role_request_without_token[id-cac81cf4-c1d2-47dc-90d3-f2b7eb572286,negative] +tempest.api.network.test_networks.NetworksIpV6TestAttrs.test_create_delete_subnet_with_v6_attributes_stateful[id-da40cd1b-a833-4354-9a85-cd9b8a3b74ca] +tempest.api.compute.admin.test_aggregates.AggregatesAdminTestJSON.test_aggregate_create_update_metadata_get_details[id-36ec92ca-7a73-43bc-b920-7531809e8540] +tempest.api.network.test_subnetpools_extensions.SubnetPoolsTestJSON.test_create_list_show_update_delete_subnetpools[id-62595970-ab1c-4b7f-8fcc-fddfe55e9811,smoke] +tempest.api.identity.admin.v2.test_services.ServicesTestJSON.test_create_service_without_description[id-5d3252c8-e555-494b-a6c8-e11d7335da42] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_list_ports_fields[id-ff7f117f-f034-4e0e-abff-ccef05c454b4] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_suspend_server_invalid_state[id-7f323206-05a9-4bf8-996b-dd5b2036501b,negative] +tempest.api.identity.admin.v3.test_domains_negative.DomainsNegativeTestJSON.test_domain_create_duplicate[id-e6f9e4a2-4f36-4be8-bdbc-4e199ae29427,negative] +tempest.api.volume.test_volume_transfers.VolumesTransfersV355Test.test_create_list_delete_volume_transfer[id-af4a5b97-0859-4f31-aa3c-85b05bb63322] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_delete_server_pass_negative_id[id-75f79124-277c-45e6-a373-a1d6803f4cc4,negative] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_grant_list_revoke_role_to_group_on_project[id-cbf11737-1904-4690-9613-97bcbb3df1c4] +tempest.api.compute.admin.test_agents.AgentsAdminTestJSON.test_list_agents_with_filter[id-eabadde4-3cd7-4ec4-a4b5-5a936d2d4408] +tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON.test_list_images_with_detail_limit_results[id-ba2fa9a9-b672-47cc-b354-3b4c0600e2cb] +tempest.api.compute.admin.test_flavors_extra_specs_negative.FlavorsExtraSpecsNegativeTestJSON.test_flavor_non_admin_set_keys[id-a00a3b81-5641-45a8-ab2b-4a8ec41e1d7d,negative] +tempest.api.object_storage.test_container_acl_negative.ObjectACLsNegativeTest.test_write_object_without_rights[id-a3a585a7-d8cf-4b65-a1a0-edc2b1204f85,negative] +tempest.api.object_storage.test_object_services.PublicObjectTest.test_access_public_container_object_without_using_creds[id-07c9cf95-c0d4-4b49-b9c8-0ef2c9b27193] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_rebuild_server[id-aaa6cdf3-55a7-461a-add9-1c8596b9a07c] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_resume_non_existent_server[id-221cd282-bddb-4837-a683-89c2487389b6,negative] +tempest.scenario.test_security_groups_basic_ops.TestSecurityGroupsBasicOps.test_port_security_disable_security_group[compute,id-7c811dcc-263b-49a3-92d2-1b4d8405f50c,network,slow] +tempest.api.object_storage.test_object_temp_url.ObjectTempUrlTest.test_put_object_using_temp_url[id-9b08dade-3571-4152-8a4f-a4f2a873a735] +tempest.api.identity.admin.v2.test_tenant_negative.TenantsNegativeTestJSON.test_create_tenant_request_without_token[id-a3ee9d7e-6920-4dd5-9321-d4b2b7f0a638,negative] +tempest.api.compute.servers.test_server_tags.ServerTagsTestJSON.test_update_all_tags[id-a2c1af8c-127d-417d-974b-8115f7e3d831] +tempest.api.network.admin.test_dhcp_agent_scheduler.DHCPAgentSchedulersTestJSON.test_add_remove_network_from_dhcp_agent[id-a0856713-6549-470c-a656-e97c8df9a14d] +tempest.api.compute.admin.test_live_migration.LiveMigrationRemoteConsolesV26Test.test_live_migration_serial_console[id-6190af80-513e-4f0f-90f2-9714e84955d7] +tempest.api.compute.admin.test_agents.AgentsAdminTestJSON.test_list_agents[id-6a326c69-654b-438a-80a3-34bcc454e138] +tempest.api.compute.admin.test_servers.ServersAdminTestJSON.test_list_servers_by_admin_with_specified_tenant[id-7e5d6b8f-454a-4ba1-8ae2-da857af8338b] +tempest.api.compute.images.test_list_images.ListImagesTestJSON.test_list_images[id-fd51b7f4-d4a3-4331-9885-866658112a6f] +tempest.api.network.admin.test_ports.PortsAdminExtendedAttrsIpV6TestJSON.test_create_port_binding_ext_attr[compute,id-8e8569c1-9ac7-44db-8bc1-f5fb2814f29b] +tempest.api.compute.servers.test_availability_zone.AZV2TestJSON.test_get_availability_zone_list_with_non_admin_user[id-a8333aa2-205c-449f-a828-d38c2489bf25] +tempest.api.volume.test_volume_delete_cascade.VolumesDeleteCascade.test_volume_from_snapshot_cascade_delete[id-59a77ede-609b-4ee8-9f68-fc3c6ffe97b5] +tempest.api.compute.admin.test_servers.ServersAdminTestJSON.test_list_servers_filter_by_exist_host[id-86c7a8f7-50cf-43a9-9bac-5b985317134f] +tempest.api.network.test_ports.PortsTestJSON.test_port_list_filter_by_ip[id-e7fe260b-1e79-4dd3-86d9-bec6a7959fc5] +tempest.api.identity.admin.v3.test_oauth_consumers.OAUTHConsumersV3Test.test_update_consumer[id-080a9b1a-c009-47c0-9979-5305bf72e3dc] +tempest.api.compute.servers.test_multiple_create_negative.MultipleCreateNegativeTestJSON.test_max_count_less_than_one[id-a6f9c2ab-e060-4b82-b23c-4532cb9390ff,negative] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_create_additional_default_security_group_fails[id-2323061e-9fbf-4eb0-b547-7e8fafc90849,negative] +tempest.api.object_storage.test_container_services_negative.ContainerNegativeTest.test_get_metadata_headers_with_invalid_container_name[id-1a95ab2e-b712-4a98-8a4d-8ce21b7557d6,negative] +tempest.api.compute.security_groups.test_security_group_rules_negative.SecurityGroupRulesNegativeTestJSON.test_delete_security_group_rule_with_non_existent_id[id-56fddcca-dbb8-4494-a0db-96e9f869527c,negative] +tempest.api.volume.admin.test_multi_backend.VolumeMultiBackendTest.test_backend_name_reporting_with_prefix[id-f38e647f-ab42-4a31-a2e7-ca86a6485215] +tempest.api.volume.test_volumes_actions.VolumesActionsTest.test_volume_upload[id-d8f1ca95-3d5b-44a3-b8ca-909691c9532d,image] +tempest.scenario.test_unified_limits.ImageQuotaTest.test_image_count_uploading_quota[id-b103788b-5329-4aa9-8b0d-97f8733460db,identity,image] +tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON.test_list_flavors_detailed_using_marker[id-6db2f0c0-ddee-4162-9c84-0703d3dd1107] +tempest.api.object_storage.test_account_services.AccountTest.test_list_no_account_metadata[id-b904c2e3-24c2-4dba-ad7d-04e90a761be5] +tempest.api.compute.admin.test_volumes_negative.VolumesAdminNegativeTest.test_update_attached_volume_with_nonexistent_volume_in_body[id-7dcac15a-b107-46d3-a5f6-cb863f4e454a,negative] +tempest.api.object_storage.test_object_slo.ObjectSloTest.test_list_large_object_metadata[id-e69ad766-e1aa-44a2-bdd2-bf62c09c1456] +tempest.api.identity.admin.v3.test_domain_configuration.DomainConfigurationTestJSON.test_show_default_group_config_and_options[id-11a02bf0-6f94-4380-b3b0-c8dc18fc0d22] +tempest.api.network.admin.test_routers.RoutersAdminTest.test_update_router_reset_gateway_without_snat[id-f2faf994-97f4-410b-a831-9bc977b64374] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_size_zero[id-41331caa-eaf4-4001-869d-bc18c1869360,negative] +tempest.api.compute.servers.test_servers.ServersTestJSON.test_update_server_name[id-5e6ccff8-349d-4852-a8b3-055df7988dd2] +tempest.api.network.test_routers_negative.RoutersNegativeTest.test_add_router_interfaces_on_overlapping_subnets_returns_400[id-957751a3-3c68-4fa2-93b6-eb52ea10db6e,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_delete_server_pass_id_exceeding_length_limit[id-f4d7279b-5fd2-4bf2-9ba4-ae35df0d18c5,negative] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_port_list_filter_by_router_id[id-5ad01ed0-0e6e-4c5d-8194-232801b15c72] +tempest.api.compute.admin.test_flavors_extra_specs.FlavorsExtraSpecsTestJSON.test_flavor_non_admin_get_specific_key[id-12805a7f-39a3-4042-b989-701d5cad9c90] +tempest.api.compute.admin.test_hosts.HostsAdminTestJSON.test_list_hosts_with_a_blank_zone[id-9af3c171-fbf4-4150-a624-22109733c2a6] +tempest.api.object_storage.test_account_bulk.BulkTest.test_extract_archive[id-a407de51-1983-47cc-9f14-47c2b059413c] +tempest.api.compute.admin.test_servers_negative.ServersAdminNegativeTestJSON.test_reset_state_server_nonexistent_server[id-e741298b-8df2-46f0-81cb-8f814ff2504c,negative] +tempest.api.object_storage.test_container_sync.ContainerSyncTest.test_container_synchronization[id-be008325-1bba-4925-b7dd-93b58f22ce9b,slow] +tempest.api.compute.images.test_images_negative.ImagesNegativeTestJSON.test_create_image_specify_uuid_37_characters_or_more[id-36741560-510e-4cc2-8641-55fe4dfb2437,negative] +tempest.api.image.v2.test_images_member.ImagesMemberTest.test_image_share_accept[id-5934c6ea-27dc-4d6e-9421-eeb5e045494a] +tempest.api.compute.admin.test_servers_negative.ServersAdminNegativeTestJSON.test_resize_server_using_overlimit_vcpus[id-7368a427-2f26-4ad9-9ba9-911a0ec2b0db,negative] +tempest.api.identity.admin.v3.test_default_project_id.TestDefaultProjectId.test_default_project_id[id-d6110661-6a71-49a7-a453-b5e26640ff6d] +tempest.api.network.test_networks.NetworksIpV6TestAttrs.test_create_delete_slaac_subnet_with_ports[id-88554555-ebf8-41ef-9300-4926d45e06e9] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_add_host_as_user[id-7324c334-bd13-4c93-8521-5877322c3d51,negative] +tempest.scenario.test_snapshot_pattern.TestSnapshotPattern.test_snapshot_pattern[compute,id-608e604b-1d63-4a82-8e3e-91bc665c90b4,image,network,slow] +tempest.api.compute.admin.test_servers.ServersAdminTestJSON.test_list_servers_filter_by_error_status[id-06f960bb-15bb-48dc-873d-f96e89be7870] +tempest.api.compute.servers.test_create_server.ServersTestJSON.test_verify_server_details[id-5de47127-9977-400a-936f-abcfbec1218f,smoke] +tempest.api.network.test_networks_negative.NetworksNegativeTestJSON.test_show_non_existent_subnet[id-d746b40c-5e09-4043-99f7-cba1be8b70df,negative] +tempest.api.compute.servers.test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_set_metadata_invalid_key[id-0025fbd6-a4ba-4cde-b8c2-96805dcfdabc,negative] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcp_stateful_fixedips_outrange[id-98244d88-d990-4570-91d4-6b25d70d08af] +tempest.api.compute.servers.test_novnc.NoVNCConsoleTestJSON.test_novnc_bad_token[id-f9c79937-addc-4aaa-9e0e-841eef02aeb7] +tempest.api.object_storage.test_object_formpost_negative.ObjectFormPostNegativeTest.test_post_object_using_form_expired[id-d3fb3c4d-e627-48ce-9379-a1631f21336d,negative] +tempest.api.object_storage.test_container_services.ContainerTest.test_create_container_overwrite[id-49f866ed-d6af-4395-93e7-4187eb56d322] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_create_with_invalid_group_name[id-1759c3cb-b0fc-44b7-86ce-c99236be911d,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_restore_server_invalid_state[id-7fcadfab-bd6a-4753-8db7-4a51e51aade9,negative] +tempest.api.network.admin.test_routers_dvr.RoutersTestDVR.test_distributed_router_creation[id-08a2a0a8-f1e4-4b34-8e30-e522e836c44e] +tempest.api.object_storage.test_object_services.ObjectTest.test_create_object_with_x_fresh_metadata[id-0f3d62a6-47e3-4554-b0e5-1a5dc372d501] +tempest.api.volume.admin.test_volume_quotas_negative.VolumeQuotasNegativeTestJSON.test_quota_volume_gigabytes[id-2dc27eee-8659-4298-b900-169d71a91374,negative] +tempest.api.volume.test_image_metadata.VolumesImageMetadata.test_update_show_delete_image_metadata[id-03efff0b-5c75-4822-8f10-8789ac15b13e,image] +tempest.api.volume.test_volume_absolute_limits.AbsoluteLimitsTests.test_get_volume_absolute_limits[id-8e943f53-e9d6-4272-b2e9-adcf2f7c29ad] +tempest.api.network.admin.test_metering_extensions.MeteringTestJSON.test_show_metering_label_rule[id-b7354489-96ea-41f3-9452-bace120fb4a7] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_remove_server_all_security_groups[id-1d1c9104-1b0a-11e7-a3d4-fa163e65f5ce] +tempest.api.volume.test_volumes_snapshots.VolumesSnapshotTestJSON.test_snapshot_backup[id-bbcfa285-af7f-479e-8c1a-8c34fc16543c] +tempest.api.identity.admin.v3.test_endpoints_negative.EndpointsNegativeTestJSON.test_create_with_enabled_False[id-ac6c137e-4d3d-448f-8c83-4f13d0942651,negative] +tempest.api.network.test_networks.NetworksIpV6Test.test_external_network_visibility[id-af774677-42a9-4e4b-bb58-16fe6a5bc1ec,smoke] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcp_stateful_fixedips[id-51a5e97f-f02e-4e4e-9a17-a69811d300e3] +tempest.api.network.test_allowed_address_pair.AllowedAddressPairIpV6TestJSON.test_update_port_with_multiple_ip_mac_address_pair[id-b3f20091-6cd5-472b-8487-3516137df933] +tempest.api.network.admin.test_routers.RoutersAdminTest.test_update_router_unset_gateway[id-ad81b7ee-4f81-407b-a19c-17e623f763e8] +tempest.api.volume.admin.test_volume_services.VolumesServicesTestJSON.test_get_service_by_service_and_host_name[id-ffa6167c-4497-4944-a464-226bbdb53908] +tempest.api.network.test_ports.PortsTestJSON.test_create_show_delete_port_user_defined_mac[id-13e95171-6cbd-489c-9d7c-3f9c58215c18] +tempest.api.object_storage.test_container_services.ContainerTest.test_create_container_with_metadata_value[id-e1e8df32-7b22-44e1-aa08-ccfd8d446b58] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_update_volume_with_nonexistent_volume_id[id-0186422c-999a-480e-a026-6a665744c30c,negative] +tempest.api.volume.test_versions.VersionsTest.test_list_versions[id-77838fc4-b49b-4c64-9533-166762517369,smoke] +tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON.test_list_images_with_detail_filter_by_name[id-644ea267-9bd9-4f3b-af9f-dffa02396a17] +tempest.api.compute.servers.test_create_server.ServersTestJSON.test_host_name_is_same_as_server_name[id-ac1ad47f-984b-4441-9274-c9079b7a0666] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_flavor[id-80c574cc-0925-44ba-8602-299028357dd9] +tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_get_server_metadata_item[id-3043c57d-7e0e-49a6-9a96-ad569c265e6a] +tempest.api.compute.admin.test_quotas.QuotasAdminTestJSON.test_get_default_quotas[id-3b0a7c8f-cf58-46b8-a60c-715a32a8ba7d] +tempest.api.volume.admin.test_volume_types_extra_specs_negative.ExtraSpecsNegativeTest.test_update_nonexistent_type_id[id-474090d2-0824-eb3b-9335-f506b4aa49d8,negative] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_update_volume_with_invalid_volume_id[id-e66e40d6-65e6-4e75-bdc7-636792fa152d,negative] +tempest.api.network.test_networks.NetworksIpV6Test.test_create_delete_subnet_without_gateway[id-d2d596e2-8e76-47a9-ac51-d4648009f4d3] +tempest.api.network.test_networks.NetworksIpV6TestAttrs.test_create_delete_subnet_with_v6_attributes_stateless[id-7d410310-8c86-4902-adf9-865d08e31adb] +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_gw_and_allocation_pools[id-8217a149-0c6c-4cfb-93db-0486f707d13f] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_rebuild_server_in_stop_state[id-30449a88-5aff-4f9b-9866-6ee9b17f906d] +tempest.api.volume.admin.test_volume_quotas.VolumeQuotasAdminTestJSON.test_show_quota_usage[id-18c51ae9-cb03-48fc-b234-14a19374dbed] +tempest.api.identity.admin.v3.test_domain_configuration.DomainConfigurationTestJSON.test_create_update_and_delete_domain_config[id-7161023e-5dd0-4612-9da0-1bac6ac30b63] +tempest.api.identity.admin.v3.test_list_users.UsersV3TestJSON.test_list_users_with_name[id-c285bb37-7325-4c02-bff3-3da5d946d683] +tempest.api.identity.v3.test_ec2_credentials.EC2CredentialsTest.test_show_ec2_credential[id-8b8d1010-5958-48df-a6cd-5e3df72e6bcf] +tempest.api.compute.volumes.test_volumes_negative.VolumesNegativeTest.test_create_volume_without_passing_size[id-131cb3a1-75cc-4d40-b4c3-1317f64719b0,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_resize_server_with_null_flavor[id-45436a7d-a388-4a35-a9d8-3adc5d0d940b,negative] +tempest.api.compute.admin.test_hypervisor.HypervisorAdminTestJSON.test_get_hypervisor_list_details[id-1e7fdac2-b672-4ad1-97a4-bad0e3030118] +tempest.api.compute.admin.test_simple_tenant_usage_negative.TenantUsagesNegativeTestJSON.test_list_usage_all_tenants_with_non_admin_user[id-bbe6fe2c-15d8-404c-a0a2-44fad0ad5cc7,negative] +tempest.api.object_storage.test_object_services.ObjectTest.test_get_object_with_x_object_manifest[id-11b4515b-7ba7-4ca8-8838-357ded86fc10] +tempest.api.image.v2.test_images.MultiStoresImportImagesTest.test_glance_direct_import_image_to_specific_stores[id-82fb131a-dd2b-11ea-aec7-340286b6c574] +tempest.api.identity.v3.test_users.IdentityV3UsersTest.test_password_history_check_self_service_api[id-941784ee-5342-4571-959b-b80dd2cea516] +tempest.api.compute.admin.test_flavors_access_negative.FlavorsAccessNegativeTestJSON.test_remove_flavor_access_not_found[id-1f710927-3bc7-4381-9f82-0ca6e42644b7,negative] +tempest.api.compute.volumes.test_attach_volume_negative.AttachVolumeNegativeTest.test_attach_attached_volume_to_different_server[id-ee37a796-2afb-11e7-bc0f-fa163e65f5ce,negative] +tempest.api.compute.admin.test_hypervisor.HypervisorAdminUnderV252Test.test_search_hypervisor[id-d7e1805b-3b14-4a3b-b6fd-50ec6d9f361f] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_create_user_with_empty_name[id-d80d0c2f-4514-4d1e-806d-0930dfc5a187,negative] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_create_security_group_rule_with_remote_ip_and_group[id-b5c4b247-6b02-435b-b088-d10d45650881,negative] +tempest.api.object_storage.test_object_formpost.ObjectFormPostTest.test_post_object_using_form[id-80fac02b-6e54-4f7b-be0d-a965b5cbef76] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_startup_nonexistent_host[id-0d981ac3-4320-4898-b674-82b61fbb60e4,negative] +tempest.api.network.admin.test_routers_negative.RoutersAdminNegativeTest.test_router_set_gateway_used_ip_returns_409[id-7101cc02-058a-11e7-93e1-fa163e4fa634,negative] +tempest.api.volume.admin.test_volumes_actions.VolumesActionsTest.test_volume_force_delete_when_volume_is_maintenance[id-b957cabd-1486-4e21-90cf-a9ed3c39dfb2] +tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_server_with_ipv6_addr_only[id-38fb1d02-c3c5-41de-91d3-9bc2025a75eb] +tempest.api.compute.floating_ips.test_list_floating_ips.FloatingIPDetailsTestJSON.test_list_floating_ip_pools[id-df389fc8-56f5-43cc-b290-20eda39854d3] +tempest.api.volume.test_volumes_snapshots.VolumesSnapshotTestJSON.test_volume_from_snapshot[id-677863d1-3142-456d-b6ac-9924f667a7f4] +tempest.api.object_storage.test_account_quotas.AccountQuotasTest.test_admin_modify_quota[id-63f51f9f-5f1d-4fc6-b5be-d454d70949d6,smoke] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_non_existent_security_group[id-27edee9c-873d-4da6-a68a-3c256efebe8f,negative] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_list_volumes_with_nonexistent_name[id-0f4aa809-8c7b-418f-8fb3-84c7a5dfc52f,negative] +tempest.api.object_storage.test_object_services.ObjectTest.test_create_object_with_x_remove_object_metakey[id-ad21e342-7916-4f9e-ab62-a1f885f2aaf9] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_create_as_user[id-86a1cb14-da37-4a70-b056-903fd56dfe29,negative] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_list_volumes_with_invalid_status[id-143b279b-7522-466b-81be-34a87d564a7c,negative] +tempest.api.object_storage.test_object_services.ObjectTest.test_create_object_with_x_remove_object_meta[id-ce798afc-b278-45de-a5ce-2ea124b98b99] +tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_delete_image[id-f848bb94-1c6e-45a4-8726-39e3a5b23535,smoke] +tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON.test_list_images_filter_by_changes_since[id-18bac3ae-da27-436c-92a9-b22474d13aab] +tempest.api.compute.admin.test_services_negative.ServicesAdminNegativeV253TestJSON.test_list_services_with_non_admin_user[id-1126d1f8-266e-485f-a687-adc547492646,negative] +tempest.api.compute.admin.test_flavors_extra_specs_negative.FlavorsExtraSpecsNegativeTestJSON.test_flavor_update_mismatch_key[id-25b822b8-9f49-44f6-80de-d99f0482e5cb,negative] +tempest.api.object_storage.test_container_acl_negative.ObjectACLsNegativeTest.test_delete_object_without_using_creds[id-af85af0b-a025-4e72-a90e-121babf55720,negative] +tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON.test_list_flavors_detailed_filter_by_min_ram[id-09fe7509-b4ee-4b34-bf8b-39532dc47292] +tempest.api.network.admin.test_ports.PortsAdminExtendedAttrsTestJSON.test_update_port_binding_ext_attr[compute,id-6f6c412c-711f-444d-8502-0ac30fbf5dd5] +tempest.api.compute.images.test_list_image_filters_negative.ListImageFiltersNegativeTestJSON.test_get_nonexistent_image[id-391b0440-432c-4d4b-b5da-c5096aa247eb,negative] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_create_show_delete_port_user_defined_mac[id-13e95171-6cbd-489c-9d7c-3f9c58215c18] +tempest.api.compute.admin.test_servers.ServersAdminTestJSON.test_list_servers_by_admin[id-51717b38-bdc1-458b-b636-1cf82d99f62f] +tempest.api.compute.servers.test_server_rescue.ServerRescueTestJSONUnderV235.test_rescued_vm_add_remove_security_group[id-affca41f-7195-492d-8065-e09eee245404] +tempest.api.volume.admin.test_volume_types_extra_specs_negative.ExtraSpecsNegativeTest.test_update_none_extra_spec_id[id-9bf7a657-b011-4aec-866d-81c496fbe5c8,negative] +tempest.api.volume.test_volumes_snapshots.VolumesSnapshotTestJSON.test_snapshot_create_get_list_update_delete[id-2a8abbe4-d871-46db-b049-c41f5af8216e] +tempest.api.network.test_extra_dhcp_options.ExtraDHCPOptionsTestJSON.test_update_show_port_with_extra_dhcp_options[id-9a6aebf4-86ee-4f47-b07a-7f7232c55607] +tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_associate_ip_to_server_with_floating_ip[id-58a80596-ffb2-11e6-9393-fa163e4fa634,negative] +tempest.api.compute.admin.test_servers_negative.ServersAdminNegativeTestJSON.test_migrate_server_invalid_state[id-b0b17f83-d14e-4fc4-8f31-bcc9f3cfa629,negative] +tempest.api.identity.admin.v2.test_endpoints.EndPointsTestJSON.test_create_list_delete_endpoint[id-9974530a-aa28-4362-8403-f06db02b26c1] +tempest.api.image.v1.test_images_negative.CreateDeleteImagesNegativeTest.test_delete_non_existent_image[id-ec652588-7e3c-4b67-a2f2-0fa96f57c8fc,negative] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_from_image_with_decreasing_size[id-5b810c91-0ad1-47ce-aee8-615f789be78f,image,negative] +tempest.api.object_storage.test_object_services.ObjectTest.test_create_object_with_content_disposition[id-5daebb1d-f0d5-4dc9-b541-69672eff00b0] +tempest.api.image.v2.test_images_metadefs_schema.MetadataSchemaTest.test_get_metadata_tags_schema[id-cde4a891-b38b-3bf0-a3b2-e03ee67b3a3a] +tempest.api.identity.admin.v2.test_tenants.TenantsTestJSON.test_tenant_update_name[id-781f2266-d128-47f3-8bdb-f70970add238] +tempest.api.identity.admin.v2.test_tenant_negative.TenantsNegativeTestJSON.test_update_non_existent_tenant[id-bd20dc2a-9557-4db7-b755-f48d952ad706,negative] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_delete_volume_without_passing_volume_id[id-441a1550-5d44-4b30-af0f-a6d402f52026,negative] +tempest.api.network.admin.test_routers.RoutersAdminTest.test_create_router_with_default_snat_value[id-847257cc-6afd-4154-b8fb-af49f5670ce8] +tempest.api.network.test_security_groups.SecGroupIPv6Test.test_create_security_group_rule_with_additional_args[id-87dfbcf9-1849-43ea-b1e4-efa3eeae9f71] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_authentication_with_invalid_tenant[id-921f1ad6-7907-40b8-853f-637e7ee52178,negative] +tempest.api.network.test_networks.NetworksTest.test_list_networks[id-f7ffdeda-e200-4a7a-bcbe-05716e86bf43,smoke] +tempest.api.compute.servers.test_create_server.ServersTestJSON.test_list_servers[id-9a438d88-10c6-4bcd-8b5b-5b6e25e1346f,smoke] +tempest.api.compute.images.test_images.ImagesTestJSON.test_create_image_from_suspended_server[id-8ca07fec-0262-11e7-907e-fa163e4fa634] +tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_remote_ip_prefix[id-16459776-5da2-4634-bce4-4b55ee3ec188] +tempest.api.compute.images.test_image_metadata.ImagesMetadataTestJSON.test_get_image_metadata_item[id-4f5db52f-6685-4c75-b848-f4bb363f9aa6] +tempest.api.compute.images.test_image_metadata.ImagesMetadataTestJSON.test_set_image_metadata_item[id-f2de776a-4778-4d90-a5da-aae63aee64ae] +tempest.api.identity.admin.v2.test_users.UsersTestJSON.test_delete_user[id-29ed26f4-a74e-4425-9a85-fdb49fa269d2] +tempest.api.volume.admin.test_snapshots_actions.SnapshotsActionsTest.test_snapshot_force_delete_when_snapshot_is_error_deleting[id-bf89080f-8129-465e-9327-b2f922666ba5] +tempest.scenario.test_unified_limits.ImageQuotaTest.test_image_count_quota[id-9b74fe24-183b-41e6-bf42-84c2958a7be8,identity,image] +tempest.api.image.v2.test_images_tags_negative.ImagesTagsNegativeTest.test_delete_non_existing_tag[id-39c023a2-325a-433a-9eea-649bf1414b19,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_non_existent_keypair[id-7a2efc39-530c-47de-b875-2dd01c8d39bd,negative] +tempest.api.volume.admin.test_volume_services_negative.VolumeServicesNegativeTest.test_thaw_host_with_invalid_host[id-7c6287c9-d655-47e1-9a11-76f6657a6dce,negative] +tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_list_api_versions[id-721f480f-35b6-46c7-846e-047e6acea0dc,smoke] +tempest.api.network.admin.test_metering_extensions.MeteringTestJSON.test_show_metering_label[id-30abb445-0eea-472e-bd02-8649f54a5968] +tempest.api.object_storage.test_object_services.ObjectTest.test_update_object_metadata_with_x_remove_object_metakey[id-9a88dca4-b684-425b-806f-306cd0e57e42] +tempest.api.network.admin.test_negative_quotas.QuotasNegativeTest.test_network_quota_exceeding[id-644f4e1b-1bf9-4af0-9fd8-eb56ac0f51cf,negative] +tempest.scenario.test_object_storage_basic_ops.TestObjectStorageBasicOps.test_swift_basic_ops[id-b920faf1-7b8a-4657-b9fe-9c4512bfb381,object_storage] +tempest.api.object_storage.test_container_services.ContainerTest.test_create_container[id-92139d73-7819-4db1-85f8-3f2f22a8d91f,smoke] +tempest.api.identity.admin.v3.test_application_credentials.ApplicationCredentialsV3AdminTest.test_create_application_credential_with_roles[id-3b3dd48f-3388-406a-a9e6-4d078a552d0e] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_delete_non_existent_security_group[id-1f1bb89d-5664-4956-9fcd-83ee0fa603df,negative] +tempest.api.compute.keypairs.test_keypairs_negative.KeyPairsNegativeTestJSON.test_keypair_create_with_invalid_pub_key[id-29cca892-46ae-4d48-bc32-8fe7e731eb81,negative] +tempest.api.compute.admin.test_quotas.QuotasAdminTestV257.test_get_updated_quotas[id-e641e6c6-e86c-41a4-9e5c-9493c0ae47ad] +tempest.api.compute.servers.test_delete_server.DeleteServersTestJSON.test_delete_server_while_in_pause_state[id-943bd6e8-4d7a-4904-be83-7a6cc2d4213b] +tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_server_specify_multibyte_character_name[id-defbaca5-d611-49f5-ae21-56ee25d2db49] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_size_negative[id-8b472729-9eba-446e-a83b-916bdb34bef7,negative] +tempest.api.image.v1.test_image_members_negative.ImageMembersNegativeTest.test_add_member_with_non_existing_image[id-147a9536-18e3-45da-91ea-b037a028f364,negative] +tempest.api.volume.test_volume_transfers.VolumesTransfersTest.test_create_list_delete_volume_transfer[id-ab526943-b725-4c07-b875-8e8ef87a2c30] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_grant_list_revoke_role_to_group_on_system[id-c888fe4f-8018-48db-b959-542225c1b4b6] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_nonexistent_source_volid[id-47c73e08-4be8-45bb-bfdf-0c4e79b88344,negative] +tempest.api.object_storage.test_object_services.ObjectTest.test_update_object_metadata_with_create_and_remove_metadata[id-f726174b-2ded-4708-bff7-729d12ce1f84] +tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_update_router_admin_state[compute,id-04b9fe4e-85e8-4aea-b937-ea93885ac59f,network,slow] +tempest.api.network.test_routers_negative.RoutersNegativeTest.test_update_non_existent_router_returns_404[id-b23d1569-8b0c-4169-8d4b-6abd34fad5c7,negative] +tempest.api.network.admin.test_dhcp_agent_scheduler.DHCPAgentSchedulersTestJSON.test_list_networks_hosted_by_one_dhcp[id-30c48f98-e45d-4ffb-841c-b8aad57c7587] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_create_user_with_enabled_non_bool[id-23a2f3da-4a1a-41da-abdd-632328a861ad,negative] +tempest.api.compute.images.test_images_negative.ImagesDeleteNegativeTestJSON.test_delete_image_negative_image_id[id-68e2c175-bd26-4407-ac0f-4ea9ce2139ea,negative] +tempest.api.object_storage.test_container_acl_negative.ObjectACLsNegativeTest.test_write_object_with_non_authorized_user[id-63d84e37-55a6-42e2-9e5f-276e60e26a00,negative] +tempest.api.compute.images.test_images_negative.ImagesNegativeTestJSON.test_create_image_from_deleted_server[id-6cd5a89d-5b47-46a7-93bc-3916f0d84973,negative] +tempest.api.compute.admin.test_migrations.MigrationsAdminTest.test_revert_cold_migration[id-caa1aa8b-f4ef-4374-be0d-95f001c2ac2d] +tempest.api.compute.admin.test_fixed_ips.FixedIPsTestJson.test_list_fixed_ip_details[id-16b7d848-2f7c-4709-85a3-2dfb4576cc52] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_visibility[id-7a95bb92-d99e-4b12-9718-7bc6ab73e6d2] +tempest.api.network.test_networks_negative.NetworksNegativeTestJSON.test_delete_non_existent_subnet[id-a176c859-99fb-42ec-a208-8a85b552a239,negative] +tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers[id-9a438d88-10c6-4bcd-8b5b-5b6e25e1346f,smoke] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_reserve_volume_with_nonexistent_volume_id[id-ac6084c0-0546-45f9-b284-38a367e0e0e2,negative] +tempest.scenario.test_minbw_allocation_placement.MinBwAllocationPlacementTest.test_qos_min_bw_allocation_update_policy_from_zero[compute,id-9cfc3bb8-f433-4c91-87b6-747cadc8958a,network] +tempest.api.compute.servers.test_delete_server.DeleteServersTestJSON.test_delete_active_server[id-925fdfb4-5b13-47ea-ac8a-c36ae6fddb05] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_delete_image_null_id[id-32248db1-ab88-4821-9604-c7c369f1f88c,negative] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_create_security_group_rule_with_invalid_ports[id-0d9c7791-f2ad-4e2f-ac73-abf2373b0d2d,negative] +tempest.api.compute.servers.test_server_rescue.ServerStableDeviceRescueTestIDE.test_stable_device_rescue_cdrom_ide[id-947004c3-e8ef-47d9-9f00-97b74f9eaf96] +tempest.api.network.test_networks.NetworksIpV6Test.test_list_networks[id-f7ffdeda-e200-4a7a-bcbe-05716e86bf43,smoke] +tempest.api.identity.v3.test_ec2_credentials.EC2CredentialsTest.test_create_ec2_credential[id-b0f55a29-54e5-4166-999d-712347e0c920] +tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_live_migration[compute,id-03fd1562-faad-11e7-9ea0-fa163e65f5ce,network,slow] +tempest.api.identity.admin.v3.test_domains.DomainsTestJSON.test_list_domains[id-8cf516ef-2114-48f1-907b-d32726c734d4] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_authentication_with_invalid_username[id-bde9aecd-3b1c-4079-858f-beb5deaa5b5e,negative] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_volume_delete_nonexistent_volume_id[id-555efa6e-efcd-44ef-8a3b-4a7ca4837a29,negative] +tempest.api.compute.admin.test_migrations.MigrationsAdminTest.test_list_migrations[id-75c0b83d-72a0-4cf8-a153-631e83e7d53f] +tempest.api.network.test_floating_ips.FloatingIPTestJSON.test_floating_ip_delete_port[id-e1f6bffd-442f-4668-b30e-df13f2705e77] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_server_from_non_bootable_volume[id-74085be3-a370-4ca2-bc51-2d0e10e0f573,image,negative,volume] +tempest.api.network.admin.test_floating_ips_admin_actions.FloatingIPAdminTestJSON.test_create_list_show_floating_ip_with_tenant_id_by_admin[id-32727cc3-abe2-4485-a16e-48f2d54c14f2] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volumes_list_by_availability_zone[id-c0cfa863-3020-40d7-b587-e35f597d5d87] +tempest.api.identity.admin.v3.test_groups.GroupsV3TestJSON.test_group_create_update_get[id-2e80343b-6c81-4ac3-88c7-452f3e9d5129] +tempest.api.network.test_routers_negative.RoutersNegativeTest.test_router_add_gateway_net_not_external_returns_400[id-11836a18-0b15-4327-a50b-f0d9dc66bddd,negative] +tempest.api.identity.admin.v3.test_inherits.InheritsV3TestJSON.test_inherit_assign_list_revoke_user_roles_on_domain[id-3acf666e-5354-42ac-8e17-8b68893bcd36] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_details_with_multiple_params[id-2a7064eb-b9c3-429b-b888-33928fc5edd3] +tempest.api.object_storage.test_object_services.ObjectTest.test_copy_object_in_same_container[id-1a9ab572-1b66-4981-8c21-416e2a5e6011] +tempest.api.compute.servers.test_delete_server.DeleteServersTestJSON.test_delete_server_while_in_shelved_state[id-bb0cb402-09dd-4947-b6e5-5e7e1cfa61ad] +tempest.api.object_storage.test_container_services_negative.ContainerNegativeTest.test_delete_with_nonexistent_container_name[id-65387dbf-a0e2-4aac-9ddc-16eb3f1f69ba,negative] +tempest.api.image.v2.test_images_metadefs_schema.MetadataSchemaTest.test_get_metadata_namespaces_schema[id-ffe44891-678b-3ba0-a3e2-e0a3967b3aeb] +tempest.api.compute.servers.test_server_personality.ServerPersonalityTestJSON.test_create_server_with_personality[id-3cfe87fd-115b-4a02-b942-7dc36a337fdf,slow] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_remove_user_role_by_unauthorized_user[id-d0537987-0977-448f-a435-904c15de7298,negative] +tempest.api.identity.admin.v3.test_projects.ProjectsTestJSON.test_project_update_desc[id-f138b715-255e-4a7d-871d-351e1ef2e153] +tempest.api.compute.images.test_image_metadata_negative.ImagesMetadataNegativeTestJSON.test_set_nonexistent_image_metadata_item[id-2154fd03-ab54-457c-8874-e6e3eb56e9cf,negative] +tempest.api.compute.test_quotas.QuotasTestJSON.test_get_quotas[id-f1ef0a97-dbbb-4cca-adc5-c9fbc4f76107] +tempest.api.compute.servers.test_virtual_interfaces_negative.VirtualInterfacesNegativeTestJSON.test_list_virtual_interfaces_invalid_server_id[id-64ebd03c-1089-4306-93fa-60f5eb5c803c,negative,network] +tempest.api.volume.admin.test_volume_types.VolumeTypesTest.test_volume_type_encryption_create_get_update_delete[id-7830abd0-ff99-4793-a265-405684a54d46] +tempest.api.compute.admin.test_agents.AgentsAdminTestJSON.test_update_agent[id-dc9ffd51-1c50-4f0e-a820-ae6d2a568a9e] +tempest.api.identity.admin.v3.test_projects_negative.ProjectsNegativeStaticTestJSON.test_create_projects_name_length_over_64[id-502b6ceb-b0c8-4422-bf53-f08fdb21e2f0,negative] +tempest.api.volume.admin.test_volume_types_extra_specs_negative.ExtraSpecsNegativeTest.test_create_none_body[id-c821bdc8-43a4-4bf4-86c8-82f3858d5f7d,negative] +tempest.api.network.test_networks.NetworksTest.test_create_update_network_description[id-c72c1c0c-2193-4aca-ccc4-b1442640bbbb] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_limit[id-5cfa4ab2-4373-48dd-a41f-a532b12b08b2] +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_without_gateway[id-d2d596e2-8e76-47a9-ac51-d4648009f4d3] +tempest.api.identity.admin.v3.test_endpoints_negative.EndpointsNegativeTestJSON.test_create_with_enabled_True[id-9c43181e-0627-484a-8c79-923e8a59598b,negative] +tempest.api.network.test_routers.RoutersTest.test_add_remove_router_interface_with_subnet_id[id-b42e6e39-2e37-49cc-a6f4-8467e940900a,smoke] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_role_create_blank_name[id-c0b89e56-accc-4c73-85f8-9c0f866104c1,negative] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_update_volume_with_empty_volume_id[id-72aeca85-57a5-4c1f-9057-f320f9ea575b,negative] +tempest.api.compute.admin.test_hypervisor_negative.HypervisorAdminNegativeUnderV252Test.test_search_hypervisor_with_non_admin_user[id-5b6a6c79-5dc1-4fa5-9c58-9c8085948e74,negative] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_limit_and_marker[id-f7064ae8-dbcc-48da-b594-82feef6ea5af] +tempest.api.compute.servers.test_server_rescue_negative.ServerRescueNegativeTestJSON.test_rescued_vm_rebuild[id-70cdb8a1-89f8-437d-9448-8844fd82bf46,negative] +tempest.api.network.test_versions.NetworksApiDiscovery.test_show_api_v2_details[id-e64b7216-3178-4263-967c-d389290988bf,smoke] +tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_delete_server_metadata_item[id-127642d6-4c7b-4486-b7cd-07265a378658] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_suspend_resume_server[id-0d8ee21e-b749-462d-83da-b85b41c86c7f] +tempest.api.compute.keypairs.test_keypairs_v22.KeyPairsV22TestJSON.test_keypairsv22_create_list_show[id-8726fa85-7f98-4b20-af9e-f710a4f3391c] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_details_pagination[id-e9138a2c-f67b-4796-8efa-635c196d01de] +tempest.api.network.test_floating_ips.FloatingIPTestJSON.test_create_update_floatingip_with_port_multiple_ip_address[id-45c4c683-ea97-41ef-9c51-5e9802f2f3d7] +tempest.api.object_storage.test_container_staticweb.StaticWebTest.test_web_listing[id-941814cf-db9e-4b21-8112-2b6d0af10ee5] +tempest.api.image.v1.test_image_members.ImageMembersTest.test_get_shared_images[id-6a5328a5-80e8-4b82-bd32-6c061f128da9] +tempest.api.compute.admin.test_hosts.HostsAdminTestJSON.test_show_host_detail[id-38adbb12-aee2-4498-8aec-329c72423aa4] +tempest.api.image.v1.test_images_negative.CreateDeleteImagesNegativeTest.test_register_with_invalid_container_format[id-036ede36-6160-4463-8c01-c781eee6369d,negative] +tempest.api.compute.admin.test_flavors_microversions.FlavorsV255TestJSON.test_crud_flavor[id-61976b25-488d-41dc-9dcb-cb9693a7b075] +tempest.api.identity.admin.v3.test_oauth_consumers.OAUTHConsumersV3Test.test_list_consumers[id-09ca50de-78f2-4ffb-ac71-f2254036b2b8] +tempest.api.network.test_networks.NetworksTest.test_show_subnet_fields[id-270fff0b-8bfc-411f-a184-1e8fd35286f0] +tempest.api.volume.admin.test_volume_snapshot_quotas_negative.VolumeSnapshotQuotasNegativeTestJSON.test_quota_volume_gigabytes_snapshots[id-c99a1ca9-6cdf-498d-9fdf-25832babef27,negative] +tempest.api.identity.admin.v3.test_list_projects.ListProjectsStaticTestJSON.test_list_projects[id-1d830662-22ad-427c-8c3e-4ec854b0af44] +tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_version_resources[id-b9232f5e-d9e5-4d97-b96c-28d3db4de1bd,smoke] +tempest.api.compute.admin.test_delete_server.DeleteServersAdminTestJSON.test_delete_server_while_in_error_state[id-99774678-e072-49d1-9d2a-49a59bc56063] +tempest.api.compute.servers.test_attach_interfaces.AttachInterfacesTestJSON.test_create_list_show_delete_interfaces_by_fixed_ip[id-d290c06c-f5b3-11e7-8ec8-002293781009,network] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_image_null_id[id-ef45000d-0a72-4781-866d-4cb7bf2562ad,negative] +tempest.api.object_storage.test_object_services.PublicObjectTest.test_access_public_object_with_another_user_creds[id-54e2a2fe-42dc-491b-8270-8e4217dd4cdc] +tempest.api.compute.servers.test_create_server.ServersTestBootFromVolume.test_verify_created_server_vcpus[id-cbc0f52f-05aa-492b-bdc1-84b575ca294b] +tempest.api.identity.admin.v3.test_inherits.InheritsV3TestJSON.test_inherit_assign_list_revoke_user_roles_on_project_tree[id-9f02ccd9-9b57-46b4-8f77-dd5a736f3a06] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_create_security_group_rule_with_non_existent_security_group[id-be308db6-a7cf-4d5c-9baf-71bafd73f35e,negative] +tempest.api.volume.test_volumes_snapshots.VolumesSnapshotTestJSON.test_snapshot_create_delete_with_volume_in_use[compute,id-8567b54c-4455-446d-a1cf-651ddeaa3ff2] +tempest.scenario.test_volume_backup_restore.TestVolumeBackupRestore.test_volume_backup_restore[compute,id-2ce5e55c-4085-43c1-98c6-582525334ad7,image,slow,volume] +tempest.api.volume.admin.test_volume_types_negative.VolumeTypesNegativeTest.test_create_volume_type_encryption_nonexistent_type_id[id-a5924b5f-b6c1-49ba-994c-b4af55d26e52,negative] +tempest.api.compute.admin.test_servers.ServersAdminTestJSON.test_list_servers_detailed_filter_by_invalid_status[id-d56e9540-73ed-45e0-9b88-98fc419087eb] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_unshelve_non_existent_server[id-23d23b37-afaf-40d7-aa5d-5726f82d8821,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_resume_server_invalid_state[id-ccb6294d-c4c9-498f-8a43-554c098bfadb,negative] +tempest.scenario.test_unified_limits.ImageQuotaTest.test_image_stage_quota[id-fc76b8d9-aae5-46fb-9285-099e37f311f7,identity,image] +tempest.api.compute.admin.test_flavors_access.FlavorsAccessTestJSON.test_flavor_access_add_remove[id-59e622f6-bdf6-45e3-8ba8-fedad905a6b4] +tempest.api.identity.admin.v3.test_trusts.TrustsV3TestJSON.test_trust_expire[id-0ed14b66-cefd-4b5c-a964-65759453e292] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_volume_get_nonexistent_volume_id[id-f131c586-9448-44a4-a8b0-54ca838aa43e,negative] +tempest.api.network.admin.test_routers.RoutersAdminTest.test_update_router_set_gateway[id-6cc285d8-46bf-4f36-9b1a-783e3008ba79] +tempest.api.network.test_networks.NetworksIpV6Test.test_show_network[id-2bf13842-c93f-4a69-83ed-717d2ec3b44e,smoke] +tempest.api.identity.admin.v3.test_groups.GroupsV3TestJSON.test_list_user_groups[id-64573281-d26a-4a52-b899-503cb0f4e4ec] +tempest.api.identity.admin.v3.test_tokens.TokensV3TestJSON.test_get_available_project_scopes[id-08ed85ce-2ba8-4864-b442-bcc61f16ae89] +tempest.api.image.v2.admin.test_images.ImportCopyImagesTest.test_image_copy_image_import[id-9b3b644e-03d1-11eb-a036-fa163e2eaf49] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_unshelve_server_invalid_state[id-8f198ded-1cca-4228-9e65-c6b449c54880,negative] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_attach_volumes_with_nonexistent_volume_id[compute,id-f5e56b0a-5d02-43c1-a2a7-c9b792c2e3f6,negative] +tempest.api.compute.admin.test_services_negative.ServicesAdminNegativeV253TestJSON.test_get_service_by_invalid_service_and_valid_host[id-1e966d4a-226e-47c7-b601-0b18a27add54,negative] +tempest.api.compute.images.test_image_metadata.ImagesMetadataTestJSON.test_set_image_metadata[id-ece7befc-d3ce-42a4-b4be-c3067a418c29] +tempest.api.compute.servers.test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_list_server_metadata_non_existent_server[id-f408e78e-3066-4097-9299-3b0182da812e,negative] +tempest.api.volume.test_volumes_clone_negative.VolumesCloneNegativeTest.test_create_from_volume_decreasing_size[id-9adae371-a257-43a5-459a-dc7c88e66e0e,negative] +tempest.api.identity.admin.v3.test_list_projects.ListProjectsTestJSON.test_list_projects_with_enabled[id-0fe7a334-675a-4509-b00e-1c4b95d5dae8] +tempest.api.image.v1.test_images.UpdateImageMetaTest.test_list_image_metadata[id-01752c1c-0275-4de3-9e5b-876e44541928] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volumes_list_details_by_status[id-2943f712-71ec-482a-bf49-d5ca06216b9f] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volumes_list_by_bootable[id-2016a942-3020-40d7-95ce-7613bf8407ce] +tempest.api.compute.volumes.test_attach_volume.AttachVolumeMultiAttachTest.test_list_get_volume_attachments_multiattach[id-8d5853f7-56e7-4988-9b0c-48cea3c7049a] +tempest.api.compute.security_groups.test_security_group_rules.SecurityGroupRulesTestJSON.test_security_group_rules_list[id-a6154130-5a55-4850-8be4-5e9e796dbf17,smoke] +tempest.scenario.test_minbw_allocation_placement.MinBwAllocationPlacementTest.test_migrate_with_qos_min_bw_allocation[compute,id-8a98150c-a506-49a5-96c6-73a5e7b04ada,network] +tempest.api.compute.admin.test_servers.ServersAdminTestJSON.test_create_server_with_scheduling_hint[id-fdcd9b33-0903-4e00-a1f7-b5f6543068d6] +tempest.api.network.admin.test_external_network_extension.ExternalNetworksTestJSON.test_list_external_networks[id-39be4c9b-a57e-4ff9-b7c7-b218e209dfcc] +tempest.api.identity.admin.v2.test_roles.RolesTestJSON.test_get_role_by_id[id-db6870bd-a6ed-43be-a9b1-2f10a5c9994f] +tempest.api.identity.admin.v3.test_regions.RegionsTestJSON.test_create_region_with_specific_id[id-2c12c5b5-efcf-4aa5-90c5-bff1ab0cdbe2,smoke] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_server_metadata_exceeds_length_limit[id-7fc74810-0bd2-4cd7-8244-4f33a9db865a,negative] +tempest.api.network.admin.test_metering_extensions.MeteringTestJSON.test_create_delete_metering_label_rule_with_filters[id-f4d547cd-3aee-408f-bf36-454f8825e045] +tempest.api.network.admin.test_external_network_extension.ExternalNetworksTestJSON.test_update_external_network[id-4db5417a-e11c-474d-a361-af00ebef57c5] +tempest.api.network.test_ports.PortsTestJSON.test_update_port_with_security_group_and_extra_attributes[id-58091b66-4ff4-4cc1-a549-05d60c7acd1a] +tempest.api.compute.images.test_image_metadata_negative.ImagesMetadataNegativeTestJSON.test_get_nonexistent_image_metadata_item[id-41ae052c-6ee6-405c-985e-5712393a620d,negative] +tempest.api.compute.admin.test_simple_tenant_usage.TenantUsagesTestJSON.test_get_usage_tenant[id-94135049-a4c5-4934-ad39-08fa7da4f22e] +tempest.api.object_storage.test_object_services.ObjectTest.test_copy_object_to_itself[id-2248abba-415d-410b-9c30-22dff9cd6e67] +tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_preserve_preexisting_port[compute,id-759462e1-8535-46b0-ab3a-33aa45c55aaa,network,slow] +tempest.api.volume.admin.test_volumes_list.VolumesListAdminTestJSON.test_volume_list_param_tenant[id-5866286f-3290-4cfd-a414-088aa6cdc469] +tempest.api.identity.v3.test_catalog.IdentityCatalogTest.test_catalog_standardization[id-56b57ced-22b8-4127-9b8a-565dfb0207e2] +tempest.api.object_storage.test_container_quotas.ContainerQuotasTest.test_upload_valid_object[id-9a0fb034-86af-4df0-86fa-f8bd7db21ae0,smoke] +tempest.api.volume.admin.test_groups.GroupsTest.test_group_update[id-4a8a6fd2-8b3b-4641-8f54-6a6f99320006] +tempest.api.compute.keypairs.test_keypairs_negative.KeyPairsNegativeTestJSON.test_create_keypair_when_public_key_bits_exceeds_maximum[id-fc100c19-2926-4b9c-8fdc-d0589ee2f9ff,negative] +tempest.api.identity.admin.v3.test_projects.ProjectsTestJSON.test_associate_user_to_project[id-59398d4a-5dc5-4f86-9a4c-c26cc804d6c6] +tempest.api.network.admin.test_routers.RoutersAdminTest.test_create_router_with_snat_explicit[id-ea74068d-09e9-4fd7-8995-9b6a1ace920f] +tempest.api.object_storage.test_object_services.ObjectTest.test_get_object_with_metadata[id-005f9bf6-e06d-41ec-968e-96c78e0b1d82] +tempest.api.compute.admin.test_flavors_extra_specs_negative.FlavorsExtraSpecsNegativeTestJSON.test_flavor_non_admin_unset_keys[id-28f12249-27c7-44c1-8810-1f382f316b11,negative] +tempest.api.compute.servers.test_create_server_multi_nic.ServersTestMultiNic.test_verify_multiple_nics_order[id-0578d144-ed74-43f8-8e57-ab10dbf9b3c2] +tempest.api.identity.admin.v3.test_inherits.InheritsV3TestJSON.test_inherit_assign_check_revoke_roles_on_projects_group[id-26021436-d5a4-4256-943c-ded01e0d4b45] +tempest.api.volume.admin.test_volume_quotas_negative.VolumeQuotasNegativeTestJSON.test_quota_volumes[id-bf544854-d62a-47f2-a681-90f7a47d86b6,negative] +tempest.api.identity.admin.v3.test_projects_negative.ProjectsNegativeStaticTestJSON.test_list_projects_by_unauthorized_user[id-24c49279-45dd-4155-887a-cb738c2385aa,negative] +tempest.api.compute.admin.test_fixed_ips_negative.FixedIPsNegativeTestJson.test_list_fixed_ip_details_with_non_admin_user[id-9f17f47d-daad-4adc-986e-12370c93e407,negative] +tempest.api.identity.admin.v3.test_tokens.TokensV3TestJSON.test_rescope_token[id-565fa210-1da1-4563-999b-f7b5b67cf112] +tempest.api.identity.admin.v3.test_endpoints_negative.EndpointsNegativeTestJSON.test_update_with_enabled_True[id-faba3587-f066-4757-a48e-b4a3f01803bb,negative] +tempest.api.compute.security_groups.test_security_group_rules_negative.SecurityGroupRulesNegativeTestJSON.test_create_security_group_rule_with_invalid_port_range[id-00296fa9-0576-496a-ae15-fbab843189e0,negative] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_shutdown_nonexistent_host[id-9e637444-29cf-4244-88c8-831ae82c31b6,negative] +tempest.api.network.test_floating_ips_negative.FloatingIPNegativeTestJSON.test_create_floatingip_in_private_network[id-50b9aeb4-9f0b-48ee-aa31-fa955a48ff54,negative] +tempest.scenario.test_network_v6.TestGettingAddress.test_dualnet_dhcp6_stateless_from_os[compute,id-76f26acd-9688-42b4-bc3e-cd134c4cb09e,network,slow] +tempest.api.identity.admin.v3.test_domains.DomainsTestJSON.test_create_domain_without_description[id-2abf8764-309a-4fa9-bc58-201b799817ad] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcpv6_stateless_eui64[id-e5517e62-6f16-430d-a672-f80875493d4c] +tempest.api.network.admin.test_ports.PortsAdminExtendedAttrsTestJSON.test_create_port_binding_ext_attr[compute,id-8e8569c1-9ac7-44db-8bc1-f5fb2814f29b] +tempest.api.object_storage.test_object_services.ObjectTest.test_get_object[id-02610ba7-86b7-4272-9ed8-aa8d417cb3cd,smoke] +tempest.api.volume.test_volumes_get.VolumesGetTest.test_volume_create_get_update_delete[id-27fb0e9f-fb64-41dd-8bdb-1ffa762f0d51,smoke] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_list_volumes_detail_with_invalid_status[id-ba94b27b-be3f-496c-a00e-0283b373fa75,negative] +tempest.api.object_storage.test_object_slo.ObjectSloTest.test_delete_large_object[id-87b6dfa1-abe9-404d-8bf0-6c3751e6aa77] +tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON.test_list_flavors_using_marker[id-e800f879-9828-4bd0-8eae-4f17189951fb] +tempest.api.identity.v2.test_ec2_credentials.EC2CredentialsTest.test_delete_ec2_credential[id-6aba0d4c-b76b-4e46-aa42-add79bc1551d] +tempest.api.network.test_networks.NetworksTest.test_update_subnet_gw_dns_host_routes_dhcp[id-3d3852eb-3009-49ec-97ac-5ce83b73010a] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_register_with_invalid_disk_format[id-70c6040c-5a97-4111-9e13-e73665264ce1,negative] +tempest.api.compute.servers.test_server_group.ServerGroupTestJSON.test_create_server_with_scheduler_hint_group[id-ed20d3fb-9d1f-4329-b160-543fbd5d9811] +tempest.api.identity.admin.v2.test_tenant_negative.TenantsNegativeTestJSON.test_tenant_create_duplicate[id-af16f44b-a849-46cb-9f13-a751c388f739,negative] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_ethertype[id-5666968c-fff3-40d6-9efc-df1c8bd01abb,negative] +tempest.api.compute.volumes.test_attach_volume.AttachVolumeMultiAttachTest.test_resize_server_with_multiattached_volume[id-f01c7169-a124-4fc7-ae60-5e380e247c9c] +tempest.api.network.test_routers_negative.RoutersNegativeTest.test_router_remove_interface_in_use_returns_409[id-04df80f9-224d-47f5-837a-bf23e33d1c20,negative] +tempest.scenario.test_security_groups_basic_ops.TestSecurityGroupsBasicOps.test_cross_tenant_traffic[compute,id-e79f879e-debb-440c-a7e4-efeda05b6848,network] +tempest.api.volume.admin.test_volume_types_extra_specs_negative.ExtraSpecsNegativeTest.test_update_nonexistent_extra_spec_id[id-25e5a0ee-89b3-4c53-8310-236f76c75365,negative] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_role_create_update_show_list[id-18afc6c0-46cf-4911-824e-9989cc056c3a,smoke] +tempest.api.network.test_allowed_address_pair.AllowedAddressPairIpV6TestJSON.test_create_list_port_with_address_pair[id-86c3529b-1231-40de-803c-00e40882f043] +tempest.api.compute.admin.test_aggregates.AggregatesAdminTestJSON.test_aggregate_add_host_list[id-7f6a1cc5-2446-4cdb-9baa-b6ae0a919b72] +tempest.api.compute.admin.test_create_server.ServersWithSpecificFlavorTestJSON.test_verify_created_server_ephemeral_disk[id-b3c7bcfc-bb5b-4e22-b517-c7f686b802ca] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcp_stateful_fixedips_duplicate[id-57b8302b-cba9-4fbb-8835-9168df029051] +tempest.api.compute.images.test_images_oneserver_negative.ImagesOneServerNegativeTestJSON.test_delete_image_that_is_not_yet_active[id-0894954d-2db2-4195-a45b-ffec0bc0187e,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_flavor[id-18f5227f-d155-4429-807c-ccb103887537,negative] +tempest.api.compute.admin.test_agents.AgentsAdminTestJSON.test_create_agent[id-1fc6bdc8-0b6d-4cc7-9f30-9b04fabe5b90] +tempest.api.compute.admin.test_hypervisor_negative.HypervisorAdminNegativeUnderV252Test.test_show_servers_with_nonexistent_hypervisor[id-02463d69-0ace-4d33-a4a8-93d7883a2bba,negative] +tempest.api.compute.admin.test_flavors_access_negative.FlavorsAccessNegativeTestJSON.test_flavor_access_list_with_public_flavor[id-0621c53e-d45d-40e7-951d-43e5e257b272,negative] +tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_suspend_resume[compute,id-5cdf9499-541d-4923-804e-b9a60620a7f0,network,slow] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_create_bulk_port[id-67f1b811-f8db-43e2-86bd-72c074d4a42c] +tempest.api.volume.test_volumes_actions.VolumesActionsTest.test_attach_detach_volume_to_instance[compute,id-fff42874-7db5-4487-a8e1-ddda5fb5288d,smoke] +tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsAssociationTestJSON.test_associate_disassociate_floating_ip[id-307efa27-dc6f-48a0-8cd2-162ce3ef0b52] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volumes_list_details_by_bootable[id-2016a939-72ec-482a-bf49-d5ca06216b9f] +tempest.api.identity.v2.test_api_discovery.TestApiDiscovery.test_api_media_types[id-007a0be0-78fe-4fdb-bbee-e9216cc17bb2,smoke] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_create_duplicate_security_group_rule_fails[id-8fde898f-ce88-493b-adc9-4e4692879fc5,negative] +tempest.api.compute.servers.test_server_personality.ServerPersonalityTestJSON.test_personality_files_exceed_limit[id-176cd8c9-b9e8-48ee-a480-180beab292bf] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_remote_ip_and_group[id-b5c4b247-6b02-435b-b088-d10d45650881,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_nonexistent_security_group[id-c5fa6041-80cd-483b-aa6d-4e45f19d093c,negative] +tempest.api.network.test_routers.RoutersIpV6Test.test_add_remove_router_interface_with_port_id[id-2b7d2f37-6748-4d78-92e5-1d590234f0d5,smoke] +tempest.api.volume.admin.test_snapshots_actions.SnapshotsActionsTest.test_snapshot_force_delete_when_snapshot_is_deleting[id-92ce8597-b992-43a1-8868-6316b22a969e] +tempest.api.image.v2.test_images_metadefs_schema.MetadataSchemaTest.test_get_metadata_resource_type_schema[id-fde34891-678b-3b40-ae32-e0a3e67b6beb] +tempest.api.identity.admin.v2.test_tokens.TokensTestJSON.test_rescope_token[id-25ba82ee-8a32-4ceb-8f50-8b8c71e8765e] +tempest.api.volume.admin.test_qos.QosSpecsTestJSON.test_create_delete_qos_with_front_end_consumer[id-7e15f883-4bef-49a9-95eb-f94209a1ced1] +tempest.scenario.test_security_groups_basic_ops.TestSecurityGroupsBasicOps.test_port_update_new_security_group[compute,id-f4d556d7-1526-42ad-bafb-6bebf48568f6,network,slow] +tempest.api.image.v1.test_image_members.ImageMembersTest.test_remove_member[id-a76a3191-8948-4b44-a9d6-4053e5f2b138] +tempest.api.network.test_ports.PortsTestJSON.test_create_update_port_with_second_ip[id-63aeadd4-3b49-427f-a3b1-19ca81f06270] +tempest.api.compute.admin.test_hypervisor.HypervisorAdminTestJSON.test_get_hypervisor_show_details[id-94ff9eae-a183-428e-9cdb-79fde71211cc] +tempest.api.compute.admin.test_services_negative.ServicesAdminNegativeV253TestJSON.test_disable_service_with_invalid_service_id[id-a9eeeade-42b3-419f-87aa-c9342aa068cf,negative] +tempest.api.compute.admin.test_services_negative.ServicesAdminNegativeTestJSON.test_get_service_by_invalid_service_and_valid_host[id-1e966d4a-226e-47c7-b601-0b18a27add54,negative] +tempest.api.compute.servers.test_server_tags.ServerTagsTestJSON.test_create_delete_tag[id-8d95abe2-c658-4c42-9a44-c0258500306b] +tempest.api.compute.security_groups.test_security_groups.SecurityGroupsTestJSON.test_server_security_groups[id-fe4abc0d-83f5-4c50-ad11-57a1127297a2] +tempest.api.compute.servers.test_server_group.ServerGroupTestJSON.test_create_delete_server_group_with_affinity_policy[id-5dc57eda-35b7-4af7-9e5f-3c2be3d2d68b] +tempest.api.identity.admin.v2.test_users.UsersTestJSON.test_create_user_with_enabled[id-89d9fdb8-15c2-4304-a429-48715d0af33d] +tempest.api.identity.admin.v3.test_tokens.TokensV3TestJSON.test_get_available_domain_scopes[id-ec5ecb05-af64-4c04-ac86-4d9f6f12f185] +tempest.api.image.v2.test_images.ListUserImagesTest.test_get_images_schema[id-25c8d7b2-df21-460f-87ac-93130bcdc684] +tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_remote_group_id[id-c2ed2deb-7a0c-44d8-8b4c-a5825b5c310b] +tempest.scenario.test_network_v6.TestGettingAddress.test_dhcp6_stateless_from_os[compute,id-d7e1f858-187c-45a6-89c9-bdafde619a9f,network,slow] +tempest.api.compute.keypairs.test_keypairs.KeyPairsV2TestJSON.test_keypair_create_delete[id-6c1d3123-4519-4742-9194-622cb1714b7d] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_show_port_fields[id-45fcdaf2-dab0-4c13-ac6c-fcddfb579dbd] +tempest.api.object_storage.test_container_staticweb.StaticWebTest.test_web_index[id-c1f055ab-621d-4a6a-831f-846fcb578b8b] +tempest.api.compute.limits.test_absolute_limits_negative.AbsoluteLimitsNegativeTestJSON.test_max_metadata_exceed_limit[id-215cd465-d8ae-49c9-bf33-9c911913a5c8,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_server_set_empty_name[id-38204696-17c6-44da-9590-40f87fb5a899,negative] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_get_vnc_console[id-c6bc11bf-592e-4015-9319-1c98dc64daf5] +tempest.api.compute.admin.test_migrations.MigrationsAdminTest.test_list_migrations_in_flavor_resize_situation[id-1b512062-8093-438e-b47a-37d2f597cd64] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_create_user_with_duplicate_name[id-57ae8558-120c-4723-9308-3751474e7ecf,negative] +tempest.api.volume.admin.test_snapshots_actions.SnapshotsActionsTest.test_snapshot_force_delete_when_snapshot_is_error[id-645a4a67-a1eb-4e8e-a547-600abac1525d] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_list_user_roles_by_unauthorized_user[id-7391ab4c-06f3-477a-a64a-c8e55ce89837,negative] +tempest.api.compute.admin.test_servers_negative.ServersAdminNegativeTestJSON.test_migrate_non_existent_server[id-46a4e1ca-87ae-4d28-987a-1b6b136a0221,negative] +tempest.api.network.test_ports.PortsTestJSON.test_create_port_in_allowed_allocation_pools[id-0435f278-40ae-48cb-a404-b8a087bc09b1,smoke] +tempest.api.compute.admin.test_server_diagnostics.ServerDiagnosticsV248Test.test_get_server_diagnostics[id-64d0d48c-dff1-11e6-bf01-fe55135034f3] +tempest.api.compute.admin.test_volume.AttachSCSIVolumeTestJSON.test_attach_scsi_disk_with_config_drive[id-777e468f-17ca-4da4-b93d-b7dbf56c0494] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_list_as_user[id-b7d475a6-5dcd-4ff4-b70a-cd9de66a6672,negative] +tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_additional_args[id-87dfbcf9-1849-43ea-b1e4-efa3eeae9f71] +tempest.api.network.test_floating_ips.FloatingIPTestJSON.test_floating_ip_update_different_router[id-1bb2f731-fe5a-4b8c-8409-799ade1bed4d] +tempest.api.image.v2.test_images_member.ImagesMemberTest.test_remove_image_member[id-72989bc7-2268-48ed-af22-8821e835c914] +tempest.api.compute.servers.test_server_addresses_negative.ServerAddressesNegativeTestJSON.test_list_server_addresses_by_network_neg[id-a2ab5144-78c0-4942-a0ed-cc8edccfd9ba,negative,network] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_show_non_existent_security_group_rule[id-4c094c09-000b-4e41-8100-9617600c02a6,negative] +tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_dissociate_nonexistent_floating_ip[id-0a081a66-e568-4e6b-aa62-9587a876dca8,negative] +tempest.scenario.test_volume_migrate_attached.TestVolumeMigrateRetypeAttached.test_volume_migrate_attached[compute,id-fe47b1ed-640e-4e3b-a090-200e25607362,slow,volume] +tempest.api.identity.v2.test_tokens.TokensTest.test_create_token[id-65ae3b78-91ff-467b-a705-f6678863b8ec] +tempest.api.identity.admin.v3.test_projects_negative.ProjectsNegativeStaticTestJSON.test_create_project_with_empty_name[id-7828db17-95e5-475b-9432-9a51b4aa79a9,negative] +tempest.api.compute.keypairs.test_keypairs_v22.KeyPairsV22TestJSON.test_keypair_create_with_pub_key[id-39c90c6a-304a-49dd-95ec-2366129def05] +tempest.api.identity.v3.test_domains.DefaultDomainTestJSON.test_default_domain_exists[id-17a5de24-e6a0-4e4a-a9ee-d85b6e5612b5,smoke] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_reboot_deleted_server[id-581a397d-5eab-486f-9cf9-1014bbd4c984,negative] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_sort_created_at_desc[id-dcbbe24a-f3c0-4ec8-9274-55d48db8d1cf] +tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON.test_list_flavors_with_detail[id-6e85fde4-b3cd-4137-ab72-ed5f418e8c24] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_list_hosts_with_non_admin_user[id-dd032027-0210-4d9c-860e-69b1b8deed5f,negative] +tempest.api.object_storage.test_container_acl_negative.ObjectACLsNegativeTest.test_write_object_without_write_rights[id-8ba512ad-aa6e-444e-b882-2906a0ea2052,negative] +tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_register_upload_get_image_file[id-139b765e-7f3d-4b3d-8b37-3ca3876ee318,smoke] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_update_name_default[id-966e2b96-023a-11e7-a9e4-fa163e4fa634,negative] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_delete_invalid_volume_id[id-1f035827-7c32-4019-9240-b4ec2dbd9dfd,negative] +tempest.api.object_storage.test_container_services_negative.ContainerNegativeTest.test_create_container_metadata_exceeds_overall_metadata_count[id-ac666539-d566-4f02-8ceb-58e968dfb732,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_reboot_non_existent_server[id-d4c023a0-9c55-4747-9dd5-413b820143c7,negative] +tempest.api.volume.admin.test_backends_capabilities.BackendsCapabilitiesAdminTestsJSON.test_compare_volume_stats_values[id-a9035743-d46a-47c5-9cb7-3c80ea16dea0] +tempest.api.object_storage.test_object_expiry.ObjectExpiryTest.test_get_object_after_expiry_time[id-fb024a42-37f3-4ba5-9684-4f40a7910b41] +tempest.api.volume.test_volume_metadata.VolumesMetadataTest.test_crud_volume_metadata[id-6f5b125b-f664-44bf-910f-751591fe5769] +tempest.api.object_storage.test_account_services.AccountTest.test_update_account_metadata_with_create_metadata_key[id-64fd53f3-adbd-4639-af54-436e4982dbfb] +tempest.api.object_storage.test_account_bulk.BulkTest.test_bulk_delete[id-c075e682-0d2a-43b2-808d-4116200d736d] +tempest.api.volume.admin.test_multi_backend.VolumeMultiBackendTest.test_backend_name_distinction[id-46435ab1-a0af-4401-8373-f14e66b0dd58] +tempest.api.compute.admin.test_quotas_negative.QuotasSecurityGroupAdminNegativeTest.test_security_groups_exceed_limit[id-7c6c8f3b-2bf6-4918-b240-57b136a66aa0,negative,network] +tempest.api.volume.test_volumes_get.VolumesGetTest.test_volume_create_get_update_delete_as_clone[id-3f591b4a-7dc6-444c-bd51-77469506b3a1] +tempest.api.object_storage.test_object_temp_url.ObjectTempUrlTest.test_get_object_using_temp_url_key_2[id-671f9583-86bd-4128-a034-be282a68c5d8] +tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON.test_list_flavors_limit_results[id-8d7691b3-6ed4-411a-abc9-2839a765adab] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_additional_default_security_group_fails[id-2323061e-9fbf-4eb0-b547-7e8fafc90849,negative] +tempest.api.compute.security_groups.test_security_group_rules.SecurityGroupRulesTestJSON.test_security_group_rules_delete_when_peer_group_deleted[id-fc5c5acf-2091-43a6-a6ae-e42760e9ffaf] +tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_mtu_sized_frames[compute,id-b158ea55-472e-4086-8fa9-c64ac0c6c1d0,network,slow] +tempest.api.volume.test_volume_transfers.VolumesTransfersV355Test.test_create_get_list_accept_volume_transfer[id-9f36bb2b-619f-4507-b246-76aeb9a28851] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_format_xml[id-4477b609-1ca6-4d4b-b25d-ad3f01086089] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_no_params[id-1e341d7a-90a9-494c-b143-2cdf2aeb6aee] +tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON.test_list_images_filter_by_type[id-e3356918-4d3e-4756-81d5-abc4524ba29f] +tempest.api.compute.admin.test_flavors_access_negative.FlavorsAccessNegativeTestJSON.test_flavor_non_admin_add[id-41eaaade-6d37-4f28-9c74-f21b46ca67bd,negative] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_remove_user_role_non_existent_tenant[id-67a679ec-03dd-4551-bbfc-d1c93284f023,negative] +tempest.api.identity.admin.v3.test_domains.DomainsTestJSON.test_list_domains_filter_by_enabled[id-3fd19840-65c1-43f8-b48c-51bdd066dff9] +tempest.api.compute.servers.test_multiple_create_negative.MultipleCreateNegativeTestJSON.test_max_count_less_than_min_count[id-476da616-f1ef-4271-a9b1-b9fc87727cdf,negative] +tempest.api.compute.volumes.test_volumes_list.VolumesTestJSON.test_volume_list_with_details[id-bad0567a-5a4f-420b-851e-780b55bb867c] +tempest.api.compute.keypairs.test_keypairs_negative.KeyPairsNegativeTestJSON.test_create_keypair_invalid_name[id-45fbe5e0-acb5-49aa-837a-ff8d0719db91,negative] +tempest.api.object_storage.test_container_services.ContainerTest.test_update_container_metadata_with_delete_metadata_key[id-a2e36378-6f1f-43f4-840a-ffd9cfd61914] +tempest.api.compute.admin.test_live_migration.LiveAutoBlockMigrationV225Test.test_volume_backed_live_migration[id-5071cf17-3004-4257-ae61-73a84e28badd,volume] +tempest.api.compute.servers.test_create_server.ServersTestFqdnHostnames.test_create_server_with_fqdn_name[id-622066d2-39fc-4c09-9eeb-35903c114a0a] +tempest.api.identity.admin.v3.test_projects.ProjectsTestJSON.test_project_update_enable[id-b6b25683-c97f-474d-a595-55d410b68100] +tempest.api.identity.admin.v3.test_projects.ProjectsTestJSON.test_project_create_with_description[id-0ecf465c-0dc4-4532-ab53-91ffeb74d12d] +tempest.api.compute.admin.test_fixed_ips_negative.FixedIPsNegativeTestJson.test_fixed_ip_with_invalid_action[id-fd26ef50-f135-4232-9d32-281aab3f9176,negative] +tempest.api.compute.images.test_images.ImagesTestJSON.test_create_image_from_stopped_server[id-aaacd1d0-55a2-4ce8-818a-b5439df8adc9] +tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_server_with_admin_password[id-b92d5ec7-b1dd-44a2-87e4-45e888c46ef0] +tempest.api.object_storage.test_container_staticweb.StaticWebTest.test_web_error[id-f18b4bef-212e-45e7-b3ca-59af3a465f82] +tempest.api.network.test_ports.PortsTestJSON.test_port_list_filter_by_router_id[id-5ad01ed0-0e6e-4c5d-8194-232801b15c72] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filtered_by_name_regex[id-24a89b0c-0d55-4a28-847f-45075f19b27b] +tempest.api.compute.admin.test_security_groups.SecurityGroupsTestAdminJSON.test_list_security_groups_list_all_tenants_filter[id-49667619-5af9-4c63-ab5d-2cfdd1c8f7f1,network] +tempest.api.identity.admin.v3.test_regions.RegionsTestJSON.test_list_regions[id-d180bf99-544a-445c-ad0d-0c0d27663796] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_disk_format[id-4a4735a7-f22f-49b6-b0d9-66e1ef7453eb] +tempest.api.compute.admin.test_servers_on_multinodes.ServersOnMultiNodesTest.test_create_servers_on_different_hosts_with_list_of_servers[id-7869cc84-d661-4e14-9f00-c18cdc89cf57] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_assignments_for_domain_roles[id-3859df7e-5b78-4e4d-b10e-214c8953842a] +tempest.api.compute.certificates.test_certificates.CertificatesV2TestJSON.test_create_root_certificate[id-c070a441-b08e-447e-a733-905909535b1b] +tempest.api.network.test_networks.NetworksIpV6Test.test_delete_network_with_subnet[id-f04f61a9-b7f3-4194-90b2-9bcf660d1bfe] +tempest.api.compute.admin.test_hypervisor.HypervisorAdminTestJSON.test_get_hypervisor_stats[id-797e4f28-b6e0-454d-a548-80cc77c00816] +tempest.api.volume.admin.test_group_type_specs.GroupTypeSpecsTest.test_group_type_specs_create_show_update_list_delete[id-bb4e30d0-de6e-4f4d-866c-dcc48d023b4e] +tempest.api.object_storage.test_container_services.ContainerTest.test_create_container_with_remove_metadata_key[id-24d16451-1c0c-4e4f-b59c-9840a3aba40e] +tempest.api.network.test_networks.NetworksTest.test_show_network_fields[id-867819bb-c4b6-45f7-acf9-90edcf70aa5e] +tempest.api.network.test_networks.NetworksIpV6Test.test_create_delete_subnet_with_gw_and_allocation_pools[id-8217a149-0c6c-4cfb-93db-0486f707d13f] +tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_hotplug_nic[compute,id-c5adff73-e961-41f1-b4a9-343614f18cfa,network] +tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_cold_migration[compute,id-a4858f6c-401e-4155-9a49-d5cd053d1a2f,network,slow] +tempest.api.compute.images.test_images.ImagesTestJSON.test_create_image_from_paused_server[id-71bcb732-0261-11e7-9086-fa163e4fa634] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_authentication_when_tenant_is_disabled[id-440a7a8d-9328-4b7b-83e0-d717010495e4,negative] +tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_list_server_metadata[id-479da087-92b3-4dcf-aeb3-fd293b2d14ce] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents_with_no_object[id-4646ac2d-9bfb-4c7d-a3c5-0f527402b3df] +tempest.api.network.test_networks.NetworksTest.test_create_update_delete_network_subnet[id-0e269138-0da6-4efc-a46d-578161e7b221,smoke] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_pause_unpause_server[id-bd61a9fd-062f-4670-972b-2d6c3e3b9e73] +tempest.api.compute.admin.test_hypervisor_negative.HypervisorAdminNegativeTestJSON.test_get_hypervisor_uptime_with_non_admin_user[id-6c3461f9-c04c-4e2a-bebb-71dc9cb47df2,negative] +tempest.api.object_storage.test_object_services.ObjectTest.test_object_upload_in_segments[id-e3e6a64a-9f50-4955-b987-6ce6767c97fb] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_zero_limit[id-b1495414-2d93-414c-8019-849afe8d319e] +tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_subnet_details[compute,id-d8bb918e-e2df-48b2-97cd-b73c95450980,network,slow] +tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_cold_migration_revert[compute,id-25b188d7-0183-4b1e-a11d-15840c8e2fd6,network,slow] +tempest.api.compute.servers.test_server_addresses.ServerAddressesTestJSON.test_list_server_addresses[id-6eb718c0-02d9-4d5e-acd1-4e0c269cef39,network,smoke] +tempest.api.volume.admin.test_volume_services.VolumesServicesTestJSON.test_get_service_by_service_binary_name[id-63a3e1ca-37ee-4983-826d-83276a370d25] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_details_by_name[id-2de3a6d4-12aa-403b-a8f2-fdeb42a89623] +tempest.scenario.test_server_multinode.TestServerMultinode.test_schedule_to_all_nodes[compute,id-9cecbe35-b9d4-48da-a37e-7ce70aa43d30,network,smoke] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volumes_list_details_by_availability_zone[id-e1b80d13-94f0-4ba2-a40e-386af29f8db1] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_pause_non_existent_server[id-6a8dc0c6-6cd4-4c0a-9f32-413881828091,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_personality_file_contents_not_encoded[id-b8a7235e-5246-4a8f-a08e-b34877c6586f,negative] +tempest.api.compute.test_tenant_networks.ComputeTenantNetworksTest.test_list_show_tenant_networks[id-edfea98e-bbe3-4c7a-9739-87b986baff26,network] +tempest.api.identity.v2.test_ec2_credentials.EC2CredentialsTest.test_list_ec2_credentials[id-9e2ea42f-0a4f-468c-a768-51859ce492e0] +tempest.api.compute.test_extensions.ExtensionsTest.test_get_extension[id-05762f39-bdfa-4cdb-9b46-b78f8e78e2fd] +tempest.api.identity.admin.v3.test_inherits.InheritsV3TestJSON.test_inherit_assign_check_revoke_roles_on_projects_user[id-18b70e45-7687-4b72-8277-b8f1a47d7591] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_create_security_group_update_name_default[id-966e2b96-023a-11e7-a9e4-fa163e4fa634,negative] +tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_router_rescheduling[compute,id-2e788c46-fb3f-4ac9-8f82-0561555bea73,network,slow] +tempest.api.object_storage.test_container_acl_negative.ObjectACLsNegativeTest.test_write_object_without_using_creds[id-af587587-0c24-4e15-9822-8352ce711013,negative] +tempest.api.network.test_networks.NetworksIpV6Test.test_create_delete_subnet_with_gw[id-e41a4888-65a6-418c-a095-f7c2ef4ad59a] +tempest.api.identity.admin.v2.test_users.UsersTestJSON.test_create_user[id-2d55a71e-da1d-4b43-9c03-d269fd93d905,smoke] +tempest.api.network.admin.test_routers.RoutersIpV6AdminTest.test_update_router_reset_gateway_without_snat[id-f2faf994-97f4-410b-a831-9bc977b64374] +tempest.api.volume.admin.test_volume_types_extra_specs.VolumeTypesExtraSpecsTest.test_volume_type_extra_specs_list[id-b42923e9-0452-4945-be5b-d362ae533e60] +tempest.api.identity.admin.v2.test_users.UsersTestJSON.test_user_authentication[id-aca696c3-d645-4f45-b728-63646045beb1] +tempest.api.object_storage.test_object_services.ObjectTest.test_create_object[id-5b4ce26f-3545-46c9-a2ba-5754358a4c62,smoke] +tempest.api.image.v2.test_images_member.ImagesMemberTest.test_get_image_members_schema[id-6ae916ef-1052-4e11-8d36-b3ae14853cbb] +tempest.scenario.test_network_v6.TestGettingAddress.test_multi_prefix_dhcpv6_stateless[compute,id-7ab23f41-833b-4a16-a7c9-5b42fe6d4123,network,slow] +tempest.api.compute.volumes.test_attach_volume.AttachVolumeShelveTestJSON.test_detach_volume_shelved_or_offload_server[id-b54e86dd-a070-49c4-9c07-59ae6dae15aa,slow] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_update_host_without_param[id-0cd85f75-6992-4a4a-b1bd-d11e37fd0eee,negative] +tempest.api.identity.admin.v2.test_services.ServicesTestJSON.test_list_services[id-34ea6489-012d-4a86-9038-1287cadd5eca,smoke] +tempest.api.network.test_networks.BulkNetworkOpsTest.test_bulk_create_delete_network[id-d4f9024d-1e28-4fc1-a6b1-25dbc6fa11e2,smoke] +tempest.api.network.test_networks.NetworksIpV6Test.test_update_subnet_gw_dns_host_routes_dhcp[id-3d3852eb-3009-49ec-97ac-5ce83b73010a] +tempest.api.identity.v3.test_application_credentials.ApplicationCredentialsV3Test.test_query_application_credentials[id-9bb5e5cc-5250-493a-8869-8b665f6aa5f6] +tempest.api.object_storage.test_container_acl.ObjectTestACLs.test_read_object_with_rights[id-a3270f3f-7640-4944-8448-c7ea783ea5b6] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_update_user_request_without_a_token[id-3cc2a64b-83aa-4b02-88f0-d6ab737c4466,negative] +tempest.api.compute.images.test_images_negative.ImagesDeleteNegativeTestJSON.test_delete_image_with_invalid_image_id[id-381acb65-785a-4942-94ce-d8f8c84f1f0f,negative] +tempest.api.volume.admin.test_volume_services.VolumesServicesTestJSON.test_get_service_by_host_name[id-178710e4-7596-4e08-9333-745cb8bc4f8d] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_resize_server_revert_with_volume_attached[id-fbbf075f-a812-4022-bc5c-ccb8047eef12,volume] +tempest.api.volume.test_volumes_snapshots_negative.VolumesSnapshotNegativeTestJSON.test_create_snapshot_without_passing_volume_id[id-bb9da53e-d335-4309-9c15-7e76fd5e4d6d,negative] +tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON.test_list_images_with_detail_filter_by_type[id-888c0cc0-7223-43c5-9db0-b125fd0a393b] +tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_server_details[id-5de47127-9977-400a-936f-abcfbec1218f,smoke] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_limit[id-e914a891-3cc8-4b40-ad32-e0a39ffbddbb] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_limit_and_end_marker[id-888a3f0e-7214-4806-8e50-5e0c9a69bb5e] +tempest.api.identity.v2.test_ec2_credentials.EC2CredentialsTest.test_show_ec2_credential[id-cb284075-b613-440d-83ca-fe0b33b3c2b8] +tempest.scenario.test_dashboard_basic_ops.TestDashboardBasicOps.test_basic_scenario[dashboard,id-4f8851b1-0e69-482b-b63b-84c6e76f6c80,smoke] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_server_name_length_exceeds_256[id-c3e0fb12-07fc-4d76-a22e-37409887afe8,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_unpause_non_existent_server[id-705b8e3a-e8a7-477c-a19b-6868fc24ac75,negative] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_volume_extend_with_nonexistent_volume_id[id-8f05a943-013c-4063-ac71-7baf561e82eb,negative] +tempest.api.compute.admin.test_hypervisor_negative.HypervisorAdminNegativeUnderV252Test.test_search_nonexistent_hypervisor[id-19a45cc1-1000-4055-b6d2-28e8b2ec4faa,negative] +tempest.api.volume.admin.test_volume_retype.VolumeRetypeWithoutMigrationTest.test_available_volume_retype[id-b90412ee-465d-46e9-b249-ec84a47d5f25] +tempest.scenario.test_server_advanced_ops.TestServerAdvancedOps.test_server_sequence_suspend_resume[compute,id-949da7d5-72c8-4808-8802-e3d70df98e2c,slow] +tempest.api.identity.v3.test_application_credentials.ApplicationCredentialsV3Test.test_create_application_credential[id-8080c75c-eddc-4786-941a-c2da7039ae61] +tempest.api.compute.admin.test_flavors.FlavorsAdminTestJSON.test_create_flavor_verify_entry_in_list_details[id-8261d7b0-be58-43ec-a2e5-300573c3f6c5] +tempest.api.compute.volumes.test_attach_volume.AttachVolumeShelveTestJSON.test_attach_volume_shelved_or_offload_server[id-13a940b6-3474-4c3c-b03f-29b89112bfee,slow] +tempest.api.compute.keypairs.test_keypairs_v22.KeyPairsV22TestJSON.test_get_keypair_detail[id-a4233d5d-52d8-47cc-9a25-e1864527e3df] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_force_delete_nonexistent_server_id[id-6f47992b-5144-4250-9f8b-f00aa33950f3,negative] +tempest.api.network.test_networks.BulkNetworkOpsTest.test_bulk_create_delete_port[id-48037ff2-e889-4c3b-b86a-8e3f34d2d060,smoke] +tempest.api.volume.admin.test_snapshots_actions.SnapshotsActionsTest.test_update_snapshot_status[id-41288afd-d463-485e-8f6e-4eea159413eb] +tempest.api.volume.admin.test_qos.QosSpecsTestJSON.test_create_delete_qos_with_both_consumer[id-f88d65eb-ea0d-487d-af8d-71f4011575a4] +tempest.api.compute.admin.test_instance_usage_audit_log_negative.InstanceUsageAuditLogNegativeTestJSON.test_instance_usage_audit_logs_with_nonadmin_user[id-a9d33178-d2c9-4131-ad3b-f4ca8d0308a2,negative] +tempest.api.image.v2.test_images.ImportImagesTest.test_image_glance_direct_import[id-32ca0c20-e16f-44ac-8590-07869c9b4cc2] +tempest.api.volume.test_volumes_extend.VolumesExtendTest.test_volume_extend[id-9a36df71-a257-43a5-9555-dc7c88e66e0e] +tempest.api.compute.servers.test_virtual_interfaces.VirtualInterfacesTestJSON.test_list_virtual_interfaces[id-96c4e2ef-5e4d-4d7f-87f5-fed6dca18016,network] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_delete_as_user[id-cd6de795-c15d-45f1-8d9e-813c6bb72a3d,negative] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_role_create_duplicate[id-c0cde2c8-81c1-4bb0-8fe2-cf615a3547a8,negative] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_create_security_group_rule_with_bad_remote_ip_prefix[id-5f8daf69-3c5f-4aaa-88c9-db1d66f68679,negative] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_list_users_with_invalid_tenant[id-f5d39046-fc5f-425c-b29e-bac2632da28e,negative] +tempest.api.compute.servers.test_server_personality.ServerPersonalityTestJSON.test_can_create_server_with_max_number_personality_files[id-52f12ee8-5180-40cc-b417-31572ea3d555,slow] +tempest.api.network.test_allowed_address_pair.AllowedAddressPairTestJSON.test_update_port_with_multiple_ip_mac_address_pair[id-b3f20091-6cd5-472b-8487-3516137df933] +tempest.api.compute.security_groups.test_security_group_rules_negative.SecurityGroupRulesNegativeTestJSON.test_create_security_group_rule_duplicate[id-8bd56d02-3ffa-4d67-9933-b6b9a01d6089,negative] +tempest.api.network.test_networks.NetworksIpV6Test.test_create_delete_subnet_with_default_gw[id-ebb4fd95-524f-46af-83c1-0305b239338f] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_with_param_metadata[id-b5ebea1b-0603-40a0-bb41-15fcd0a53214] +tempest.api.compute.servers.test_attach_interfaces.AttachInterfacesUnderV243Test.test_add_remove_fixed_ip[id-c7e0e60b-ee45-43d0-abeb-8596fd42a2f9,network,smoke] +tempest.api.network.test_floating_ips.FloatingIPTestJSON.test_create_floating_ip_specifying_a_fixed_ip_address[id-36de4bd0-f09c-43e3-a8e1-1decc1ffd3a5,smoke] +tempest.api.compute.images.test_image_metadata_negative.ImagesMetadataNegativeTestJSON.test_list_nonexistent_image_metadata[id-94069db2-792f-4fa8-8bd3-2271a6e0c095,negative] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_changes_since_invalid_date[id-87d12517-e20a-4c9c-97b6-dd1628d6d6c9,negative] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_update_host_with_invalid_maintenance_mode[id-ab1e230e-5e22-41a9-8699-82b9947915d4,negative] +tempest.api.compute.admin.test_flavors_extra_specs.FlavorsExtraSpecsTestJSON.test_flavor_non_admin_get_all_keys[id-a99dad88-ae1c-4fba-aeb4-32f898218bd0] +tempest.api.network.test_ports.PortsTestJSON.test_create_port_with_no_securitygroups[id-4179dcb9-1382-4ced-84fe-1b91c54f5735,smoke] +tempest.api.network.admin.test_metering_extensions.MeteringIpV6TestJSON.test_list_metering_labels[id-e2fb2f8c-45bf-429a-9f17-171c70444612] +tempest.api.compute.images.test_images_negative.ImagesNegativeTestJSON.test_create_image_from_invalid_server[id-82c5b0c4-9dbd-463c-872b-20c4755aae7f,negative] +tempest.api.volume.test_snapshot_metadata.SnapshotMetadataTestJSON.test_update_show_snapshot_metadata_item[id-e8ff85c5-8f97-477f-806a-3ac364a949ed] +tempest.api.network.test_networks.NetworksIpV6Test.test_create_delete_subnet_with_host_routes_and_dns_nameservers[id-d830de0a-be47-468f-8f02-1fd996118289] +tempest.api.identity.admin.v3.test_domains_negative.DomainsNegativeTestJSON.test_delete_active_domain[gate,id-1f3fbff5-4e44-400d-9ca1-d953f05f609b,negative] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_show_port[id-c9a685bd-e83f-499c-939f-9f7863ca259f,smoke] +tempest.api.image.v2.admin.test_images_metadefs_namespace_objects.MetadataNamespaceObjectsTest.test_create_update_delete_meta_namespace_objects[id-b1a3775e-3b5c-4f6a-a3b4-1ba3574ae718] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_invalid_access_ip_v4_address[id-7f70a4d1-608f-4794-9e56-cb182765972c,negative] +tempest.scenario.test_server_basic_ops.TestServerBasicOps.test_server_basic_ops[compute,id-7fff3fb3-91d8-4fd0-bd7d-0204f1f180ba,network,smoke] +tempest.api.compute.security_groups.test_security_groups.SecurityGroupsTestJSON.test_list_security_groups_by_server[id-79517d60-535a-438f-af3d-e6feab1cbea7] +tempest.api.compute.admin.test_quotas_negative.QuotasAdminNegativeTest.test_create_server_when_cpu_quota_is_full[id-91058876-9947-4807-9f22-f6eb17140d9b,negative] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents_with_path[id-58ca6cc9-6af0-408d-aaec-2a6a7b2f0df9] +tempest.api.volume.admin.test_volume_types_negative.VolumeTypesNegativeTest.test_create_with_empty_name[id-878b4e57-faa2-4659-b0d1-ce740a06ae81,negative] +tempest.api.compute.admin.test_availability_zone_negative.AZAdminNegativeTestJSON.test_get_availability_zone_list_detail_with_non_admin_user[id-bf34dca2-fdc3-4073-9c02-7648d9eae0d7,negative] +tempest.api.compute.admin.test_flavors_extra_specs_negative.FlavorsExtraSpecsNegativeTestJSON.test_flavor_non_admin_update_specific_key[id-1ebf4ef8-759e-48fe-a801-d451d80476fb,negative] +tempest.api.volume.test_volumes_snapshots_negative.VolumesSnapshotNegativeTestJSON.test_volume_from_snapshot_decreasing_size[id-677863d1-34f9-456d-b6ac-9924f667a7f4,negative] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_create_role_by_unauthorized_user[id-585c8998-a8a4-4641-a5dd-abef7a8ced00,negative] +tempest.api.identity.admin.v3.test_list_users.UsersV3TestJSON.test_get_user[id-b4baa3ae-ac00-4b4e-9e27-80deaad7771f] +tempest.api.compute.images.test_list_images.ListImagesTestJSON.test_list_images_with_detail[id-9f94cb6b-7f10-48c5-b911-a0b84d7d4cd6] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_startup_host_with_non_admin_user[id-9f4ebb7e-b2ae-4e5b-a38f-0fd1bb0ddfca,negative] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_delete_role_by_unauthorized_user[id-15347635-b5b1-4a87-a280-deb2bd6d865e,negative] +tempest.api.compute.admin.test_hypervisor.HypervisorAdminTestJSON.test_get_hypervisor_uptime[id-91a50d7d-1c2b-4f24-b55a-a1fe20efca70] +tempest.api.compute.servers.test_attach_interfaces.AttachInterfacesTestJSON.test_reassign_port_between_servers[id-2f3a0127-95c7-4977-92d2-bc5aec602fb4] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_delete_non_existent_security_group[id-1f1bb89d-5664-4956-9fcd-83ee0fa603df,negative] +tempest.api.identity.admin.v2.test_tenants.TenantsTestJSON.test_tenant_create_not_enabled[id-3be22093-b30f-499d-b772-38340e5e16fb] +tempest.api.network.test_floating_ips_negative.FloatingIPNegativeTestJSON.test_create_floatingip_with_port_ext_net_unreachable[id-22996ea8-4a81-4b27-b6e1-fa5df92fa5e8,negative] +tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_reboot[compute,id-7b6860c2-afa3-4846-9522-adeb38dfbe08,network,slow] +tempest.api.object_storage.test_account_services.AccountTest.test_update_account_metadata_with_delete_metadata_key[id-d4d884d3-4696-4b85-bc98-4f57c4dd2bf1] +tempest.scenario.test_encrypted_cinder_volumes.TestEncryptedCinderVolumes.test_encrypted_cinder_volumes_luks[compute,id-79165fb4-5534-4b9d-8429-97ccffb8f86e,image,slow,volume] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_update_user_by_unauthorized_user[id-424868d5-18a7-43e1-8903-a64f95ee3aac,negative] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_param_display_name_and_status[id-777c87c1-2fc4-4883-8b8e-5c0b951d1ec8] +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_allocation_pools[id-bec949c4-3147-4ba6-af5f-cd2306118404] +tempest.api.volume.test_volume_delete_cascade.VolumesDeleteCascade.test_volume_delete_cascade[id-994e2d40-de37-46e8-b328-a58fba7e4a95] +tempest.api.network.test_extra_dhcp_options.ExtraDHCPOptionsTestJSON.test_create_list_port_with_extra_dhcp_options[id-d2c17063-3767-4a24-be4f-a23dbfa133c9] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_shutoff_status[id-451dbbb2-f330-4a9f-b0e1-5f5d2cb0f34c] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcpv6_stateless_no_ra[id-ae2f4a5d-03ff-4c42-a3b0-ce2fcb7ea832] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_update_host_with_non_admin_user[id-e40c72b1-0239-4ed6-ba21-81a184df1f7c,negative] +tempest.api.compute.limits.test_absolute_limits.AbsoluteLimitsTestJSON.test_absLimits_get[id-b54c66af-6ab6-4cf0-a9e5-a0cb58d75e0b] +tempest.api.compute.admin.test_servers_negative.ServersAdminNegativeTestJSON.test_reset_state_server_invalid_state[id-b0b4d8af-1256-41ef-9ee7-25f1c19dde80,negative] +tempest.api.volume.admin.test_volumes_actions.VolumesActionsTest.test_volume_force_delete_when_volume_is_error[id-3e33a8a8-afd4-4d64-a86b-c27a185c5a4a] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_delete_non_existent_server[id-1041b4e6-514b-4855-96a5-e974b60870a3,negative] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_sort_created_at_asc[id-4052c3a0-2415-440a-a8cc-305a875331b0] +tempest.api.object_storage.test_object_services.ObjectTest.test_get_object_with_if_unmodified_since[id-0aa1201c-10aa-467a-bee7-63cbdd463152] +tempest.api.compute.floating_ips.test_list_floating_ips_negative.FloatingIPDetailsNegativeTestJSON.test_get_nonexistent_floating_ip_details[id-7ab18834-4a4b-4f28-a2c5-440579866695,negative] +tempest.api.volume.admin.test_volume_services_negative.VolumeServicesNegativeTest.test_enable_service_with_invalid_host[id-3246ce65-ba70-4159-aa3b-082c28e4b484,negative] +tempest.api.identity.v3.test_application_credentials.ApplicationCredentialsV3Test.test_create_application_credential_expires[id-852daf0c-42b5-4239-8466-d193d0543ed3] +tempest.api.network.test_security_groups.SecGroupIPv6Test.test_list_security_groups[id-e30abd17-fef9-4739-8617-dc26da88e686,smoke] +tempest.api.identity.admin.v2.test_tenant_negative.TenantsNegativeTestJSON.test_create_tenant_with_empty_name[id-5a2e4ca9-b0c0-486c-9c48-64a94fba2395,negative] +tempest.api.object_storage.test_object_temp_url.ObjectTempUrlTest.test_get_object_using_temp_url[id-f91c96d4-1230-4bba-8eb9-84476d18d991] +tempest.api.object_storage.test_container_services.ContainerTest.test_update_container_metadata_with_create_metadata_key[id-31f40a5f-6a52-4314-8794-cd89baed3040] +tempest.scenario.test_minbw_allocation_placement.MinBwAllocationPlacementTest.test_qos_min_bw_allocation_basic[compute,id-78625d92-212c-400e-8695-dd51706858b8,network] +tempest.api.identity.admin.v3.test_list_users.UsersV3TestJSON.test_list_user_domains[id-08f9aabb-dcfe-41d0-8172-82b5fa0bd73d] +tempest.api.network.test_ports.PortsTestJSON.test_create_bulk_port[id-67f1b811-f8db-43e2-86bd-72c074d4a42c] +tempest.api.identity.admin.v3.test_domains.DomainsTestJSON.test_list_domains_filter_by_name[id-c6aee07b-4981-440c-bb0b-eb598f58ffe9] +tempest.api.network.admin.test_routers.RoutersIpV6AdminTest.test_update_router_set_gateway[id-6cc285d8-46bf-4f36-9b1a-783e3008ba79] +tempest.scenario.test_network_v6.TestGettingAddress.test_dualnet_slaac_from_os[compute,id-b6399d76-4438-4658-bcf5-0d6c8584fde2,network,slow] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_rebuild_non_existent_server[id-d86141a7-906e-4731-b187-d64a2ea61422,negative] +tempest.api.image.v2.test_images_metadefs_schema.MetadataSchemaTest.test_get_metadata_namespace_schema[id-e9e44891-3cb8-3b40-a532-e0a39fea3dab] +tempest.api.compute.servers.test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_set_metadata_non_existent_server[id-0df38c2a-3d4e-4db5-98d8-d4d9fa843a12,negative] +tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON.test_list_flavors_detailed_limit_results[id-b26f6327-2886-467a-82be-cef7a27709cb] +tempest.api.object_storage.test_container_quotas.ContainerQuotasTest.test_upload_too_many_objects[id-3a387039-697a-44fc-a9c0-935de31f426b,smoke] +tempest.api.object_storage.test_container_services.ContainerTest.test_update_container_metadata_with_delete_metadata[id-3a5ce7d4-6e4b-47d0-9d87-7cd42c325094] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_limit_and_marker_and_end_marker[id-8cf98d9c-e3a0-4e44-971b-c87656fdddbd] +tempest.api.network.test_networks.NetworksIpV6TestAttrs.test_create_delete_stateless_subnet_with_ports[id-2de6ab5a-fcf0-4144-9813-f91a940291f1] +tempest.scenario.test_volume_boot_pattern.TestVolumeBootPattern.test_volume_boot_pattern[compute,id-557cd2c2-4eb8-4dce-98be-f86765ff311b,image,slow,volume] +tempest.api.network.test_security_groups.SecGroupTest.test_create_list_update_show_delete_security_group[id-bfd128e5-3c92-44b6-9d66-7fe29d22c802,smoke] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_create_with_duplicate_name[id-9fdb4abc-6b66-4b27-b89c-eb215a956168,negative] +tempest.api.compute.volumes.test_attach_volume.AttachVolumeMultiAttachTest.test_boot_from_multiattach_volume[id-65e33aa2-185b-44c8-b22e-e524973ed625] +tempest.api.volume.admin.test_volume_types.VolumeTypesTest.test_volume_type_update[id-cf9f07c6-db9e-4462-a243-5933ad65e9c8] +tempest.api.volume.admin.test_backends_capabilities.BackendsCapabilitiesAdminTestsJSON.test_get_capabilities_backend[id-3750af44-5ea2-4cd4-bc3e-56e7e6caf854] +tempest.api.compute.test_extensions.ExtensionsTest.test_list_extensions[id-3bb27738-b759-4e0d-a5fa-37d7a6df07d1] +tempest.api.compute.admin.test_services_negative.ServicesAdminNegativeV253TestJSON.test_get_service_with_valid_service_and_invalid_host[id-64e7e7fb-69e8-4cb6-a71d-8d5eb0c98655,negative] +tempest.api.compute.servers.test_servers.ServersTestJSON.test_update_access_server_address[id-89b90870-bc13-4b73-96af-f9d4f2b70077] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_create_backup[id-b963d4f1-94b3-4c40-9e97-7b583f46e470,image] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_resize_server_revert[id-c03aab19-adb1-44f5-917d-c419577e9e68] +tempest.api.compute.servers.test_server_rescue.ServerStableDeviceRescueTest.test_stable_device_rescue_disk_usb[id-647d04cf-ad35-4956-89ab-b05c5c16f30c] +tempest.api.object_storage.test_container_services.ContainerTest.test_update_container_metadata_with_create_and_delete_metadata[id-cf19bc0b-7e16-4a5a-aaed-cb0c2fe8deef] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_shelve_shelved_server[id-443e4f9b-e6bf-4389-b601-3a710f15fddd,negative] +tempest.api.network.test_routers.RoutersTest.test_add_multiple_router_interfaces[id-802c73c9-c937-4cef-824b-2191e24a6aab,smoke] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_marker[id-638f876d-6a43-482a-bbb3-0840bca101c6] +tempest.api.object_storage.test_object_services.ObjectTest.test_list_object_metadata_with_x_object_manifest[id-23a3674c-d6de-46c3-86af-ff92bfc8a3da] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_grant_list_revoke_role_to_user_on_system[id-e5a81737-d294-424d-8189-8664858aae4c] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_list_user_roles_request_without_token[id-682adfb2-fd5f-4b0a-a9ca-322e9bebb907,negative] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_update_port_with_security_group_and_extra_attributes[id-58091b66-4ff4-4cc1-a549-05d60c7acd1a] +tempest.api.image.v2.admin.test_images_metadefs_resource_types.MetadataResourceTypesTest.test_basic_meta_def_resource_type_association[id-6f358a4e-5ef0-11e6-a795-080027d0d606] +tempest.api.network.test_security_groups.SecGroupTest.test_list_security_groups[id-e30abd17-fef9-4739-8617-dc26da88e686,smoke] +tempest.api.identity.v2.test_users.IdentityUsersTest.test_user_update_own_password[id-165859c9-277f-4124-9479-a7d1627b0ca7] +tempest.api.compute.images.test_images.ImagesTestJSON.test_delete_saving_image[id-aa06b52b-2db5-4807-b218-9441f75d74e3] +tempest.api.compute.servers.test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_set_server_metadata_blank_key[id-96100343-7fa9-40d8-80fa-d29ef588ce1c,negative] +tempest.api.image.v1.test_image_members_negative.ImageMembersNegativeTest.test_get_image_without_membership[id-f25f89e4-0b6c-453b-a853-1f80b9d7ef26,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_restore_nonexistent_server_id[id-9c6d38cc-fcfb-437a-85b9-7b788af8bf01,negative] +tempest.api.volume.test_volumes_snapshots_negative.VolumesSnapshotNegativeTestJSON.test_list_snapshots_invalid_param_marker[id-b68deeda-ca79-4a32-81af-5c51179e553a,negative] +tempest.api.network.test_networks.NetworksIpV6Test.test_show_subnet[id-bd635d81-6030-4dd1-b3b9-31ba0cfdf6cc,smoke] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_limit_equals_zero[id-e3b44b7f-ae87-45b5-8a8c-66110eb24d0a] +tempest.api.volume.admin.test_volume_quotas.VolumeQuotasAdminTestJSON.test_list_quotas[id-59eada70-403c-4cef-a2a3-a8ce2f1b07a0] +tempest.scenario.test_network_v6.TestGettingAddress.test_slaac_from_os[compute,id-2c92df61-29f0-4eaa-bee3-7c65bef62a43,network,slow] +tempest.api.identity.admin.v2.test_users.UsersTestJSON.test_update_user[id-39d05857-e8a5-4ed4-ba83-0b52d3ab97ee] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_nonexistent_snapshot_id[id-0c36f6ae-4604-4017-b0a9-34fdc63096f9,negative] +tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON.test_list_flavors[id-e36c0eaa-dff5-4082-ad1f-3f9a80aa3f59,smoke] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_image[id-fcba1052-0a50-4cf3-b1ac-fae241edf02f,negative] +tempest.api.image.v1.test_images.ListImagesTest.test_index_status_active_detail[id-e5dc26d9-9aa2-48dd-bda5-748e1445da98] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_by_name[id-a28e8da4-0b56-472f-87a8-0f4d3f819c02] +tempest.api.identity.admin.v3.test_trusts.TrustsV3TestJSON.test_get_trusts_query[id-6268b345-87ca-47c0-9ce3-37792b43403a] +tempest.api.network.admin.test_routers_negative.RoutersAdminNegativeIpV6Test.test_router_set_gateway_used_ip_returns_409[id-7101cc02-058a-11e7-93e1-fa163e4fa634,negative] +tempest.api.network.test_allowed_address_pair.AllowedAddressPairTestJSON.test_update_port_with_address_pair[id-9599b337-272c-47fd-b3cf-509414414ac4] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_delete_role_non_existent[id-38373691-8551-453a-b074-4260ad8298ef,negative] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_authentication_with_invalid_password[id-d5308b33-3574-43c3-8d87-1c090c5e1eca,negative] +tempest.api.compute.admin.test_servers_on_multinodes.ServersOnMultiNodesTest.test_create_servers_on_different_hosts[id-cc7ca884-6e3e-42a3-a92f-c522fcf25e8e] +tempest.api.volume.admin.test_user_messages.UserMessagesTest.test_delete_message[id-c6eb6901-cdcc-490f-b735-4fe251842aed] +tempest.api.object_storage.test_object_services.ObjectTest.test_copy_object_2d_way[id-06f90388-2d0e-40aa-934c-e9a8833e958a] +tempest.api.object_storage.test_object_services.ObjectTest.test_update_object_metadata[id-7a94c25d-66e6-434c-9c38-97d4e2c29945,smoke] +tempest.api.identity.admin.v2.test_tenant_negative.TenantsNegativeTestJSON.test_tenant_delete_request_without_token[id-e450db62-2e9d-418f-893a-54772d6386b1,negative] +tempest.api.identity.admin.v3.test_credentials.CredentialsTestJSON.test_credentials_create_get_update_delete[id-7cd59bf9-bda4-4c72-9467-d21cab278355,smoke] +tempest.api.identity.admin.v2.test_tokens_negative.TokensAdminTestNegative.test_check_token_existence_negative[id-a0a0a600-4292-4364-99c5-922c834fdf05,negative] +tempest.api.compute.admin.test_flavors.FlavorsAdminTestJSON.test_create_flavor_with_uuid_id[id-94c9bb4e-2c2a-4f3c-bb1f-5f0daf918e6d] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_reverse_order[id-b1811cff-d1ed-4c15-a52e-efd8de41cf34] +tempest.api.compute.security_groups.test_security_group_rules.SecurityGroupRulesTestJSON.test_security_group_rules_create_with_optional_cidr[id-7a01873e-3c38-4f30-80be-31a043cfe2fd] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_get_non_existent_server[id-3436b02f-1b1e-4f03-881e-c6a602327439,negative] +tempest.api.identity.admin.v3.test_services.ServicesTestJSON.test_create_update_get_service[id-5193aad5-bcb7-411d-85b0-b3b61b96ef06,smoke] +tempest.api.identity.admin.v3.test_projects.ProjectsTestJSON.test_project_create_with_domain[id-5f50fe07-8166-430b-a882-3b2ee0abe26f] +tempest.scenario.test_encrypted_cinder_volumes.TestEncryptedCinderVolumes.test_encrypted_cinder_volumes_cryptsetup[compute,id-cbc752ed-b716-4717-910f-956cce965722,image,slow,volume] +tempest.api.compute.admin.test_services_negative.ServicesAdminNegativeV253TestJSON.test_get_service_by_invalid_params[id-d0884a69-f693-4e79-a9af-232d15643bf7,negative] +tempest.api.compute.admin.test_quotas.QuotasAdminTestV236.test_get_updated_quotas[id-4268b5c9-92e5-4adc-acf1-3a2798f3d803] +tempest.api.object_storage.test_container_staticweb.StaticWebTest.test_web_listing_css[id-bc37ec94-43c8-4990-842e-0e5e02fc8926] +tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsNegativeTestJSON.test_allocate_floating_ip_from_nonexistent_pool[id-6e0f059b-e4dd-48fb-8207-06e3bba5b074,negative] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshots_list_details_with_params[id-220a1022-1fcd-4a74-a7bd-6b859156cda2] +tempest.api.identity.admin.v3.test_domains_negative.DomainsNegativeTestJSON.test_create_domain_with_empty_name[id-9018461d-7d24-408d-b3fe-ae37e8cd5c9e,negative] +tempest.api.image.v1.test_images.ListImagesTest.test_index_name[id-097af10a-bae8-4342-bff4-edf89969ed2a] +tempest.api.compute.admin.test_volumes_negative.UpdateMultiattachVolumeNegativeTest.test_multiattach_rw_volume_update_failure[id-7576d497-b7c6-44bd-9cc5-c5b4e50fec71,negative,volume] +tempest.api.compute.servers.test_create_server.ServersTestBootFromVolume.test_host_name_is_same_as_server_name[id-ac1ad47f-984b-4441-9274-c9079b7a0666] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_numeric_server_name[id-fd57f159-68d6-4c2a-902b-03070828a87e,negative] +tempest.api.compute.keypairs.test_keypairs_negative.KeyPairsNegativeTestJSON.test_create_keypair_with_long_keynames[id-3faa916f-779f-4103-aca7-dc3538eee1b7,negative] +tempest.api.network.test_routers.RoutersTest.test_add_remove_router_interface_with_port_id[id-2b7d2f37-6748-4d78-92e5-1d590234f0d5,smoke] +tempest.api.network.admin.test_metering_extensions.MeteringTestJSON.test_list_metering_label_rules[id-cc832399-6681-493b-9d79-0202831a1281] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_get_users_request_without_token[id-a73591ec-1903-4ffe-be42-282b39fefc9d,negative] +tempest.api.compute.servers.test_multiple_create_negative.MultipleCreateNegativeTestJSON.test_min_count_non_integer[id-999aa722-d624-4423-b813-0d1ac9884d7a,negative] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_with_a_deleted_server[id-24a26f1a-1ddc-4eea-b0d7-a90cc874ad8f,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_server_name_blank[id-dbbfd247-c40c-449e-8f6c-d2aa7c7da7cf,negative] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_get_volume_without_passing_volume_id[id-c6c3db06-29ad-4e91-beb0-2ab195fe49e3,negative] +tempest.api.compute.volumes.test_volumes_negative.VolumesNegativeTest.test_volume_get_nonexistent_volume_id[id-c03ea686-905b-41a2-8748-9635154b7c57,negative] +tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_version_statuses[id-8879a470-abfb-47bb-bb8d-5a7fd279ad1e,smoke] +tempest.api.image.v2.test_images_member.ImagesMemberTest.test_get_image_member[id-a6ee18b9-4378-465e-9ad9-9a6de58a3287] +tempest.api.compute.admin.test_flavors_access.FlavorsAccessTestJSON.test_flavor_access_list_with_private_flavor[id-ea2c2211-29fa-4db9-97c3-906d36fad3e0] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_show_non_existent_security_group[id-424fd5c3-9ddc-486a-b45f-39bf0c820fc6,negative] +tempest.api.object_storage.test_account_services_negative.AccountNegativeTest.test_list_containers_with_non_authorized_user[id-070e6aca-6152-4867-868d-1118d68fb38c,negative] +tempest.api.network.admin.test_routers.RoutersIpV6AdminTest.test_create_router_with_snat_explicit[id-ea74068d-09e9-4fd7-8995-9b6a1ace920f] +tempest.api.compute.admin.test_services_negative.ServicesAdminNegativeTestJSON.test_get_service_with_valid_service_and_invalid_host[id-64e7e7fb-69e8-4cb6-a71d-8d5eb0c98655,negative] +tempest.api.object_storage.test_object_services.ObjectTest.test_copy_object_with_x_object_metakey[id-a28a8b99-e701-4d7e-9d84-3b66f121460b] +tempest.api.volume.admin.test_volume_services_negative.VolumeServicesNegativeTest.test_disable_log_reason_with_no_reason[id-77767b36-5e8f-4c68-a0b5-2308cc21ec64,negative] +tempest.api.compute.admin.test_live_migration.LiveAutoBlockMigrationV225Test.test_live_migration_with_trunk[id-0022c12e-a482-42b0-be2d-396b5f0cffe3,network] +tempest.api.volume.test_volumes_snapshots_negative.VolumesSnapshotNegativeTestJSON.test_list_snapshot_invalid_param_limit[id-8fd92339-e22f-4591-86b4-1e2215372a40,negative] +tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_media_types[id-657c1970-4722-4189-8831-7325f3bc4265,smoke] +tempest.api.object_storage.test_container_services.ContainerTest.test_create_container_with_metadata_key[id-c2ac4d59-d0f5-40d5-ba19-0635056d48cd] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_get_invalid_volume_id[id-30799cfd-7ee4-446c-b66c-45b383ed211b,negative] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_flavor[id-5913660b-223b-44d4-a651-a0fbfd44ca75,negative] +tempest.api.object_storage.test_container_services_negative.ContainerNegativeTest.test_update_metadata_with_nonexistent_container_name[id-125a24fa-90a7-4cfc-b604-44e49d788390,negative] +tempest.api.network.test_ports.PortsTestJSON.test_create_update_delete_port[id-c72c1c0c-2193-4aca-aaa4-b1442640f51c,smoke] +tempest.api.compute.servers.test_multiple_create.MultipleCreateTestJSON.test_multiple_create_with_reservation_return[id-864777fb-2f1e-44e3-b5b9-3eb6fa84f2f7] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_assign_user_role_by_unauthorized_user[id-391df5cf-3ec3-46c9-bbe5-5cb58dd4dc41,negative] +tempest.api.identity.admin.v3.test_inherits.InheritsV3TestJSON.test_inherit_assign_list_check_revoke_roles_on_domains_user[id-4e6f0366-97c8-423c-b2be-41eae6ac91c8] +tempest.api.compute.images.test_images_oneserver.ImagesOneServerTestJSON.test_create_image_specify_multibyte_character_image_name[id-3b7c6fe4-dfe7-477c-9243-b06359db51e6] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcp_stateful_router[id-e98f65db-68f4-4330-9fea-abd8c5192d4d] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_show_non_existent_security_group[id-424fd5c3-9ddc-486a-b45f-39bf0c820fc6,negative] +tempest.api.compute.servers.test_server_rescue_negative.ServerRescueNegativeTestJSON.test_rescued_vm_attach_volume[id-d0ccac79-0091-4cf4-a1ce-26162d0cc55f,negative,volume] +tempest.api.compute.servers.test_create_server.ServersTestBootFromVolume.test_verify_server_details[id-5de47127-9977-400a-936f-abcfbec1218f,smoke] +tempest.api.network.admin.test_metering_extensions.MeteringIpV6TestJSON.test_show_metering_label[id-30abb445-0eea-472e-bd02-8649f54a5968] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_create_port_in_allowed_allocation_pools[id-0435f278-40ae-48cb-a404-b8a087bc09b1,smoke] +tempest.api.network.admin.test_floating_ips_admin_actions.FloatingIPAdminTestJSON.test_list_floating_ips_from_admin_and_nonadmin[id-64f2100b-5471-4ded-b46c-ddeeeb4f231b] +tempest.api.identity.admin.v3.test_regions.RegionsTestJSON.test_list_regions_filter_by_parent_region_id[id-2d1057cb-bbde-413a-acdf-e2d265284542] +tempest.api.compute.images.test_image_metadata.ImagesMetadataTestJSON.test_update_image_metadata[id-7b491c11-a9d5-40fe-a696-7f7e03d3fea2] +tempest.api.identity.admin.v2.test_tenants.TenantsTestJSON.test_tenant_update_desc[id-859fcfe1-3a03-41ef-86f9-b19a47d1cd87] +tempest.api.network.test_routers.RoutersTest.test_update_delete_extra_route[id-c86ac3a8-50bd-4b00-a6b8-62af84a0765c] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_show_host_detail_with_nonexistent_hostname[id-e75b0a1a-041f-47a1-8b4a-b72a6ff36d3f,negative] +tempest.api.compute.admin.test_live_migration.LiveMigrationTest.test_volume_backed_live_migration[id-5071cf17-3004-4257-ae61-73a84e28badd,volume] +tempest.scenario.test_unified_limits.ImageQuotaTest.test_image_size_quota[id-05e8d064-c39a-4801-8c6a-465df375ec5b,identity,image] +tempest.api.volume.admin.test_volume_types_extra_specs_negative.ExtraSpecsNegativeTest.test_update_no_body[id-08961d20-5cbb-4910-ac0f-89ad6dbb2da1,negative] +tempest.api.compute.admin.test_networks.NetworksTest.test_list_all_networks[id-df3d1046-6fa5-4b2c-ad0c-cfa46a351cb9] +tempest.api.identity.admin.v3.test_projects.ProjectsTestJSON.test_project_update_name[id-f608f368-048c-496b-ad63-d286c26dab6b] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_delete_user_request_without_a_token[id-57fe1df8-0aa7-46c0-ae9f-c2e785c7504a,negative] +tempest.api.object_storage.test_object_services.ObjectTest.test_copy_object_with_x_fresh_metadata[id-5a9e2cc6-85b6-46fc-916d-0cbb7a88e5fd] +tempest.api.identity.admin.v2.test_tenants.TenantsTestJSON.test_tenant_update_enable[id-8fc8981f-f12d-4c66-9972-2bdcf2bc2e1a] +tempest.api.image.v2.test_images_tags.ImagesTagsTest.test_update_delete_tags_for_image[id-10407036-6059-4f95-a2cd-cbbbee7ed329] +tempest.api.network.admin.test_metering_extensions.MeteringIpV6TestJSON.test_show_metering_label_rule[id-b7354489-96ea-41f3-9452-bace120fb4a7] +tempest.api.identity.admin.v3.test_endpoints.EndPointsTestJSON.test_list_endpoints[id-c19ecf90-240e-4e23-9966-21cee3f6a618] +tempest.api.identity.admin.v2.test_services.ServicesTestJSON.test_create_get_delete_service[id-84521085-c6e6-491c-9a08-ec9f70f90110] +tempest.api.network.test_networks.NetworksIpV6Test.test_list_networks_fields[id-6ae6d24f-9194-4869-9c85-c313cb20e080] +tempest.api.compute.servers.test_server_group.ServerGroupTestJSON.test_create_delete_multiple_server_groups_with_same_name_policy[id-154dc5a4-a2fe-44b5-b99e-f15806a4a113] +tempest.api.compute.volumes.test_volumes_list.VolumesTestJSON.test_volume_list[id-bc2dd1a0-15af-48e5-9990-f2e75a48325d] +tempest.api.compute.admin.test_services_negative.ServicesAdminNegativeTestJSON.test_list_services_with_non_admin_user[id-1126d1f8-266e-485f-a687-adc547492646,negative] +tempest.api.compute.keypairs.test_keypairs_negative.KeyPairsNegativeTestJSON.test_keypair_delete_nonexistent_key[id-7cc32e47-4c42-489d-9623-c5e2cb5a2fa5,negative] +tempest.api.network.test_tags.TagsExtTest.test_update_and_delete_all_tags[id-663a90f5-f334-4b44-afe0-c5fc1d408791] +tempest.api.identity.admin.v3.test_list_users.UsersV3TestJSON.test_list_users_with_not_enabled[id-bff8bf2f-9408-4ef5-b63a-753c8c2124eb] +tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_associate_ip_to_server_without_passing_floating_ip[id-804b4fcb-bbf5-412f-925d-896672b61eb3,negative] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_limit_results[id-67aec2d0-35fe-4503-9f92-f13272b867ed] +tempest.api.compute.servers.test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_update_metadata_with_blank_key[id-a452f38c-05c2-4b47-bd44-a4f0bf5a5e48,negative] +tempest.api.network.test_ports.PortsTestJSON.test_port_list_filter_by_ip_substr[id-79895408-85d5-460d-94e7-9531c5fd9123] +tempest.api.volume.admin.test_group_snapshots.GroupSnapshotsTest.test_create_group_from_group_snapshot[id-eff52c70-efc7-45ed-b47a-4ad675d09b81] +tempest.api.compute.servers.test_server_rescue.ServerStableDeviceRescueTest.test_stable_device_rescue_disk_scsi[id-12340157-6306-4745-bdda-cfa019908b48] +tempest.api.object_storage.test_object_services.ObjectTest.test_update_object_metadata_with_x_object_metakey[id-0dbbe89c-6811-4d84-a2df-eca2bdd40c0e] +tempest.api.compute.volumes.test_attach_volume_negative.AttachVolumeNegativeTest.test_attach_attached_volume_to_same_server[id-aab919e2-d992-4cbb-a4ed-745c2475398c,negative] +tempest.api.volume.admin.test_volume_types.VolumeTypesTest.test_volume_type_create_get_delete[id-4e955c3b-49db-4515-9590-0c99f8e471ad] +tempest.api.image.v2.test_images.ImportImagesTest.test_image_web_download_import[id-f6feb7a4-b04f-4706-a011-206129f83e62] +tempest.api.identity.admin.v3.test_project_tags.IdentityV3ProjectTagsTest.test_list_update_delete_project_tags[id-7c123aac-999d-416a-a0fb-84b915ab10de] +tempest.api.object_storage.test_account_services.AccountTest.test_update_account_metadata_with_delete_metadata[id-9f60348d-c46f-4465-ae06-d51dbd470953] +tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_icmp_type_code[id-c9463db8-b44d-4f52-b6c0-8dbda99f26ce] +tempest.api.object_storage.test_object_services.ObjectTest.test_create_object_with_content_encoding[id-605f8317-f945-4bee-ae91-013f1da8f0a0] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_create_security_group_rule_wrong_ip_prefix_version[id-7607439c-af73-499e-bf64-f687fd12a842,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_network_uuid[id-4e72dc2d-44c5-4336-9667-f7972e95c402,negative] +tempest.api.identity.v3.test_application_credentials.ApplicationCredentialsV3Test.test_list_application_credentials[id-ff0cd457-6224-46e7-b79e-0ada4964a8a6] +tempest.api.compute.admin.test_migrations.MigrationsAdminTest.test_cold_migration[id-4bf0be52-3b6f-4746-9a27-3143636fe30d] +tempest.api.volume.test_volumes_backup.VolumesBackupsTest.test_bootable_volume_backup_and_restore[id-2a8ba340-dff2-4511-9db7-646f07156b15,image] +tempest.api.image.v1.test_images_negative.CreateDeleteImagesNegativeTest.test_delete_image_blank_id[id-04f72aa3-fcec-45a3-81a3-308ef7cc82bc,negative] +tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_created_server_vcpus[id-cbc0f52f-05aa-492b-bdc1-84b575ca294b] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_create_role_request_without_token[id-a7edd17a-e34a-4aab-8bb7-fa6f498645b8,negative] +tempest.api.identity.admin.v3.test_endpoints.EndPointsTestJSON.test_create_list_show_delete_endpoint[id-0e2446d2-c1fd-461b-a729-b9e73e3e3b37] +tempest.api.compute.servers.test_servers_microversions.ServerShowV257Test.test_rebuild_server[id-803df848-080a-4261-8f11-b020cd9b6f60] +tempest.api.compute.admin.test_hypervisor_negative.HypervisorAdminNegativeTestJSON.test_get_nonexistent_hypervisor_uptime[id-f60aa680-9a3a-4c7d-90e1-fae3a4891303,negative] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_get_nonexistent_group[id-673eaec1-9b3e-48ed-bdf1-2786c1b9661c,negative] +tempest.api.object_storage.test_object_services.ObjectTest.test_update_object_metadata_with_remove_metadata[id-48650ed0-c189-4e1e-ad6b-1d4770c6e134] +tempest.api.compute.admin.test_flavors.FlavorsAdminTestJSON.test_create_list_flavor_without_extra_data[id-63dc64e6-2e79-4fdf-868f-85500d308d66] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_list_volumes_detail_with_nonexistent_name[id-9ca17820-a0e7-4cbd-a7fa-f4468735e359,negative] +tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_rebuild[compute,id-88a529c2-1daa-4c85-9aec-d541ba3eb699,network,slow] +tempest.api.compute.servers.test_delete_server.DeleteServersTestJSON.test_delete_server_while_in_shutoff_state[id-546d368c-bb6c-4645-979a-83ed16f3a6be] +tempest.api.identity.admin.v3.test_list_users.UsersV3TestJSON.test_list_users[id-b30d4651-a2ea-4666-8551-0c0e49692635] +tempest.api.compute.admin.test_hypervisor_negative.HypervisorAdminNegativeTestJSON.test_get_hypervisor_stats_with_non_admin_user[id-e2b061bb-13f9-40d8-9d6e-d5bf17595849,negative] +tempest.api.network.test_networks.NetworksIpV6Test.test_list_subnets[id-db68ba48-f4ea-49e9-81d1-e367f6d0b20a,smoke] +tempest.api.network.admin.test_ports.PortsAdminExtendedAttrsIpV6TestJSON.test_show_port_binding_ext_attr[id-b54ac0ff-35fc-4c79-9ca3-c7dbd4ea4f13] +tempest.api.image.v1.test_images.CreateRegisterImagesTest.test_register_then_upload[id-3027f8e6-3492-4a11-8575-c3293017af4d] +tempest.api.object_storage.test_container_acl.ObjectTestACLs.test_write_object_with_rights[id-aa58bfa5-40d9-4bc3-82b4-d07f4a9e392a] +tempest.api.volume.admin.test_volume_types.VolumeTypesTest.test_volume_type_list[id-9d9b28e3-1b2e-4483-a2cc-24aa0ea1de54] +tempest.api.compute.admin.test_live_migration.LiveMigrationTest.test_live_migration_with_trunk[id-0022c12e-a482-42b0-be2d-396b5f0cffe3,network] +tempest.api.volume.admin.test_volume_manage.VolumeManageAdminTest.test_unmanage_manage_volume[id-70076c71-0ce1-4208-a8ff-36a66e65cc1e] +tempest.api.volume.admin.test_volume_pools.VolumePoolsAdminTestsJSON.test_get_pools_without_details[id-0248a46c-e226-4933-be10-ad6fca8227e7] +tempest.scenario.test_minbw_allocation_placement.MinBwAllocationPlacementTest.test_empty_update[compute,id-0805779e-e03c-44fb-900f-ce97a790653b,network] +tempest.api.identity.admin.v2.test_users.UsersTestJSON.test_authentication_request_without_token[id-5d1fa498-4c2d-4732-a8fe-2b054598cfdd] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_create_security_group_rule_with_non_existent_remote_groupid[id-4bf786fd-2f02-443c-9716-5b98e159a49a,negative] +tempest.api.network.test_routers_negative.RoutersNegativeIpV6Test.test_router_add_gateway_invalid_network_returns_404[id-37a94fc0-a834-45b9-bd23-9a81d2fd1e22,negative] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_status_non_existing[id-fcdf192d-0f74-4d89-911f-1ec002b822c4,negative] +tempest.api.network.test_routers.RoutersTest.test_create_show_list_update_delete_router[id-f64403e2-8483-4b34-8ccd-b09a87bcc68c,smoke] +tempest.api.identity.admin.v3.test_projects.ProjectsTestJSON.test_project_create_enabled[id-1f66dc76-50cc-4741-a200-af984509e480] +tempest.api.compute.keypairs.test_keypairs.KeyPairsV2TestJSON.test_get_keypair_detail[id-a4233d5d-52d8-47cc-9a25-e1864527e3df] +tempest.api.compute.security_groups.test_security_group_rules_negative.SecurityGroupRulesNegativeTestJSON.test_create_security_group_rule_with_invalid_ip_protocol[id-84c81249-9f6e-439c-9bbf-cbb0d2cddbdf,negative] +tempest.api.compute.floating_ips.test_list_floating_ips.FloatingIPDetailsTestJSON.test_list_floating_ips[id-16db31c3-fb85-40c9-bbe2-8cf7b67ff99f] +tempest.api.network.admin.test_routers.RoutersIpV6AdminTest.test_create_router_set_gateway_with_fixed_ip[id-cbe42f84-04c2-11e7-8adb-fa163e4fa634] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_volume_extend_with_None_size[id-355218f1-8991-400a-a6bb-971239287d92,negative] +tempest.api.image.v1.test_images.ListImagesTest.test_index_max_size[id-feb32ac6-22bb-4a16-afd8-9454bb714b14] +tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_connectivity_between_vms_on_different_networks[compute,id-1546850e-fbaa-42f5-8b5f-03d8a6a95f15,network,slow] +tempest.api.compute.admin.test_aggregates.AggregatesAdminTestJSON.test_aggregate_create_delete_with_az[id-5873a6f8-671a-43ff-8838-7ce430bb6d0b] +tempest.api.compute.admin.test_instance_usage_audit_log.InstanceUsageAuditLogTestJSON.test_get_instance_usage_audit_log[id-6e40459d-7c5f-400b-9e83-449fbc8e7feb] +tempest.api.image.v2.admin.test_images_metadefs_namespace_objects.MetadataNamespaceObjectsTest.test_list_meta_namespace_objects[id-a2a3615e-3b5c-3f6a-a2b1-1ba3574ae738] +tempest.api.network.test_ports.PortsTestJSON.test_show_port[id-c9a685bd-e83f-499c-939f-9f7863ca259f,smoke] +tempest.api.volume.admin.test_volume_quotas.VolumeQuotasAdminTestJSON.test_quota_usage_after_volume_transfer[id-8911036f-9d54-4720-80cc-a1c9796a8805] +tempest.api.compute.test_quotas.QuotasTestJSON.test_compare_tenant_quotas_with_default_quotas[id-cd65d997-f7e4-4966-a7e9-d5001b674fdc] +tempest.api.compute.admin.test_flavors_access_negative.FlavorsAccessNegativeTestJSON.test_add_flavor_access_duplicate[id-f3592cc0-0306-483c-b210-9a7b5346eddc,negative] +tempest.api.compute.admin.test_floating_ips_bulk.FloatingIPsBulkAdminTestJSON.test_create_list_delete_floating_ips_bulk[id-2c8f145f-8012-4cb8-ac7e-95a587f0e4ab,network] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_delete_role_request_without_token[id-44b60b20-70de-4dac-beaf-a3fc2650a16b,negative] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_assign_user_role_for_non_existent_role[id-99b297f6-2b5d-47c7-97a9-8b6bb4f91042,negative] +tempest.api.network.test_routers_negative.RoutersNegativeIpV6Test.test_router_remove_interface_in_use_returns_409[id-04df80f9-224d-47f5-837a-bf23e33d1c20,negative] +tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON.test_list_images_with_detail_filter_by_status[id-9b0ea018-6185-4f71-948a-a123a107988e] +tempest.api.network.admin.test_quotas.QuotasTest.test_show_quota_details[id-7b05ec5f-bf44-43cb-b28f-ddd72a824288] +tempest.api.object_storage.test_account_bulk.BulkTest.test_bulk_delete_by_POST[id-dbea2bcb-efbb-4674-ac8a-a5a0e33d1d79] +tempest.api.image.v1.test_images_negative.CreateDeleteImagesNegativeTest.test_register_with_invalid_disk_format[id-993face5-921d-4e84-aabf-c1bba4234a67,negative] +tempest.api.compute.keypairs.test_keypairs_v22.KeyPairsV22TestJSON.test_keypairsv22_create_list_show_with_type[id-89d59d43-f735-441a-abcf-0601727f47b6] +tempest.api.compute.admin.test_quotas_negative.QuotasAdminNegativeTest.test_create_server_when_instances_quota_is_full[id-7c6be468-0274-449a-81c3-ac1c32ee0161,negative] +tempest.api.volume.admin.test_group_types.GroupTypesTest.test_group_type_create_list_update_show[id-dd71e5f9-393e-4d4f-90e9-fa1b8d278864] +tempest.api.object_storage.test_account_quotas.AccountQuotasTest.test_upload_valid_object[id-a22ef352-a342-4587-8f47-3bbdb5b039c4,smoke] +tempest.api.volume.admin.test_volume_quotas.VolumeQuotasAdminTestJSON.test_delete_quota[id-874b35a9-51f1-4258-bec5-cd561b6690d3] +tempest.api.identity.admin.v3.test_users.UsersV3TestJSON.test_list_user_projects[id-a831e70c-e35b-430b-92ed-81ebbc5437b8] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_reboot_nonexistent_host[id-f86bfd7b-0b13-4849-ae29-0322e83ee58b,negative] +tempest.api.image.v1.test_images_negative.CreateDeleteImagesNegativeTest.test_delete_image_negative_image_id[id-4ed757cd-450c-44b1-9fd1-c819748c650d,negative] +tempest.api.volume.admin.test_volume_quota_classes.VolumeQuotaClassesTest.test_show_default_quota[id-abb9198e-67d0-4b09-859f-4f4a1418f176] +tempest.api.volume.admin.test_snapshot_manage.SnapshotManageAdminTest.test_unmanage_manage_snapshot[id-0132f42d-0147-4b45-8501-cc504bbf7810] +tempest.api.compute.servers.test_create_server.ServersTestBootFromVolume.test_list_servers[id-9a438d88-10c6-4bcd-8b5b-5b6e25e1346f,smoke] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_create_user_for_non_existent_tenant[id-0132cc22-7c4f-42e1-9e50-ac6aad31d59a,negative] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_detail_server_is_deleted[id-93055106-2d34-46fe-af68-d9ddbf7ee570,negative] +tempest.api.object_storage.test_container_acl_negative.ObjectACLsNegativeTest.test_read_object_with_non_authorized_user[id-abf63359-be52-4feb-87dd-447689fc77fd,negative] +tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON.test_list_flavors_filter_by_min_disk[id-10645a4d-96f5-443f-831b-730711e11dd4] +tempest.api.volume.admin.test_volume_types_extra_specs_negative.ExtraSpecsNegativeTest.test_list_nonexistent_volume_type_id[id-dee5cf0c-cdd6-4353-b70c-e847050d71fb,negative] +tempest.api.identity.admin.v3.test_oauth_consumers.OAUTHConsumersV3Test.test_create_and_show_consumer[id-c8307ea6-a86c-47fd-ae7b-5b3b2caca76d] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_image[id-05e8a8e7-9659-459a-989d-92c2f501f4ba] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_flavor[id-573637f5-7325-47bb-9144-3476d0416908] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents_with_marker[id-c31ddc63-2a58-4f6b-b25c-94d2937e6867] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_sort[id-eeadce49-04e0-43b7-aec7-52535d903e7a] +tempest.api.network.test_extra_dhcp_options.ExtraDHCPOptionsIpV6TestJSON.test_create_list_port_with_extra_dhcp_options[id-d2c17063-3767-4a24-be4f-a23dbfa133c9] +tempest.api.network.admin.test_ports.PortsAdminExtendedAttrsTestJSON.test_list_ports_binding_ext_attr[compute,id-1c82a44a-6c6e-48ff-89e1-abe7eaf8f9f8] +tempest.api.compute.admin.test_servers_negative.ServersAdminNegativeTestJSON.test_resize_server_using_overlimit_ram[id-28dcec23-f807-49da-822c-56a92ea3c687,negative] +tempest.api.network.test_networks.NetworksTest.test_list_subnets[id-db68ba48-f4ea-49e9-81d1-e367f6d0b20a,smoke] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_shelve_non_existent_server[id-abca56e2-a892-48ea-b5e5-e07e69774816,negative] +tempest.api.image.v2.admin.test_images.BasicOperationsImagesAdminTest.test_update_image_owner_param[id-525ba546-10ef-4aad-bba1-1858095ce553] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_pagination[id-af55e775-8e4b-4feb-8719-215c43b0238c] +tempest.api.identity.admin.v3.test_projects.ProjectsTestJSON.test_create_is_domain_project[id-a7eb9416-6f9b-4dbb-b71b-7f73aaef59d5] +tempest.api.network.test_routers_negative.DvrRoutersNegativeTest.test_router_create_tenant_distributed_returns_forbidden[id-4990b055-8fc7-48ab-bba7-aa28beaad0b9,negative] +tempest.api.compute.servers.test_delete_server.DeleteServersTestJSON.test_delete_server_while_in_suspended_state[id-1f82ebd3-8253-4f4e-b93f-de9b7df56d8b] +tempest.api.compute.servers.test_disk_config.ServerDiskConfigTestJSON.test_resize_server_from_manual_to_auto[id-414e7e93-45b5-44bc-8e03-55159c6bfc97] +tempest.api.compute.admin.test_flavors.FlavorsAdminTestJSON.test_create_flavor_with_none_id[id-f83fe669-6758-448a-a85e-32d351f36fe0] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_create_port_with_no_securitygroups[id-4179dcb9-1382-4ced-84fe-1b91c54f5735,smoke] +tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON.test_list_images_with_detail_filter_by_changes_since[id-7d439e18-ac2e-4827-b049-7e18004712c4] +tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_stop_start[compute,id-61f1aa9a-1573-410e-9054-afa557cab021,network,slow] +tempest.api.compute.admin.test_live_migration.LiveMigrationTest.test_live_block_migration[id-1dce86b8-eb04-4c03-a9d8-9c1dc3ee0c7b] +tempest.api.compute.admin.test_live_migration_negative.LiveMigrationNegativeTest.test_invalid_host_for_migration[id-7fb7856e-ae92-44c9-861a-af62d7830bcb,negative] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_delete_the_default_security_group[id-36a1629f-c6da-4a26-b8b8-55e7e5d5cd58,negative] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents_with_end_marker[id-55b4fa5c-e12e-4ca9-8fcf-a79afe118522] +tempest.api.compute.admin.test_flavors.FlavorsAdminTestJSON.test_create_server_with_non_public_flavor[id-bcc418ef-799b-47cc-baa1-ce01368b8987] +tempest.api.network.test_networks.NetworksTest.test_list_subnets_fields[id-842589e3-9663-46b0-85e4-7f01273b0412] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_create_user_by_unauthorized_user[id-60a1f5fa-5744-4cdf-82bf-60b7de2d29a4,negative] +tempest.api.image.v1.test_images_negative.CreateDeleteImagesNegativeTest.test_delete_image_id_over_character_limit[id-a4a448ab-3db2-4d2d-b9b2-6a1271241dfe,negative] +tempest.api.compute.admin.test_flavors_access_negative.FlavorsAccessNegativeTestJSON.test_flavor_non_admin_remove[id-073e79a6-c311-4525-82dc-6083d919cb3a,negative] +tempest.api.volume.admin.test_volume_types_extra_specs_negative.ExtraSpecsNegativeTest.test_update_multiple_extra_spec[id-a77dfda2-9100-448e-9076-ed1711f4bdfc,negative] +tempest.api.compute.servers.test_servers.ServerShowV263Test.test_show_update_rebuild_list_server[id-71b8e3d5-11d2-494f-b917-b094a4afed3c] +tempest.api.compute.servers.test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_server_metadata_non_existent_server[id-4d9cd7a3-2010-4b41-b8fe-3bbf0b169466,negative] +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_dhcp_enabled[id-94ce038d-ff0a-4a4c-a56b-09da3ca0b55d] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_invalid_ports[id-0d9c7791-f2ad-4e2f-ac73-abf2373b0d2d,negative] +tempest.api.identity.admin.v2.test_users.UsersTestJSON.test_list_users_with_roles_for_tenant[id-a8b54974-40e1-41c0-b812-50fc90827971] +tempest.api.network.admin.test_metering_extensions.MeteringIpV6TestJSON.test_list_metering_label_rules[id-cc832399-6681-493b-9d79-0202831a1281] +tempest.api.volume.admin.test_multi_backend.VolumeMultiBackendTest.test_backend_name_distinction_with_prefix[id-4236305b-b65a-4bfc-a9d2-69cb5b2bf2ed] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_limit[id-614cdfc1-d557-4bac-915b-3e67b48eee76] +tempest.api.compute.images.test_images_oneserver_negative.ImagesOneServerNegativeTestJSON.test_create_image_specify_metadata_over_limits[id-3d24d11f-5366-4536-bd28-cff32b748eca,negative] +tempest.api.volume.test_volume_transfers.VolumesTransfersTest.test_create_get_list_accept_volume_transfer[id-4d75b645-a478-48b1-97c8-503f64242f1a] +tempest.api.identity.admin.v2.test_tenant_negative.TenantsNegativeTestJSON.test_delete_non_existent_tenant[id-9c9a2aed-6e3c-467a-8f5c-89da9d1b516b,negative] +tempest.api.image.v1.test_images.ListImagesTest.test_index_no_params[id-246178ab-3b33-4212-9a4b-a7fe8261794d] +tempest.api.network.test_routers.RoutersIpV6Test.test_router_interface_port_update_with_fixed_ip[id-96522edf-b4b5-45d9-8443-fa11c26e6eff] +tempest.api.compute.admin.test_flavors.FlavorsAdminTestJSON.test_list_public_flavor_with_other_user[id-b345b196-bfbd-4231-8ac1-6d7fe15ff3a3] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_tag[id-aa8ac4df-cff9-418b-8d0f-dd9c67b072c9] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_delete_nonexistent_security_group[id-6727c00b-214c-4f9e-9a52-017ac3e98411,negative] +tempest.api.compute.servers.test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_set_server_metadata_missing_metadata[id-64a91aee-9723-4863-be44-4c9d9f1e7d0e,negative] +tempest.api.identity.admin.v3.test_oauth_consumers.OAUTHConsumersV3Test.test_delete_consumer[id-fdfa1b7f-2a31-4354-b2c7-f6ae20554f93] +tempest.scenario.test_volume_boot_pattern.TestVolumeBootPattern.test_boot_server_from_encrypted_volume_luks[compute,id-cb78919a-e553-4bab-b73b-10cf4d2eb125,volume] +tempest.api.compute.servers.test_delete_server.DeleteServersTestJSON.test_delete_server_while_in_building_state[id-9e6e0c87-3352-42f7-9faf-5d6210dbd159] +tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_update_instance_port_admin_state[compute,id-f5dfcc22-45fd-409f-954c-5bd500d7890b,network,slow] +tempest.api.compute.security_groups.test_security_groups.SecurityGroupsTestJSON.test_update_security_groups[id-7d4e1d3c-3209-4d6d-b020-986304ebad1f] +tempest.api.network.test_security_groups.SecGroupTest.test_create_show_delete_security_group_rule[id-cfb99e0e-7410-4a3d-8a0c-959a63ee77e9,smoke] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_resize_server_with_non_existent_flavor[id-ced1a1d7-2ab6-45c9-b90f-b27d87b30efd,negative] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_list_roles_request_without_token[id-11a3c7da-df6c-40c2-abc2-badd682edf9f,negative] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_shutdown_host_with_non_admin_user[id-a803529c-7e3f-4d3c-a7d6-8e1c203d27f6,negative] +tempest.api.object_storage.test_object_slo.ObjectSloTest.test_retrieve_large_object[id-49bc49bc-dd1b-4c0f-904e-d9f10b830ee8] +tempest.api.compute.admin.test_aggregates.AggregatesAdminTestV241.test_create_update_show_aggregate_add_remove_host[id-fdf24d9e-8afa-4700-b6aa-9c498351504f] +tempest.api.identity.admin.v2.test_tenants.TenantsTestJSON.test_tenant_create_enabled[id-670bdddc-1cd7-41c7-b8e2-751cfb67df50] +tempest.api.volume.admin.test_volume_retype.VolumeRetypeWithMigrationTest.test_volume_from_snapshot_retype_with_migration[id-d0d9554f-e7a5-4104-8973-f35b27ccb60d] +tempest.api.volume.admin.test_volume_types.VolumeTypesTest.test_volume_crud_with_volume_type_and_extra_specs[id-c03cc62c-f4e9-4623-91ec-64ce2f9c1260] +tempest.api.network.test_routers_negative.RoutersNegativeTest.test_delete_non_existent_router_returns_404[id-c7edc5ad-d09d-41e6-a344-5c0c31e2e3e4,negative] +tempest.api.identity.v3.test_projects.IdentityV3ProjectsTest.test_list_projects_returns_only_authorized_projects[id-86128d46-e170-4644-866a-cc487f699e1d] +tempest.api.object_storage.test_object_services.ObjectTest.test_create_object_with_x_object_metakey[id-e4183917-33db-4153-85cc-4dacbb938865] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_exceed_limit[id-37791bbd-90c0-4de0-831e-5f38cba9c6b3] +tempest.api.compute.security_groups.test_security_groups.SecurityGroupsTestJSON.test_security_groups_create_list_delete[id-eb2b087d-633d-4d0d-a7bd-9e6ba35b32de,smoke] +tempest.api.object_storage.test_account_services.AccountTest.test_list_no_containers[id-884ec421-fbad-4fcc-916b-0580f2699565] +tempest.api.compute.servers.test_create_server.ServersTestJSON.test_list_servers_with_detail[id-585e934c-448e-43c4-acbf-d06a9b899997] +tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers_with_detail[id-585e934c-448e-43c4-acbf-d06a9b899997] +tempest.api.image.v2.test_images.ListUserImagesTest.test_get_image_schema[id-622b925c-479f-4736-860d-adeaf13bc371] +tempest.api.network.admin.test_routers.RoutersIpV6AdminTest.test_create_router_setting_project_id[id-e54dd3a3-4352-4921-b09d-44369ae17397] +tempest.api.identity.admin.v3.test_regions.RegionsTestJSON.test_create_update_get_delete_region[id-56186092-82e4-43f2-b954-91013218ba42] +tempest.api.volume.admin.test_multi_backend.VolumeMultiBackendTest.test_backend_name_reporting[id-c1a41f3f-9dad-493e-9f09-3ff197d477cc] +tempest.api.volume.admin.test_volume_quotas.VolumeQuotasAdminTestJSON.test_update_all_quota_resources_for_tenant[id-3d45c99e-cc42-4424-a56e-5cbd212b63a6] +tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_port_security_macspoofing_port[compute,id-7c0bb1a2-d053-49a4-98f9-ca1a1d849f63,network,slow] +tempest.api.identity.v3.test_access_rules.AccessRulesV3Test.test_list_access_rules[id-2354c498-5119-4ba5-9f0d-44f16f78fb0e] +tempest.api.identity.v3.test_ec2_credentials.EC2CredentialsTest.test_delete_ec2_credential[id-9408d61b-8be0-4a8d-9b85-14f61edb456b] +tempest.api.object_storage.test_container_services_negative.ContainerNegativeTest.test_create_container_metadata_name_exceeds_max_length[id-41e645bf-2e68-4f84-bf7b-c71aa5cd76ce,negative] +tempest.api.compute.admin.test_flavors_extra_specs_negative.FlavorsExtraSpecsNegativeTestJSON.test_flavor_unset_nonexistent_key[id-440b9f3f-3c7f-4293-a106-0ceda350f8de,negative] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_security_group_with_invalid_sg_des[id-97d12b1c-a610-4194-93f1-ba859e718b45,negative] +tempest.api.compute.servers.test_instance_actions_negative.InstanceActionsNegativeTestJSON.test_list_instance_actions_non_existent_server[id-67e1fce6-7ec2-45c6-92d4-0a8f1a632910,negative] +tempest.api.volume.admin.test_volume_snapshot_quotas_negative.VolumeSnapshotQuotasNegativeTestJSON.test_quota_volume_snapshots[id-02bbf63f-6c05-4357-9d98-2926a94064ff,negative] +tempest.api.compute.admin.test_instance_usage_audit_log_negative.InstanceUsageAuditLogNegativeTestJSON.test_get_instance_usage_audit_logs_with_invalid_time[id-9b952047-3641-41c7-ba91-a809fc5974c8,negative] +tempest.api.compute.admin.test_quotas_negative.QuotasSecurityGroupAdminNegativeTest.test_security_groups_rules_exceed_limit[id-6e9f436d-f1ed-4f8e-a493-7275dfaa4b4d,negative,network] +tempest.api.compute.admin.test_servers.ServersAdminTestJSON.test_reset_network_inject_network_info[id-7a1323b4-a6a2-497a-96cb-76c07b945c71] +tempest.api.object_storage.test_object_services.ObjectTest.test_create_object_with_transfer_encoding[id-4f84422a-e2f2-4403-b601-726a4220b54e] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_roles_hierarchy[id-dc6f5959-b74d-4e30-a9e5-a8255494ff00] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_non_existent_security_group[id-be308db6-a7cf-4d5c-9baf-71bafd73f35e,negative] +tempest.api.volume.admin.test_volume_hosts.VolumeHostsAdminTestsJSON.test_list_hosts[id-d5f3efa2-6684-4190-9ced-1c2f526352ad] +tempest.api.compute.admin.test_services.ServicesAdminTestJSON.test_list_services[id-5be41ef4-53d1-41cc-8839-5c2a48a1b283] +tempest.api.network.test_tags.TagsExtTest.test_create_check_list_and_delete_tags[id-c6231efa-9a89-4adf-b050-2a3156b8a1d9] +tempest.api.compute.servers.test_multiple_create.MultipleCreateTestJSON.test_multiple_create[id-61e03386-89c3-449c-9bb1-a06f423fd9d1] +tempest.api.identity.admin.v3.test_projects_negative.ProjectsNegativeTestJSON.test_project_delete_by_unauthorized_user[id-8d68c012-89e0-4394-8d6b-ccd7196def97,negative] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_create_security_group_rule_with_bad_ethertype[id-5666968c-fff3-40d6-9efc-df1c8bd01abb,negative] +tempest.api.compute.admin.test_flavors.FlavorsAdminTestJSON.test_create_flavor_using_string_ram[id-3b541a2e-2ac2-4b42-8b8d-ba6e22fcd4da] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_server_name_length_exceeds_256[id-5c8e244c-dada-4590-9944-749c455b431f,negative] +tempest.api.identity.admin.v2.test_tenant_negative.TenantsNegativeTestJSON.test_create_tenant_by_unauthorized_user[id-d26b278a-6389-4702-8d6e-5980d80137e0,negative] +tempest.api.compute.servers.test_server_rescue.ServerRescueTestJSONUnderV235.test_rescued_vm_associate_dissociate_floating_ip[id-4842e0cf-e87d-4d9d-b61f-f4791da3cacc] +tempest.api.compute.volumes.test_volumes_list.VolumesTestJSON.test_volume_list_param_offset_and_limit[id-51c22651-a074-4ea7-af0b-094f9331303e] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcpv6_stateless_no_ra_no_dhcp[id-21635b6f-165a-4d42-bf49-7d195e47342f] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_update_image_reserved_property[id-a0ae75d4-ce9c-4576-b823-aba04c8acabd,negative] +tempest.api.network.test_routers_negative.RoutersNegativeIpV6Test.test_router_add_gateway_net_not_external_returns_400[id-11836a18-0b15-4327-a50b-f0d9dc66bddd,negative] +tempest.api.compute.keypairs.test_keypairs_v22.KeyPairsV22TestJSON.test_keypair_create_delete[id-6c1d3123-4519-4742-9194-622cb1714b7d] +tempest.api.object_storage.test_healthcheck.HealthcheckTest.test_get_healthcheck[id-db5723b1-f25c-49a9-bfeb-7b5640caf337] +tempest.api.network.test_routers.RoutersIpV6Test.test_add_multiple_router_interfaces[id-802c73c9-c937-4cef-824b-2191e24a6aab,smoke] +tempest.api.compute.images.test_images_oneserver_negative.ImagesOneServerNegativeTestJSON.test_create_second_image_when_first_image_is_being_saved[id-0460efcf-ee88-4f94-acef-1bf658695456,negative] +tempest.api.network.test_allowed_address_pair.AllowedAddressPairIpV6TestJSON.test_update_port_with_address_pair[id-9599b337-272c-47fd-b3cf-509414414ac4] +tempest.api.compute.admin.test_flavors_extra_specs_negative.FlavorsExtraSpecsNegativeTestJSON.test_flavor_get_nonexistent_key[id-329a7be3-54b2-48be-8052-bf2ce4afd898,negative] +tempest.api.compute.admin.test_instance_usage_audit_log.InstanceUsageAuditLogTestJSON.test_list_instance_usage_audit_logs[id-25319919-33d9-424f-9f99-2c203ee48b9d] +tempest.api.compute.admin.test_aggregates.AggregatesAdminTestJSON.test_aggregate_add_host_get_details[id-eeef473c-7c52-494d-9f09-2ed7fc8fc036] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_stop_non_existent_server[id-a31460a9-49e1-42aa-82ee-06e0bb7c2d03,negative] +tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsNegativeTestJSON.test_delete_nonexistent_floating_ip[id-ae1c55a8-552b-44d4-bfb6-2a115a15d0ba,negative] +tempest.api.volume.admin.test_qos.QosSpecsTestJSON.test_get_qos[id-7aa214cc-ac1a-4397-931f-3bb2e83bb0fd] +tempest.api.compute.servers.test_server_password.ServerPasswordTestJSON.test_delete_server_password[id-f8229e8b-b625-4493-800a-bde86ac611ea] +tempest.api.network.admin.test_external_network_extension.ExternalNetworksTestJSON.test_show_external_networks_attribute[id-2ac50ab2-7ebd-4e27-b3ce-a9e399faaea2] +tempest.api.compute.servers.test_server_group.ServerGroupTestJSON.test_list_server_groups[id-d4874179-27b4-4d7d-80e4-6c560cdfe321] +tempest.api.volume.admin.test_volumes_actions.VolumesActionsTest.test_force_detach_volume[compute,id-d38285d9-929d-478f-96a5-00e66a115b81] +tempest.scenario.test_object_storage_basic_ops.TestObjectStorageBasicOps.test_swift_acl_anonymous_download[id-916c7111-cb1f-44b2-816d-8f760e4ea910,object_storage,slow] +tempest.api.identity.v2.test_tenants.IdentityTenantsTest.test_list_tenants_returns_only_authorized_tenants[id-ecae2459-243d-4ba1-ad02-65f15dc82b78] +tempest.api.image.v1.test_images_negative.CreateDeleteImagesNegativeTest.test_delete_image_non_hex_string_id[id-950e5054-a3c7-4dee-ada5-e576f1087abd,negative] +tempest.api.network.test_networks.NetworksIpV6Test.test_create_delete_subnet_with_allocation_pools[id-bec949c4-3147-4ba6-af5f-cd2306118404] +tempest.api.object_storage.test_object_services.ObjectTest.test_create_object_with_x_object_meta[id-1c7ed3e4-2099-406b-b843-5301d4811baf] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_protocol[id-981bdc22-ce48-41ed-900a-73148b583958,negative] +tempest.api.compute.admin.test_services.ServicesAdminTestJSON.test_get_service_by_service_binary_name[id-f345b1ec-bc6e-4c38-a527-3ca2bc00bef5] +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_all_attributes[id-a4d9ec4c-0306-4111-a75c-db01a709030b] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_create_aggregate_name_length_exceeds_255[id-4c194563-543b-4e70-a719-557bbe947fac,negative] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_resize_server_confirm_from_stopped[id-138b131d-66df-48c9-a171-64f45eb92962] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_limit_equals_infinite[id-a1427f61-420e-48a5-b6e3-0b394fa95400] +tempest.api.compute.admin.test_servers_on_multinodes.ServersOnMultiNodesTest.test_create_server_with_scheduler_hint_group_anti_affinity[id-f8bd0867-e459-45f5-ba53-59134552fe04] +tempest.api.compute.servers.test_servers_microversions.ServerShowV254Test.test_rebuild_server[id-09170a98-4940-4637-add7-1a35121f1a5a] +tempest.scenario.test_minimum_basic.TestMinimumBasicScenario.test_minimum_basic_scenario[compute,id-bdbb5441-9204-419d-a225-b4fdbfb1a1a8,image,network,volume] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_prefix[id-365e6fc7-1cfe-463b-a37c-8bd08d47b6aa] +tempest.api.network.test_routers.RoutersTest.test_router_interface_port_update_with_fixed_ip[id-96522edf-b4b5-45d9-8443-fa11c26e6eff] +tempest.api.object_storage.test_container_sync_middleware.ContainerSyncMiddlewareTest.test_container_synchronization[id-ea4645a1-d147-4976-82f7-e5a7a3065f80,slow] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_server_invalid_bdm_in_2nd_dict[id-12146ac1-d7df-4928-ad25-b1f99e5286cd,negative,volume] +tempest.api.object_storage.test_container_quotas.ContainerQuotasTest.test_upload_large_object[id-22eeeb2b-3668-4160-baef-44790f65a5a0,smoke] +tempest.api.compute.floating_ips.test_list_floating_ips.FloatingIPDetailsTestJSON.test_get_floating_ip_details[id-eef497e0-8ff7-43c8-85ef-558440574f84] +tempest.api.object_storage.test_object_services.ObjectTest.test_get_object_with_if_none_match[id-641500d5-1612-4042-a04d-01fc4528bc30] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_reboot_host_with_non_admin_user[id-02d79bb9-eb57-4612-abf6-2cb38897d2f8,negative] +tempest.api.network.test_tags.TagsTest.test_create_list_show_update_delete_tags[id-ee76bfaf-ac94-4d74-9ecc-4bbd4c583cb1] +tempest.api.identity.admin.v2.test_users.UsersTestJSON.test_list_users_for_tenant[id-6e317209-383a-4bed-9f10-075b7c82c79a] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_pass_negative_value[id-62610dd9-4713-4ee0-8beb-fd2c1aa7f950,negative] +tempest.api.object_storage.test_object_services.ObjectTest.test_create_object_with_etag[id-73820093-0503-40b1-a478-edf0e69c7d1f] +tempest.api.compute.admin.test_flavors_microversions.FlavorsV261TestJSON.test_crud_flavor[id-61976b25-488d-41dc-9dcb-cb9693a7b075] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_grant_list_revoke_role_to_group_on_domain[id-4bf8a70b-e785-413a-ad53-9f91ce02faa7] +tempest.api.identity.admin.v3.test_groups.GroupsV3TestJSON.test_group_users_add_list_delete[id-1598521a-2f36-4606-8df9-30772bd51339,smoke] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_limit[id-db4d8e0a-7a2e-41cc-a712-961f6844e896] +tempest.api.compute.servers.test_multiple_create_negative.MultipleCreateNegativeTestJSON.test_min_count_less_than_one[id-daf29d8d-e928-4a01-9a8c-b129603f3fc0,negative] +tempest.api.identity.admin.v3.test_projects.ProjectsTestJSON.test_project_create_with_parent[id-1854f9c0-70bc-4d11-a08a-1c789d339e3d] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_server_name[id-e2c77c4a-000a-4af3-a0bd-629a328bde7c,negative] +tempest.api.volume.admin.test_qos.QosSpecsTestJSON.test_set_unset_qos_key[id-ed00fd85-4494-45f2-8ceb-9e2048919aed] +tempest.api.object_storage.test_object_services.ObjectTest.test_get_object_if_different[id-50d01f12-526f-4360-9ac2-75dd508d7b68] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_non_existent_remote_groupid[id-4bf786fd-2f02-443c-9716-5b98e159a49a,negative] +tempest.api.compute.images.test_images_negative.ImagesDeleteNegativeTestJSON.test_delete_non_existent_image[id-137aef61-39f7-44a1-8ddf-0adf82511701,negative] +tempest.api.identity.admin.v3.test_domains.DomainsTestJSON.test_create_update_delete_domain[id-f2f5b44a-82e8-4dad-8084-0661ea3b18cf,smoke] +tempest.api.compute.servers.test_server_rescue.ServerStableDeviceRescueTest.test_stable_device_rescue_disk_virtio_with_volume_attached[id-a3772b42-00bf-4310-a90b-1cc6fd3e7eab,volume] +tempest.api.compute.test_quotas.QuotasTestJSON.test_get_default_quotas[id-9bfecac7-b966-4f47-913f-1a9e2c12134a] +tempest.api.identity.v2.test_api_discovery.TestApiDiscovery.test_api_version_resources[id-ea889a68-a15f-4166-bfb1-c12456eae853,smoke] +tempest.api.compute.admin.test_quotas.QuotasAdminTestJSON.test_delete_quota[id-389d04f0-3a41-405f-9317-e5f86e3c44f0] +tempest.api.network.test_extensions.ExtensionsTestJSON.test_list_show_extensions[id-ef28c7e6-e646-4979-9d67-deb207bc5564,smoke] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_create_user_request_without_a_token[id-55bbb103-d1ae-437b-989b-bcdf8175c1f4,negative] +tempest.api.compute.admin.test_servers_negative.ServersAdminNegativeTestJSON.test_reset_state_server_invalid_type[id-4cdcc984-fab0-4577-9a9d-6d558527ee9d,negative] +tempest.api.compute.volumes.test_volumes_list.VolumesTestJSON.test_volume_list_with_detail_param_limit[id-33985568-4965-49d5-9bcc-0aa007ca5b7a] +tempest.api.compute.servers.test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_delete_metadata_non_existent_server[id-6bbd88e1-f8b3-424d-ba10-ae21c45ada8d,negative] +tempest.api.compute.keypairs.test_keypairs.KeyPairsV2TestJSON.test_keypair_create_with_pub_key[id-39c90c6a-304a-49dd-95ec-2366129def05] +tempest.api.identity.admin.v3.test_credentials.CredentialsTestJSON.test_credentials_list_delete[id-13202c00-0021-42a1-88d4-81b44d448aab] +tempest.api.object_storage.test_object_services.ObjectTest.test_delete_object[id-17738d45-03bd-4d45-9e0b-7b2f58f98687] +tempest.api.network.test_networks_negative.NetworksNegativeTestJSON.test_update_non_existent_subnet[id-1cc47884-ac52-4415-a31c-e7ce5474a868,negative] +tempest.api.compute.volumes.test_volumes_list.VolumesTestJSON.test_volume_list_param_limit[id-1048ed81-2baf-487a-b284-c0622b86e7b8] +tempest.api.compute.servers.test_server_rescue_negative.ServerRescueNegativeTestJSON.test_rescue_paused_instance[id-cc3a883f-43c0-4fb6-a9bb-5579d64984ed,negative] +tempest.api.network.admin.test_routers.RoutersAdminTest.test_update_router_set_gateway_with_snat_explicit[id-b386c111-3b21-466d-880c-5e72b01e1a33] +tempest.api.network.test_networks.NetworksTest.test_show_network[id-2bf13842-c93f-4a69-83ed-717d2ec3b44e,smoke] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_name[id-55c8f5f5-bfed-409d-a6d5-4caeda985d7b] +tempest.api.image.v2.test_images_member_negative.ImagesMemberNegativeTest.test_image_share_owner_cannot_accept[id-27002f74-109e-4a37-acd0-f91cd4597967,negative] +tempest.api.volume.admin.test_volume_types_extra_specs_negative.ExtraSpecsNegativeTest.test_delete_nonexistent_volume_type_id[id-031cda8b-7d23-4246-8bf6-bbe73fd67074,negative] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_server_name[id-9b067a7b-7fee-4f6a-b29c-be43fe18fc5a] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_reboot_server_hard[id-2cb1baf6-ac8d-4429-bf0d-ba8a0ba53e32,smoke] +tempest.api.identity.admin.v3.test_users_negative.UsersNegativeTest.test_create_user_for_non_existent_domain[id-e75f006c-89cc-477b-874d-588e4eab4b17,negative] +tempest.api.compute.keypairs.test_keypairs.KeyPairsV2TestJSON.test_keypairs_create_list_delete[id-1d1dbedb-d7a0-432a-9d09-83f543c3c19b] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_grant_list_revoke_role_to_user_on_domain[id-6c9a2940-3625-43a3-ac02-5dcec62ef3bd] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_update_port_with_two_security_groups_and_extra_attributes[id-edf6766d-3d40-4621-bc6e-2521a44c257d] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_resize_nonexistent_server[id-7ea45b3e-e770-46fa-bfcc-9daaf6d987c0,negative] +tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_network_basic_ops[compute,id-f323b3ba-82f8-4db7-8ea6-6a895869ec49,network,smoke] +tempest.api.compute.security_groups.test_security_group_rules_negative.SecurityGroupRulesNegativeTestJSON.test_create_security_group_rule_with_invalid_id[id-2244d7e4-adb7-4ecb-9930-2d77e123ce4f,negative] +tempest.api.volume.admin.test_volume_type_access.VolumeTypesAccessTest.test_volume_type_access_add[id-d4dd0027-835f-4554-a6e5-50903fb79184] +tempest.api.identity.admin.v3.test_users.UsersV3TestJSON.test_update_user_password[id-2d223a0e-e457-4a70-9fb1-febe027a0ff9] +tempest.api.volume.test_volumes_snapshots_negative.VolumesSnapshotNegativeTestJSON.test_list_snapshots_invalid_param_sort[id-27b5f37f-bf69-4e8c-986e-c44f3d6819b8,negative] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcpv6_64_subnets[id-4256c61d-c538-41ea-9147-3c450c36669e] +tempest.api.network.test_networks.NetworksIpV6Test.test_create_update_network_description[id-c72c1c0c-2193-4aca-ccc4-b1442640bbbb] +tempest.api.compute.servers.test_server_rescue_negative.ServerRescueNegativeTestJSON.test_rescued_vm_detach_volume[id-f56e465b-fe10-48bf-b75d-646cda3a8bc9,negative,volume] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volumes_list_by_status[id-39654e13-734c-4dab-95ce-7613bf8407ce] +tempest.api.network.test_routers_negative.RoutersNegativeTest.test_show_non_existent_router_returns_404[id-c2a70d72-8826-43a7-8208-0209e6360c47,negative] +tempest.api.compute.admin.test_aggregates.AggregatesAdminTestJSON.test_aggregate_create_update_with_az[id-4d2b2004-40fa-40a1-aab2-66f4dab81beb] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_non_existent_image[id-668743d5-08ad-4480-b2b8-15da34f81d9f,negative] +tempest.api.compute.volumes.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_invalid_size[id-5125ae14-152b-40a7-b3c5-eae15e9022ef,negative] +tempest.api.compute.admin.test_servers.ServersAdminTestJSON.test_list_servers_by_admin_with_all_tenants[id-9f5579ae-19b4-4985-a091-2a5d56106580] +tempest.api.identity.admin.v2.test_roles.RolesTestJSON.test_assign_user_role[id-0146f675-ffbd-4208-b3a4-60eb628dbc5e] +tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON.test_list_images_limit_results[id-3a484ca9-67ba-451e-b494-7fcf28d32d62] +tempest.scenario.test_stamp_pattern.TestStampPattern.test_stamp_pattern[compute,id-10fd234a-515c-41e5-b092-8323060598c5,image,network,slow,volume] +tempest.api.compute.admin.test_hypervisor.HypervisorAdminTestJSON.test_get_hypervisor_list[id-7f0ceacd-c64d-4e96-b8ee-d02943142cc5] +tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsAssociationTestJSON.test_associate_already_associated_floating_ip[id-6edef4b2-aaf1-4abc-bbe3-993e2561e0fe] +tempest.api.image.v2.test_images_negative.ImportImagesNegativeTest.test_image_web_download_import_with_bad_url[id-c52f6a77-f522-4411-8dbe-9d14f2b1ba6b,negative] +tempest.api.compute.admin.test_hypervisor_negative.HypervisorAdminNegativeUnderV252Test.test_show_servers_with_non_admin_user[id-2a0a3938-832e-4859-95bf-1c57c236b924,negative] +tempest.api.identity.admin.v3.test_projects_negative.ProjectsNegativeStaticTestJSON.test_project_create_duplicate[id-874c3e84-d174-4348-a16b-8c01f599561b,negative] +tempest.api.compute.admin.test_aggregates.AggregatesAdminTestJSON.test_aggregate_add_host_create_server_with_az[id-96be03c7-570d-409c-90f8-e4db3c646996] +tempest.scenario.test_security_groups_basic_ops.TestSecurityGroupsBasicOps.test_boot_into_disabled_port_security_network_without_secgroup[compute,id-13ccf253-e5ad-424b-9c4a-97b88a026699,network,slow] +tempest.api.compute.security_groups.test_security_groups.SecurityGroupsTestJSON.test_security_group_create_get_delete[id-ecc0da4a-2117-48af-91af-993cca39a615] +tempest.api.compute.servers.test_disk_config.ServerDiskConfigTestJSON.test_rebuild_server_with_manual_disk_config[id-bef56b09-2e8c-4883-a370-4950812f430e] +tempest.api.object_storage.test_object_services.ObjectTest.test_copy_object_with_x_object_meta[id-edabedca-24c3-4322-9b70-d6d9f942a074] +tempest.api.network.test_networks.NetworksIpV6Test.test_show_subnet_fields[id-270fff0b-8bfc-411f-a184-1e8fd35286f0] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_changes_since_future_date[id-74745ad8-b346-45b5-b9b8-509d7447fc1f,negative] +tempest.api.volume.admin.test_snapshots_actions.SnapshotsActionsTest.test_snapshot_force_delete_when_snapshot_is_creating[id-05f711b6-e629-4895-8103-7ca069f2073a] +tempest.api.compute.servers.test_server_tags.ServerTagsTestJSON.test_delete_all_tags[id-a63b2a74-e918-4b7c-bcab-10c855f3a57e] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_size[id-cf1b9a48-8340-480e-af7b-fe7e17690876] +tempest.api.identity.admin.v3.test_endpoint_groups.EndPointGroupsTest.test_create_list_show_check_delete_endpoint_group[id-7c69e7a1-f865-402d-a2ea-44493017315a] +tempest.api.network.test_routers_negative.RoutersNegativeIpV6Test.test_show_non_existent_router_returns_404[id-c2a70d72-8826-43a7-8208-0209e6360c47,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestMultiTenantJSON.test_update_server_of_another_tenant[id-543d84c1-dd2e-4c6d-8cb2-b9da0efaa384,negative] +tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_update_image[id-f66891a7-a35c-41a8-b590-a065c2a1caa6,smoke] +tempest.api.identity.admin.v2.test_tenants.TenantsTestJSON.test_tenant_list_delete[id-16c6e05c-6112-4b0e-b83f-5e43f221b6b0] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_assign_duplicate_user_role[id-5c3132cd-c4c8-4402-b5ea-71eb44e97793,negative] +tempest.api.identity.admin.v2.test_tenant_negative.TenantsNegativeTestJSON.test_list_tenants_by_unauthorized_user[id-ca9bb202-63dd-4240-8a07-8ef9c19c04bb,negative] +tempest.api.volume.admin.test_volume_types_extra_specs.VolumeTypesExtraSpecsTest.test_volume_type_extra_spec_create_get_delete[id-d4772798-601f-408a-b2a5-29e8a59d1220] +tempest.api.volume.admin.test_volume_types_negative.VolumeTypesNegativeTest.test_get_nonexistent_type_id[id-994610d6-0476-4018-a644-a2602ef5d4aa,negative] +tempest.api.identity.v3.test_tokens.TokensV3Test.test_create_token[id-6f8e4436-fc96-4282-8122-e41df57197a9] +tempest.api.image.v1.test_images.ListImagesTest.test_index_min_size[id-6ffc16d0-4cbf-4401-95c8-4ac63eac34d8] +tempest.api.compute.admin.test_live_migration.LiveMigrationTest.test_live_block_migration_with_attached_volume[id-e19c0cc6-6720-4ed8-be83-b6603ed5c812,volume] +tempest.api.volume.test_volumes_snapshots.VolumesSnapshotTestJSON.test_volume_from_snapshot_no_size[id-053d8870-8282-4fff-9dbb-99cb58bb5e0a] +tempest.api.compute.admin.test_live_migration_negative.LiveMigrationNegativeTest.test_live_block_migration_suspended[id-6e2f94f5-2ee8-4830-bef5-5bc95bb0795b,negative] +tempest.api.compute.admin.test_availability_zone.AZAdminV2TestJSON.test_get_availability_zone_list_detail[id-ef726c58-530f-44c2-968c-c7bed22d5b8c] +tempest.api.compute.servers.test_server_group.ServerGroupTestJSON.test_create_delete_server_group_with_anti_affinity_policy[id-3645a102-372f-4140-afad-13698d850d23] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_authentication_for_disabled_user[id-593a4981-f6d4-460a-99a1-57a78bf20829,negative] +tempest.api.network.admin.test_routers.RoutersIpV6AdminTest.test_update_router_set_gateway_with_snat_explicit[id-b386c111-3b21-466d-880c-5e72b01e1a33] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_update_host_with_invalid_status[id-fbe2bf3e-3246-4a95-a59f-94e4e298ec77,negative] +tempest.api.volume.test_volumes_get.VolumesSummaryTest.test_show_volume_summary[id-c4f2431e-4920-4736-9e00-4040386b6feb] +tempest.api.identity.admin.v3.test_users_negative.UsersNegativeTest.test_authentication_for_disabled_user[id-b3c9fccc-4134-46f5-b600-1da6fb0a3b1f,negative] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_assign_user_role_request_without_token[id-f0d2683c-5603-4aee-95d7-21420e87cfd8,negative] +tempest.api.network.test_routers_negative.RoutersNegativeIpV6Test.test_update_non_existent_router_returns_404[id-b23d1569-8b0c-4169-8d4b-6abd34fad5c7,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestMultiTenantJSON.test_delete_a_server_of_another_tenant[id-5c75009d-3eea-423e-bea3-61b09fd25f9c,negative] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_security_group_with_invalid_sg_name[id-cda8d8b4-59f8-4087-821d-20cf5a03b3b1,negative] +tempest.api.compute.servers.test_server_rescue_negative.ServerRescueNegativeTestJSON.test_rescue_non_existent_server[id-6dfc0a55-3a77-4564-a144-1587b7971dde,negative] +tempest.api.compute.keypairs.test_keypairs_negative.KeyPairsNegativeTestJSON.test_create_keypair_with_empty_name_string[id-1398abe1-4a84-45fb-9294-89f514daff00,negative] +tempest.api.compute.admin.test_delete_server.DeleteServersAdminTestJSON.test_admin_delete_servers_of_others[id-73177903-6737-4f27-a60c-379e8ae8cf48] +tempest.api.image.v2.test_versions.VersionsTest.test_list_versions[id-659ea30a-a17c-4317-832c-0f68ed23c31d,smoke] +tempest.api.image.v2.test_images_member.ImagesMemberTest.test_get_image_member_schema[id-634dcc3f-f6e2-4409-b8fd-354a0bb25d83] +tempest.api.compute.servers.test_disk_config.ServerDiskConfigTestJSON.test_resize_server_from_auto_to_manual[id-693d16f3-556c-489a-8bac-3d0ca2490bad] +tempest.api.volume.admin.test_volumes_actions.VolumesActionsTest.test_volume_reset_status[id-d063f96e-a2e0-4f34-8b8a-395c42de1845] +tempest.api.compute.test_networks.ComputeNetworksTest.test_list_networks[id-3fe07175-312e-49a5-a623-5f52eeada4c2] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_port_list_filter_by_ip[id-e7fe260b-1e79-4dd3-86d9-bec6a7959fc5] +tempest.api.network.test_networks.NetworksTest.test_external_network_visibility[id-af774677-42a9-4e4b-bb58-16fe6a5bc1ec,smoke] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_sort_id_desc[id-8a7fe058-0b41-402a-8afd-2dbc5a4a718b] +tempest.api.compute.admin.test_server_diagnostics_negative.ServerDiagnosticsNegativeTest.test_get_server_diagnostics_by_non_admin[id-e84e2234-60d2-42fa-8b30-e2d3049724ac,negative] +tempest.api.network.admin.test_ports.PortsAdminExtendedAttrsIpV6TestJSON.test_update_port_binding_ext_attr[compute,id-6f6c412c-711f-444d-8502-0ac30fbf5dd5] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_rebuild_deleted_server[id-98fa0458-1485-440f-873b-fe7f0d714930,negative] +tempest.api.network.test_networks.NetworksIpV6Test.test_create_delete_subnet_with_dhcp_enabled[id-94ce038d-ff0a-4a4c-a56b-09da3ca0b55d] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_register_with_invalid_container_format[id-292bd310-369b-41c7-a7a3-10276ef76753,negative] +tempest.api.network.test_security_groups.SecGroupIPv6Test.test_create_security_group_rule_with_remote_ip_prefix[id-16459776-5da2-4634-bce4-4b55ee3ec188] +tempest.api.image.v2.admin.test_images_metadefs_namespace_properties.MetadataNamespacePropertiesTest.test_basic_meta_def_namespace_property[id-b1a3765e-3a5d-4f6d-a3a7-3ca3476ae768] +tempest.api.network.test_networks.NetworksIpV6Test.test_create_update_delete_network_subnet[id-0e269138-0da6-4efc-a46d-578161e7b221,smoke] +tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsTestJSON.test_delete_floating_ip[id-de45e989-b5ca-4a9b-916b-04a52e7bbb8b] +tempest.api.identity.admin.v2.test_tenant_negative.TenantsNegativeTestJSON.test_tenant_delete_by_unauthorized_user[id-162ba316-f18b-4987-8c0c-fd9140cd63ed,negative] +tempest.api.identity.admin.v2.test_tokens.TokensTestJSON.test_list_endpoints_for_token[id-ca3ea6f7-ed08-4a61-adbd-96906456ad31] +tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_pause_unpause[compute,id-2b2642db-6568-4b35-b812-eceed3fa20ce,network,slow] +tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON.test_list_images_with_detail_filter_by_server_ref[id-8c78f822-203b-4bf6-8bba-56ebd551cf84] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_get_console_output_with_unlimited_size[id-89104062-69d8-4b19-a71b-f47b7af093d7] +tempest.api.compute.servers.test_create_server_multi_nic.ServersTestMultiNic.test_verify_duplicate_network_nics[id-1678d144-ed74-43f8-8e57-ab10dbf9b3c2] +tempest.api.compute.servers.test_servers.ServerShowV247Test.test_update_rebuild_list_server[id-8de397c2-57d0-4b90-aa30-e5d668f21a8b] diff --git a/.automation.conf/tempest/skip-lists/ci-multinode b/.automation.conf/tempest/skip-lists/ci-multinode new file mode 100644 index 000000000..94c494687 --- /dev/null +++ b/.automation.conf/tempest/skip-lists/ci-multinode @@ -0,0 +1 @@ +tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_subnet_details.*: "Cirros image doesn't have '/var/run/udhcpc.eth0.pid" diff --git a/.automation.conf/tempest/tempest-ci-multinode.overrides.conf b/.automation.conf/tempest/tempest-ci-multinode.overrides.conf new file mode 100644 index 000000000..5cd61d03b --- /dev/null +++ b/.automation.conf/tempest/tempest-ci-multinode.overrides.conf @@ -0,0 +1,30 @@ +[DEFAULT] +openstack_client_http_timeout = 300 + +[auth] +compute_quotas = cores:-1,ram:-1 + +[identity] +v3_endpoint_type = publicURL + +[compute] +min_compute_nodes = 2 +min_microversion = 2.1 +max_microversion = 2.88 + +[service-clients] +http_timeout = 600 + +[compute-feature-enabled] +resize = true +live_migration = true +block_migration_for_live_migration = false +volume_backed_live_migration = true +console_output = true + +[volume] +min_microversion = 3.0 +max_microversion = 3.66 + +[dashboard] +dashboard_url = http://192.168.39.2 diff --git a/etc/kayobe/ansible/configure-vxlan.yml b/etc/kayobe/ansible/configure-vxlan.yml index f07933c1b..b43c8b65d 100644 --- a/etc/kayobe/ansible/configure-vxlan.yml +++ b/etc/kayobe/ansible/configure-vxlan.yml @@ -1,6 +1,6 @@ --- - name: Configure VXLAN - hosts: storage,seed,controllers,compute + hosts: controllers,compute,seed,storage gather_facts: false vars: ansible_user: "{{ bootstrap_user }}" @@ -11,10 +11,3 @@ ansible_ssh_common_args: "-o StrictHostKeyChecking=no" roles: - role: stackhpc.vxlan - vars: - vxlan_phys_dev: "{{ admin_oc_net_name | net_interface }}" - vxlan_dstport: 4790 - vxlan_interfaces: - - device: vxlan10 - group: 224.0.0.10 - bridge: breth1 diff --git a/etc/kayobe/ansible/fix-hostname.yml b/etc/kayobe/ansible/fix-hostname.yml new file mode 100644 index 000000000..92730b7f0 --- /dev/null +++ b/etc/kayobe/ansible/fix-hostname.yml @@ -0,0 +1,23 @@ +--- +- name: Fix hostname on storage nodes for cephadm + hosts: storage + gather_facts: false + vars: + ansible_user: "{{ bootstrap_user }}" + # We can't assume that a virtualenv exists at this point, so use the system + # python interpreter. + ansible_python_interpreter: /usr/bin/python3 + # Work around no known_hosts entry on first boot. + ansible_ssh_common_args: "-o StrictHostKeyChecking=no" + tasks: + - name: Get current hostname + ansible.builtin.command: + cmd: hostname + changed_when: false + register: current_hostname + + - name: Set hostname to a non FQDN + ansible.builtin.command: + cmd: hostnamectl set-hostname "{{ inventory_hostname }}" + when: current_hostname.stdout != inventory_hostname + become: true diff --git a/etc/kayobe/ansible/fix-networking.yml b/etc/kayobe/ansible/fix-networking.yml new file mode 100644 index 000000000..f24be3862 --- /dev/null +++ b/etc/kayobe/ansible/fix-networking.yml @@ -0,0 +1,21 @@ +--- +- name: Fix networking + hosts: seed,compute,controllers,storage + gather_facts: false + vars: + ansible_user: "{{ bootstrap_user }}" + # We can't assume that a virtualenv exists at this point, so use the system + # python interpreter. + ansible_python_interpreter: /usr/bin/python3 + # Work around no known_hosts entry on first boot. + ansible_ssh_common_args: "-o StrictHostKeyChecking=no" + tasks: + - name: Ensure `hosts` file contains pulp entries + ansible.builtin.copy: + content: | + 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 + ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + 10.0.0.34 pelican pelican.service.compute.sms-lab.cloud + 10.205.3.187 pulp-server pulp-server.internal.sms-cloud + dest: /etc/hosts + become: true diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index 77aaa9f99..ad3c290dc 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -1,7 +1,7 @@ --- collections: - name: stackhpc.cephadm - version: 1.10.0 + version: 1.12.2 - name: stackhpc.pulp version: 0.4.1 roles: diff --git a/etc/kayobe/environments/ci-multinode/compute.yml b/etc/kayobe/environments/ci-multinode/compute.yml index 74e7ef86c..7e7701cf0 100644 --- a/etc/kayobe/environments/ci-multinode/compute.yml +++ b/etc/kayobe/environments/ci-multinode/compute.yml @@ -1,4 +1,5 @@ --- +compute_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' else 'cloud-user' }}" # List of storage volume groups. See mrlesmithjr.manage-lvm role for # format. compute_lvm_groups: diff --git a/etc/kayobe/environments/ci-multinode/controllers.yml b/etc/kayobe/environments/ci-multinode/controllers.yml index 3f612ecd8..73c31c27f 100644 --- a/etc/kayobe/environments/ci-multinode/controllers.yml +++ b/etc/kayobe/environments/ci-multinode/controllers.yml @@ -1,4 +1,5 @@ --- +controller_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' else 'cloud-user' }}" # List of storage volume groups. See mrlesmithjr.manage-lvm role for # format. controller_lvm_groups: diff --git a/etc/kayobe/environments/ci-multinode/growroot.yml b/etc/kayobe/environments/ci-multinode/growroot.yml new file mode 100644 index 000000000..1a0c767ac --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/growroot.yml @@ -0,0 +1,2 @@ +--- +growroot_group: "seed:overcloud" diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/all/main.yml b/etc/kayobe/environments/ci-multinode/inventory/group_vars/all/main.yml new file mode 100644 index 000000000..e26394f26 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/all/main.yml @@ -0,0 +1,2 @@ +--- +cephadm_host_labels: [] diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/all/vxlan.yml b/etc/kayobe/environments/ci-multinode/inventory/group_vars/all/vxlan.yml new file mode 100644 index 000000000..808c78034 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/all/vxlan.yml @@ -0,0 +1,18 @@ +--- +############################################################################### +# VXLAN configuration. + +# These variables are used by stackhpc.vxlan to construct a VXLAN interface. +# The physical device that the VXLAN is plugged into. +vxlan_phys_dev: "{{ admin_oc_interface }}" + +# Destination port used by the VXLAN interface. Note: OpenStack uses 4789. +vxlan_dstport: 4790 + +# Virtual network identifier used by the VLXAN. This must be unique between multinode environments. +vxlan_vni: + +# This is a dictionary of interfaces to be created by the stackhpc.vxlan role. +vxlan_interfaces: + - device: "vxlan{{ vxlan_vni }}" + group: "{{ '239.0.0.0/8' | next_nth_usable(vxlan_vni) }}" diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/lvm.yml b/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/lvm.yml new file mode 100644 index 000000000..a46fa96d3 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/lvm.yml @@ -0,0 +1,34 @@ +--- +# List of extra LVs to include in the rootvg VG. +stackhpc_lvm_group_rootvg_lvs_extra: + - "{{ stackhpc_lvm_lv_docker }}" + +############################################################################### +# StackHPC LVM Logical Volume (LV) configuration. + +# StackHPC LVM lv_swap LV size. +stackhpc_lvm_lv_swap_size: 1g + +# StackHPC LVM lv_root LV size. +stackhpc_lvm_lv_root_size: 10g + +# StackHPC LVM lv_tmp LV size. +stackhpc_lvm_lv_tmp_size: 10g + +# StackHPC LVM lv_var LV size. +stackhpc_lvm_lv_var_size: 20g + +# StackHPC LVM lv_var_tmp LV size. +stackhpc_lvm_lv_var_tmp_size: 5g + +# StackHPC LVM lv_log LV size. +stackhpc_lvm_lv_log_size: 10g + +# StackHPC LVM lv_audit LV size. +stackhpc_lvm_lv_audit_size: 5g + +# StackHPC LVM lv_home LV size. +stackhpc_lvm_lv_home_size: 5g + +# StackHPC LVM lv_docker LV size. +stackhpc_lvm_lv_docker_size: 75%FREE diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces index 2e94b6c2d..aab2482b8 100644 --- a/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces @@ -1,13 +1,14 @@ --- ############################################################################### -# Network interface definitions for the controller group. +# Network interface definitions for the compute group. -admin_interface: eno1 +admin_oc_interface: "eno1" -# Controller interface on all-in-one network. -aio_interface: breth1 -# Use dummy1 if it exists, otherwise the bridge will have no ports. -aio_bridge_ports: [] +internal_interface: "{{ vxlan_interfaces[0].device }}.{{ internal_vlan }}" + +storage_interface: "{{ vxlan_interfaces[0].device }}.{{ storage_vlan }}" + +tunnel_interface: "{{ vxlan_interfaces[0].device }}.{{ tunnel_vlan }}" ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/lvm.yml b/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/lvm.yml new file mode 100644 index 000000000..a46fa96d3 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/lvm.yml @@ -0,0 +1,34 @@ +--- +# List of extra LVs to include in the rootvg VG. +stackhpc_lvm_group_rootvg_lvs_extra: + - "{{ stackhpc_lvm_lv_docker }}" + +############################################################################### +# StackHPC LVM Logical Volume (LV) configuration. + +# StackHPC LVM lv_swap LV size. +stackhpc_lvm_lv_swap_size: 1g + +# StackHPC LVM lv_root LV size. +stackhpc_lvm_lv_root_size: 10g + +# StackHPC LVM lv_tmp LV size. +stackhpc_lvm_lv_tmp_size: 10g + +# StackHPC LVM lv_var LV size. +stackhpc_lvm_lv_var_size: 20g + +# StackHPC LVM lv_var_tmp LV size. +stackhpc_lvm_lv_var_tmp_size: 5g + +# StackHPC LVM lv_log LV size. +stackhpc_lvm_lv_log_size: 10g + +# StackHPC LVM lv_audit LV size. +stackhpc_lvm_lv_audit_size: 5g + +# StackHPC LVM lv_home LV size. +stackhpc_lvm_lv_home_size: 5g + +# StackHPC LVM lv_docker LV size. +stackhpc_lvm_lv_docker_size: 75%FREE diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces index 2e94b6c2d..2dab6c5a8 100644 --- a/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces @@ -2,12 +2,15 @@ ############################################################################### # Network interface definitions for the controller group. -admin_interface: eno1 +admin_oc_interface: "eno1" -# Controller interface on all-in-one network. -aio_interface: breth1 -# Use dummy1 if it exists, otherwise the bridge will have no ports. -aio_bridge_ports: [] +internal_interface: "{{ vxlan_interfaces[0].device }}.{{ internal_vlan }}" + +storage_interface: "{{ vxlan_interfaces[0].device }}.{{ storage_vlan }}" + +tunnel_interface: "{{ vxlan_interfaces[0].device }}.{{ tunnel_vlan }}" + +public_interface: "{{ vxlan_interfaces[0].device }}.{{ public_vlan }}" ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/lvm.yml b/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/lvm.yml new file mode 100644 index 000000000..a46fa96d3 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/lvm.yml @@ -0,0 +1,34 @@ +--- +# List of extra LVs to include in the rootvg VG. +stackhpc_lvm_group_rootvg_lvs_extra: + - "{{ stackhpc_lvm_lv_docker }}" + +############################################################################### +# StackHPC LVM Logical Volume (LV) configuration. + +# StackHPC LVM lv_swap LV size. +stackhpc_lvm_lv_swap_size: 1g + +# StackHPC LVM lv_root LV size. +stackhpc_lvm_lv_root_size: 10g + +# StackHPC LVM lv_tmp LV size. +stackhpc_lvm_lv_tmp_size: 10g + +# StackHPC LVM lv_var LV size. +stackhpc_lvm_lv_var_size: 20g + +# StackHPC LVM lv_var_tmp LV size. +stackhpc_lvm_lv_var_tmp_size: 5g + +# StackHPC LVM lv_log LV size. +stackhpc_lvm_lv_log_size: 10g + +# StackHPC LVM lv_audit LV size. +stackhpc_lvm_lv_audit_size: 5g + +# StackHPC LVM lv_home LV size. +stackhpc_lvm_lv_home_size: 5g + +# StackHPC LVM lv_docker LV size. +stackhpc_lvm_lv_docker_size: 75%FREE diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces index 13f7aa8ea..0c868b110 100644 --- a/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces @@ -1,13 +1,15 @@ --- ############################################################################### -# Network interface definitions for the controller group. +# Network interface definitions for the seed group. -admin_interface: eth0 +admin_oc_interface: "{{ 'ens2' if os_distribution == 'ubuntu' else 'ens3' }}" + +provision_oc_interface: "{{ vxlan_interfaces[0].device}}.{{ provision_oc_vlan }}" + +external_interface: "{{ vxlan_interfaces[0].device }}.{{ external_vlan }}" + +public_interface: "{{ vxlan_interfaces[0].device }}.{{ public_vlan }}" -# Controller interface on all-in-one network. -aio_interface: breth1 -# Use dummy1 if it exists, otherwise the bridge will have no ports. -aio_bridge_ports: [] ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/lvm b/etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/lvm.yml similarity index 60% rename from etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/lvm rename to etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/lvm.yml index 03b356a9e..f918a0360 100644 --- a/etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/lvm +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/lvm.yml @@ -1,3 +1,11 @@ +--- +# List of extra LVs to include in the rootvg VG. +stackhpc_lvm_group_rootvg_lvs_extra: + - "{{ stackhpc_lvm_lv_docker }}" + +############################################################################### +# StackHPC LVM Logical Volume (LV) configuration. + # StackHPC LVM lv_swap LV size. stackhpc_lvm_lv_swap_size: 1g @@ -21,3 +29,6 @@ stackhpc_lvm_lv_audit_size: 1g # StackHPC LVM lv_home LV size. stackhpc_lvm_lv_home_size: 2g + +# StackHPC LVM lv_docker LV size. +stackhpc_lvm_lv_docker_size: 75%FREE diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/network-interfaces index 35000577f..00ea518f5 100644 --- a/etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/network-interfaces +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/network-interfaces @@ -1,13 +1,14 @@ --- ############################################################################### -# Network interface definitions for the stroage group. +# Network interface definitions for the storage group. -admin_interface: ens3 +admin_oc_interface: "ens3" -# Controller interface on all-in-one network. -aio_interface: breth1 -# Use dummy1 if it exists, otherwise the bridge will have no ports. -aio_bridge_ports: [] +internal_interface: "{{ vxlan_interfaces[0].device }}.{{ internal_vlan }}" + +storage_interface: "{{ vxlan_interfaces[0].device }}.{{ storage_vlan }}" + +storage_mgmt_interface: "{{ vxlan_interfaces[0].device }}.{{ storage_mgmt_vlan }}" ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/etc/kayobe/environments/ci-multinode/inventory/kayobe-automation b/etc/kayobe/environments/ci-multinode/inventory/kayobe-automation index 5ac95a0d2..7add0ce97 100644 --- a/etc/kayobe/environments/ci-multinode/inventory/kayobe-automation +++ b/etc/kayobe/environments/ci-multinode/inventory/kayobe-automation @@ -1,2 +1,2 @@ -[tempest_runner] -kayobe-controller-01 +[tempest_runner:children] +seed diff --git a/etc/kayobe/environments/ci-multinode/kolla.yml b/etc/kayobe/environments/ci-multinode/kolla.yml index 168500c8b..e645b971c 100644 --- a/etc/kayobe/environments/ci-multinode/kolla.yml +++ b/etc/kayobe/environments/ci-multinode/kolla.yml @@ -1,3 +1,15 @@ --- kolla_enable_cinder: true kolla_enable_cinder_backup: true +kolla_enable_neutron_provider_networks: true +kolla_enable_ovn: true +kolla_enable_octavia: true + +# Override these from etc/kayobe/kolla.yml +# These are set to true within etc/kayobe/kolla.yml. +# Leaving them set to true would require additional +# configuration change. +kolla_enable_central_logging: false +kolla_enable_elasticsearch_curator: false +kolla_enable_grafana: false +kolla_enable_prometheus: false diff --git a/etc/kayobe/environments/ci-multinode/kolla/config/keepalived/keepalived.conf b/etc/kayobe/environments/ci-multinode/kolla/config/keepalived/keepalived.conf new file mode 100644 index 000000000..7a18dbbcf --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/kolla/config/keepalived/keepalived.conf @@ -0,0 +1,48 @@ +{% raw %} +vrrp_script check_alive { + script "/check_alive.sh" + interval 2 + fall 2 + rise 10 +} + +vrrp_instance kolla_internal_vip_{{ keepalived_virtual_router_id }} { + state BACKUP + nopreempt + interface {{ api_interface }} + virtual_router_id {{ keepalived_virtual_router_id }} + priority {{ groups['loadbalancer'].index(inventory_hostname) + 1 }} + advert_int 3 # Increase advert interval from 1 to 3 as there was a lot of jumping amongst controllers on the internal network +{% if keepalived_traffic_mode == 'unicast' %} + unicast_src_ip {{ api_interface_address }} +{% if groups['loadbalancer'] | length > 1 %} + unicast_peer { +{% for host in groups['loadbalancer'] %} +{% set ip_addr = 'api' | kolla_address(host) %} +{% if ip_addr != api_interface_address %} + {{ ip_addr }} +{% endif %} +{% endfor %} + } +{% endif %} +{% endif %} + virtual_ipaddress { + {{ kolla_internal_vip_address }} dev {{ api_interface }} +{% if haproxy_enable_external_vip | bool %} + {{ kolla_external_vip_address }} dev {{ kolla_external_vip_interface }} +{% endif %} + } +{% if haproxy_enable_external_vip | bool and api_interface != kolla_external_vip_interface %} + track_interface { + {{ kolla_external_vip_interface }} + } +{% endif %} + authentication { + auth_type PASS + auth_pass {{ keepalived_password }} + } + track_script { + check_alive + } +} +{% endraw %} diff --git a/etc/kayobe/environments/ci-multinode/kolla/globals.yml b/etc/kayobe/environments/ci-multinode/kolla/globals.yml index e7afedf68..8d020e349 100644 --- a/etc/kayobe/environments/ci-multinode/kolla/globals.yml +++ b/etc/kayobe/environments/ci-multinode/kolla/globals.yml @@ -7,3 +7,15 @@ cinder_backend_ceph: "yes" # Nova Ceph configuration nova_backend_ceph: "yes" + +# Because this is a vxlan interface, Kayobe can't generate the config for it. +# Therefore we can't use this interface as the external network +# (external_net_names). Instead, make external_net_names an empty +# list, and add the config that would be generated from it here. +neutron_bridge_name: "{{ vxlan_interfaces[0].device }}-ovs" +neutron_external_interface: "{{ vxlan_interfaces[0].device }}" + +# Octavia load balancer configuration +octavia_auto_configure: "no" +octavia_provider_drivers: "ovn:OVN provider" +octavia_provider_agents: "ovn" diff --git a/etc/kayobe/environments/ci-multinode/networks.yml b/etc/kayobe/environments/ci-multinode/networks.yml index a42719638..86406bfad 100644 --- a/etc/kayobe/environments/ci-multinode/networks.yml +++ b/etc/kayobe/environments/ci-multinode/networks.yml @@ -8,31 +8,29 @@ # Name of the network used for admin access to the overcloud #admin_oc_net_name: -admin_oc_net_name: admin +admin_oc_net_name: admin_oc # Name of the network used by the seed to manage the bare metal overcloud # hosts via their out-of-band management controllers. #oob_oc_net_name: -oob_oc_net_name: aio +oob_oc_net_name: # Name of the network used by the seed to provision the bare metal overcloud # hosts. #provision_oc_net_name: -provision_oc_net_name: aio +provision_oc_net_name: provision_oc # Name of the network used by the overcloud hosts to manage the bare metal # compute hosts via their out-of-band management controllers. #oob_wl_net_name: -oob_wl_net_name: aio # Name of the network used by the overcloud hosts to provision the bare metal # workload hosts. #provision_wl_net_name: -provision_wl_net_name: aio # Name of the network used to expose the internal OpenStack API endpoints. #internal_net_name: -internal_net_name: aio +internal_net_name: internal # List of names of networks used to provide external network access via # Neutron. @@ -40,55 +38,89 @@ internal_net_name: aio # If external_net_name is defined, external_net_names will default to a list # containing one item, external_net_name. #external_net_names: -external_net_names: - - aio +external_net_names: [] # Name of the network used to expose the public OpenStack API endpoints. -#public_net_name: -public_net_name: aio +public_net_name: public # Name of the network used by Neutron to carry tenant overlay network traffic. -#tunnel_net_name: -tunnel_net_name: aio +tunnel_net_name: tunnel # Name of the network used to carry storage data traffic. #storage_net_name: -storage_net_name: aio +storage_net_name: storage # Name of the network used to carry storage management traffic. #storage_mgmt_net_name: -storage_mgmt_net_name: aio +storage_mgmt_net_name: storage_mgmt # Name of the network used to carry swift storage data traffic. #swift_storage_net_name: -swift_storage_net_name: aio # Name of the network used to carry swift storage replication traffic. #swift_storage_replication_net_name: -swift_storage_replication_net_name: aio # Name of the network used to perform hardware introspection on the bare metal # workload hosts. #inspection_net_name: -inspection_net_name: aio # Name of the network used to perform cleaning on the bare metal workload # hosts #cleaning_net_name: -cleaning_net_name: aio ############################################################################### # Network definitions. -# All-in-one network. -aio_cidr: 192.168.33.0/24 -aio_allocation_pool_start: 192.168.33.3 -aio_allocation_pool_end: 192.168.33.30 -aio_neutron_allocation_pool_start: 192.168.33.31 -aio_neutron_allocation_pool_end: 192.168.33.127 -aio_inspection_allocation_pool_start: 192.168.33.128 -aio_inspection_allocation_pool_end: 192.168.33.254 -aio_vip_address: 192.168.33.2 +# Internal network +internal_cidr: 192.168.37.0/24 +internal_mtu: 1450 +internal_allocation_pool_start: 192.168.37.3 +internal_allocation_pool_end: 192.168.37.254 +internal_vip_address: 192.168.37.2 +internal_vlan: 101 + +# External network +external_cidr: 192.168.38.0/24 +external_mtu: 1350 +external_allocation_pool_start: 192.168.38.3 +external_allocation_pool_end: 192.168.38.128 +external_vlan: 102 + +# Public network +public_cidr: 192.168.39.0/24 +public_mtu: 1450 +public_allocation_pool_start: 192.168.39.3 +public_allocation_pool_end: 192.168.39.254 +public_vip_address: 192.168.39.2 +public_vlan: 103 + +# Tunnel network +tunnel_cidr: 192.168.40.0/24 +tunnel_mtu: 1450 +tunnel_allocation_pool_start: 192.168.40.3 +tunnel_allocation_pool_end: 192.168.40.254 +tunnel_vlan: 104 + +# Storage network +storage_cidr: 192.168.41.0/24 +storage_mtu: 1450 +storage_allocation_pool_start: 192.168.41.3 +storage_allocation_pool_end: 192.168.41.254 +storage_vlan: 105 + +# Storage management network +storage_mgmt_cidr: 192.168.42.0/24 +storage_mgmt_mtu: 1450 +storage_mgmt_allocation_pool_start: 192.168.42.3 +storage_mgmt_allocation_pool_end: 192.168.42.254 +storage_mgmt_vlan: 106 + +# Provision overcloud network +provision_oc_cidr: 192.168.33.0/24 +provision_oc_mtu: 1450 +provision_oc_allocation_pool_start: 192.168.33.128 +provision_oc_allocation_pool_end: 192.168.33.254 +provision_oc_vlan: 107 ############################################################################### # Network virtual patch link configuration. diff --git a/etc/kayobe/environments/ci-multinode/neutron.yml b/etc/kayobe/environments/ci-multinode/neutron.yml new file mode 100644 index 000000000..1a7fd8255 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/neutron.yml @@ -0,0 +1,3 @@ +--- +kolla_neutron_ml2_network_vlan_ranges: + - physical_network: "physnet1" diff --git a/etc/kayobe/environments/ci-multinode/seed.yml b/etc/kayobe/environments/ci-multinode/seed.yml new file mode 100644 index 000000000..1ae71f1a4 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/seed.yml @@ -0,0 +1,10 @@ +--- +seed_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' else 'cloud-user' }}" +seed_lvm_groups: + - "{{ stackhpc_lvm_group_rootvg }}" + +# Seed has been provided an external interface +# for tempest tests and SSH access to machines. +seed_extra_network_interfaces: + - "external" + - "public" diff --git a/etc/kayobe/environments/ci-multinode/storage.yml b/etc/kayobe/environments/ci-multinode/storage.yml index 44b655ce7..b152af472 100644 --- a/etc/kayobe/environments/ci-multinode/storage.yml +++ b/etc/kayobe/environments/ci-multinode/storage.yml @@ -1,4 +1,5 @@ --- +storage_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' else 'cloud-user' }}" # List of storage volume groups. See mrlesmithjr.manage-lvm role for # format. storage_lvm_groups: diff --git a/releasenotes/notes/cephadm-bump-294148e93121da48.yaml b/releasenotes/notes/cephadm-bump-294148e93121da48.yaml new file mode 100644 index 000000000..82f938582 --- /dev/null +++ b/releasenotes/notes/cephadm-bump-294148e93121da48.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + Increase `stackhpc.cephadm` collection to version `1.12.2`. From 347e8892f398038062de42baca490b08e1bff672 Mon Sep 17 00:00:00 2001 From: Jakub Darmach Date: Tue, 31 Jan 2023 16:36:23 +0100 Subject: [PATCH 196/292] MAgnum bump --- etc/kayobe/kolla/globals.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 3cb843778..d41270440 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -9,13 +9,13 @@ enable_docker_repo: {% raw %}"{{ 'overcloud' not in group_names or ansible_facts {% if kolla_base_distro == 'centos' %} bifrost_tag: xena-20221128T101757 -magnum_tag: xena-20230111T093652 +magnum_tag: xena-20230131T144233 nova_tag: xena-20230120T115956 rabbitmq_tag: xena-20230105T141714 {% else %} bifrost_tag: xena-20221213T224057 kolla_toolbox_tag: xena-20230104T145414 -magnum_tag: xena-20230111T095751 +magnum_tag: xena-20230131T151601 neutron_tag: xena-20230104T145414 neutron_tls_proxy_tag: "{% raw %}{{ openstack_tag }}{% endraw %}" nova_tag: xena-20230104T145414 From f89768cfad7e3dcec605083310567dd0ce476061 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 31 Jan 2023 13:30:37 +0000 Subject: [PATCH 197/292] Add container image tags for etcd --- etc/kayobe/kolla/globals.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 77c2252e6..1d7f7dd69 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -9,6 +9,7 @@ enable_docker_repo: {% raw %}"{{ 'overcloud' not in group_names or ansible_facts {% if kolla_base_distro == 'centos' %} bifrost_tag: xena-20221128T101757 +etcd_tag: xena-20230201T095253 glance_tls_proxy_tag: "{% raw %}{{ haproxy_tag | default(openstack_tag) }}{% endraw %}" magnum_tag: xena-20230131T144233 neutron_tls_proxy_tag: "{% raw %}{{ haproxy_tag | default(openstack_tag) }}{% endraw %}" @@ -16,6 +17,7 @@ nova_tag: xena-20230120T115956 rabbitmq_tag: xena-20230105T141714 {% else %} bifrost_tag: xena-20221213T224057 +etcd_tag: xena-20230201T131415 glance_tls_proxy_tag: "{% raw %}{{ haproxy_tag | default(openstack_tag) }}{% endraw %}" kolla_toolbox_tag: xena-20230104T145414 magnum_tag: xena-20230131T151601 From 83542bb91f45c943ea7dbc920a285c08163ee899 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 3 Feb 2023 16:39:03 +0100 Subject: [PATCH 198/292] Fix description of MySQL alert --- etc/kayobe/kolla/config/prometheus/mysql.rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/kolla/config/prometheus/mysql.rules b/etc/kayobe/kolla/config/prometheus/mysql.rules index 26220ae8b..86134207b 100644 --- a/etc/kayobe/kolla/config/prometheus/mysql.rules +++ b/etc/kayobe/kolla/config/prometheus/mysql.rules @@ -58,7 +58,7 @@ groups: severity: warning annotations: summary: "MySQL Slave replication lag (instance {{ $labels.instance }})" - description: "MysqL replication lag on {{ $labels.instance }}" + description: "MySQL replication lag on {{ $labels.instance }}" - alert: MysqlSlowQueries expr: increase(mysql_global_status_slow_queries[1m]) > 0 From bb7432adf8444f26a19b29772bfde11e1e225f34 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 3 Feb 2023 17:49:31 +0100 Subject: [PATCH 199/292] Synchronise with kayobe stable/xena Change-Id: I0bad3065139a421f95b16a71ccab1b530180ef7e --- etc/kayobe/apt.yml | 35 +++++++++++++++++++++++++++++++++++ etc/kayobe/dnf.yml | 8 +++++++- etc/kayobe/globals.yml | 7 ++++--- etc/kayobe/infra-vms.yml | 5 ++++- etc/kayobe/kolla.yml | 6 ++++-- etc/kayobe/seed-vm.yml | 7 +++++-- etc/kayobe/seed.yml | 1 - etc/kayobe/time.yml | 3 +++ 8 files changed, 62 insertions(+), 10 deletions(-) diff --git a/etc/kayobe/apt.yml b/etc/kayobe/apt.yml index 5f278e322..34bfdd2ef 100644 --- a/etc/kayobe/apt.yml +++ b/etc/kayobe/apt.yml @@ -11,6 +11,41 @@ # Apt proxy URL for HTTPS. Default is {{ apt_proxy_http }}. #apt_proxy_https: +# List of Apt configuration options. Each item is a dict with the following +# keys: +# * content: free-form configuration file content +# * filename: name of a file in /etc/apt/apt.conf.d/ in which to write the +# configuration +# Default is an empty list. +#apt_config: + +# List of apt keys. Each item is a dict containing the following keys: +# * url: URL of key +# * filename: Name of a file in which to store the downloaded key. The +# extension should be '.asc' for ASCII-armoured keys, or '.gpg' otherwise. +# Default is an empty list. +#apt_keys: + +# A list of Apt repositories. Each item is a dict with the following keys: +# * types: whitespace-separated list of repository types, e.g. deb or deb-src +# (optional, default is 'deb') +# * url: URL of the repository +# * suites: whitespace-separated list of suites, e.g. focal (optional, default +# is ansible_facts.distribution_release) +# * components: whitespace-separated list of components, e.g. main (optional, +# default is 'main') +# * signed_by: whitespace-separated list of names of GPG keyring files in +# apt_keys_path (optional, default is unset) +# * architecture: whitespace-separated list of architectures that will be used +# (optional, default is unset) +# Default is an empty list. +#apt_repositories: + +# Whether to disable repositories in /etc/apt/sources.list. This may be used +# when replacing the distribution repositories via apt_repositories. +# Default is false. +#apt_disable_sources_list: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/dnf.yml b/etc/kayobe/dnf.yml index cf09b86fd..ab28060e7 100644 --- a/etc/kayobe/dnf.yml +++ b/etc/kayobe/dnf.yml @@ -12,12 +12,18 @@ # Whether or not to use a local Yum mirror. Default value is 'false'. #dnf_use_local_mirror: -# Mirror FQDN for Yum repos. Default value is 'mirror.centos.org'. +# Mirror FQDN for Yum CentOS repos. Default value is 'mirror.centos.org'. #dnf_centos_mirror_host: # Mirror directory for Yum CentOS repos. Default value is 'centos'. #dnf_centos_mirror_directory: +# Mirror FQDN for Yum Rocky repos. Default value is 'dl.rockylinux.org'. +#dnf_rocky_mirror_host: + +# Mirror directory for Yum Rocky repos. Default value is 'pub/rocky'. +#dnf_rocky_mirror_directory: + # Mirror FQDN for Yum EPEL repos. Default value is # 'download.fedoraproject.org'. #dnf_epel_mirror_host: diff --git a/etc/kayobe/globals.yml b/etc/kayobe/globals.yml index a4150d8ec..b926fc9be 100644 --- a/etc/kayobe/globals.yml +++ b/etc/kayobe/globals.yml @@ -45,12 +45,13 @@ ############################################################################### # OS distribution. -# OS distribution name. Valid options are "centos", "ubuntu". Default is -# "centos". +# OS distribution name. Valid options are "centos", "rocky", "ubuntu". Default +# is "centos". #os_distribution: # OS release. Valid options are "8-stream" when os_distribution is "centos", or -# "focal" when os_distribution is "ubuntu". +# "8" when os_distribution is "rocky", or "focal" when os_distribution is +# "ubuntu". #os_release: ############################################################################### diff --git a/etc/kayobe/infra-vms.yml b/etc/kayobe/infra-vms.yml index c8aec1cd6..a802f5acc 100644 --- a/etc/kayobe/infra-vms.yml +++ b/etc/kayobe/infra-vms.yml @@ -32,7 +32,10 @@ # Base image for the infra VM root volume. Default is # "https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img" # when os_distribution is "ubuntu", or -# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20210210.0.x86_64.qcow2" +# https://dl.rockylinux.org/pub/rocky/8/images/Rocky-8-GenericCloud.latest.x86_64.qcow2 +# when os_distribution is "rocky", +# or +# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20220913.0.x86_64.qcow2" # otherwise. #infra_vm_root_image: diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index a07406f01..3770a5283 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -62,14 +62,15 @@ # Kolla configuration. # Kolla base container image distribution. Options are "centos", "debian", -# "ubuntu". Default is {{ os_distribution }}. +# "ubuntu". Default is +# {{ 'centos' if os_distribution == 'rocky' else os_distribution }}. #kolla_base_distro: # Kolla container image type: binary or source. Default is 'source'. #kolla_install_type: # URL of docker registry to use for Kolla images. Default is not set, in which -# case Dockerhub will be used. +# case Quay.io will be used. #kolla_docker_registry: # Docker namespace to use for Kolla images. Default is 'kolla'. @@ -387,6 +388,7 @@ #kolla_enable_prometheus_elasticsearch_exporter: #kolla_enable_prometheus_fluentd_integration: #kolla_enable_prometheus_haproxy_exporter: +#kolla_enable_prometheus_libvirt_exporter: #kolla_enable_prometheus_memcached_exporter: #kolla_enable_prometheus_mysqld_exporter: #kolla_enable_prometheus_node_exporter: diff --git a/etc/kayobe/seed-vm.yml b/etc/kayobe/seed-vm.yml index 9ad7a7438..3856d51f2 100644 --- a/etc/kayobe/seed-vm.yml +++ b/etc/kayobe/seed-vm.yml @@ -25,8 +25,11 @@ # Base image for the seed VM root volume. Default is # "https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img" -# when os_distribution is "ubuntu", or -# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20210210.0.x86_64.qcow2" +# when os_distribution is "ubuntu", +# https://dl.rockylinux.org/pub/rocky/8/images/Rocky-8-GenericCloud.latest.x86_64.qcow2 +# when os_distribution is "rocky", +# or +# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20220913.0.x86_64.qcow2" # otherwise. #seed_vm_root_image: diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index 3a5186ea6..ade99307d 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -100,7 +100,6 @@ # Example: # seed_containers: # squid: -# name: "squid" # image: "stackhpc/squid:3.5.20-1" # pre: "{{ kayobe_env_config_path }}/containers/squid/pre.yml" # post: "{{ kayobe_env_config_path }}/containers/squid/post.yml" diff --git a/etc/kayobe/time.yml b/etc/kayobe/time.yml index 23c1d0dc4..8bfbd3391 100644 --- a/etc/kayobe/time.yml +++ b/etc/kayobe/time.yml @@ -30,6 +30,9 @@ # #chrony_ntp_servers: +# Synchronise hardware clock with system time. Default is true. +#chrony_rtcsync_enabled: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes From 4364e6c7ae2d707d8f7d9bcbe9dd42f93684135d Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Tue, 7 Feb 2023 10:03:54 +0100 Subject: [PATCH 200/292] OSSA-2023-002: update cinder, glance & nova images for CentOS & Ubuntu These images include backported fixes for OSSA-2023-002. https://security.openstack.org/ossa/OSSA-2023-002.html --- etc/kayobe/kolla/globals.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 1d7f7dd69..22e4fcc2a 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -9,21 +9,25 @@ enable_docker_repo: {% raw %}"{{ 'overcloud' not in group_names or ansible_facts {% if kolla_base_distro == 'centos' %} bifrost_tag: xena-20221128T101757 +cinder_tag: xena-20230206T165215 etcd_tag: xena-20230201T095253 +glance_tag: xena-20230206T165215 glance_tls_proxy_tag: "{% raw %}{{ haproxy_tag | default(openstack_tag) }}{% endraw %}" magnum_tag: xena-20230131T144233 neutron_tls_proxy_tag: "{% raw %}{{ haproxy_tag | default(openstack_tag) }}{% endraw %}" -nova_tag: xena-20230120T115956 +nova_tag: xena-20230206T165215 rabbitmq_tag: xena-20230105T141714 {% else %} bifrost_tag: xena-20221213T224057 +cinder_tag: xena-20230206T165220 etcd_tag: xena-20230201T131415 +glance_tag: xena-20230206T165220 glance_tls_proxy_tag: "{% raw %}{{ haproxy_tag | default(openstack_tag) }}{% endraw %}" kolla_toolbox_tag: xena-20230104T145414 magnum_tag: xena-20230131T151601 neutron_tag: xena-20230104T145414 neutron_tls_proxy_tag: "{% raw %}{{ haproxy_tag | default(openstack_tag) }}{% endraw %}" -nova_tag: xena-20230104T145414 +nova_tag: xena-20230206T165220 octavia_tag: xena-20230104T145414 openvswitch_tag: xena-20230104T145414 ovn_tag: xena-20230104T145414 From 64424b6c5094236052e10a3bf136e1bbbf199c5f Mon Sep 17 00:00:00 2001 From: stackhpc-ci <22933334+stackhpc-ci@users.noreply.github.com> Date: Wed, 8 Feb 2023 09:46:12 +0000 Subject: [PATCH 201/292] feat: automatic update of workflows stackhpc/xena --- .github/workflows/tag-and-release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tag-and-release.yml b/.github/workflows/tag-and-release.yml index af47d87b5..605ddb470 100644 --- a/.github/workflows/tag-and-release.yml +++ b/.github/workflows/tag-and-release.yml @@ -5,6 +5,7 @@ name: Tag & Release branches: - stackhpc/xena permissions: + actions: read contents: write jobs: tag-and-release: From 5bab33f49e23b387fd086d0ebee402902430078a Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 10 Feb 2023 16:47:10 +0100 Subject: [PATCH 202/292] Use stricter permissions for Pulp TLS cert and key Pulp runs as root inside its container. It has no problem reading files with 0600 permissions. There is no reason use 0644, especially for the key which is readable by any user on the seed. --- etc/kayobe/containers/pulp/pre.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/containers/pulp/pre.yml b/etc/kayobe/containers/pulp/pre.yml index 85af37916..2ced730eb 100644 --- a/etc/kayobe/containers/pulp/pre.yml +++ b/etc/kayobe/containers/pulp/pre.yml @@ -33,7 +33,7 @@ template: src: "{{ item.src }}" dest: "/opt/kayobe/containers/pulp/certs/{{ item.dest }}" - mode: 0644 + mode: 0600 become: true loop: - src: "{{ pulp_cert_path }}" From 538c1a8dc87f85f3b873fc474b98081ffd3cb831 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Fri, 10 Feb 2023 17:27:20 +0000 Subject: [PATCH 203/292] Remove kolla_enable_nova_libvirt_container Variable not available until yoga --- etc/kayobe/kolla.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index fb5a5a496..a9c1b1f1a 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -568,7 +568,6 @@ kolla_enable_grafana: true #kolla_enable_keystone_horizon_policy_file: #kolla_enable_kibana: #kolla_enable_kuryr: -kolla_enable_nova_libvirt_container: "{{ os_distribution not in ['rocky'] }}" #kolla_enable_loadbalancer: #kolla_enable_magnum: #kolla_enable_manila: From 949649e57b67cdafe01c9f7204901fcd18da5571 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Fri, 10 Feb 2023 17:29:28 +0000 Subject: [PATCH 204/292] Remove duplicate key --- etc/kayobe/pulp-repo-versions.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/etc/kayobe/pulp-repo-versions.yml b/etc/kayobe/pulp-repo-versions.yml index 14f7684e0..938d2788c 100644 --- a/etc/kayobe/pulp-repo-versions.yml +++ b/etc/kayobe/pulp-repo-versions.yml @@ -17,7 +17,6 @@ stackhpc_pulp_repo_epel_modular_version: 20220913T043117 stackhpc_pulp_repo_epel_version: 20230210T031947 stackhpc_pulp_repo_grafana_version: 20230209T034107 stackhpc_pulp_repo_mariadb_10_5_centos8_version: 20230206T150339 -stackhpc_pulp_repo_mlnx_ofed_5_7_1_0_2_0_rhel8_6_version: 20220920T151419 stackhpc_pulp_repo_rabbitmq_erlang_version: 20230210T031947 stackhpc_pulp_repo_rabbitmq_server_version: 20230201T025809 stackhpc_pulp_repo_treasuredata_4_version: 20221105T035018 From a5d960849b0f062b9af1f37cd6a1511339c90900 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Fri, 10 Feb 2023 18:02:12 +0000 Subject: [PATCH 205/292] Revert docker repo version to match wallaby --- etc/kayobe/pulp-repo-versions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/pulp-repo-versions.yml b/etc/kayobe/pulp-repo-versions.yml index 938d2788c..0e785c023 100644 --- a/etc/kayobe/pulp-repo-versions.yml +++ b/etc/kayobe/pulp-repo-versions.yml @@ -11,7 +11,7 @@ stackhpc_pulp_repo_centos_stream_8_opstools_version: 20220617T100837 stackhpc_pulp_repo_centos_stream_8_powertools_version: 20230201T025809 stackhpc_pulp_repo_centos_stream_8_storage_ceph_pacific_version: 20230201T025809 stackhpc_pulp_repo_docker_ce_ubuntu_version: 20220708T132615 -stackhpc_pulp_repo_docker_version: 20230210T031947 +stackhpc_pulp_repo_docker_version: 20230203T025251 stackhpc_pulp_repo_elasticsearch_logstash_kibana_7_x_version: 20230203T025251 stackhpc_pulp_repo_epel_modular_version: 20220913T043117 stackhpc_pulp_repo_epel_version: 20230210T031947 From 268b906ebae9b6ef4ac767a8b8658755f2ef3918 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 13 Feb 2023 11:50:47 +0000 Subject: [PATCH 206/292] Xena: Update local Kolla build repos --- etc/kayobe/pulp.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index 3efacd586..b20000e30 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -339,8 +339,8 @@ stackhpc_pulp_repository_rpm_repos: required: "{{ stackhpc_pulp_sync_el_8 }}" # Additional repositories required to build Kolla containers from local Pulp service. - - name: CentOS Stream 8 - OpenStack Wallaby - url: "{{ stackhpc_release_pulp_content_url }}/centos/8-stream/cloud/x86_64/openstack-wallaby/{{ stackhpc_pulp_repo_centos_stream_8_openstack_wallaby_version }}" + - name: CentOS Stream 8 - OpenStack Xena + url: "{{ stackhpc_release_pulp_content_url }}/centos/8-stream/cloud/x86_64/openstack-xena/{{ stackhpc_pulp_repo_centos_stream_8_openstack_xena_version }}" remote_username: "{{ stackhpc_release_pulp_username }}" remote_password: "{{ stackhpc_release_pulp_password }}" client_cert: "" @@ -532,9 +532,9 @@ stackhpc_pulp_distribution_rpm_development: required: "{{ stackhpc_pulp_sync_el_8 | bool }}" # Additional repositories required to build Kolla containers from local Pulp service. - - name: "centos-stream-8-openstack-wallaby-development" - base_path: "centos/8-stream/cloud/x86_64/openstack-wallaby/development" - repository: CentOS Stream 8 - OpenStack Wallaby + - name: "centos-stream-8-openstack-xena-development" + base_path: "centos/8-stream/cloud/x86_64/openstack-xena/development" + repository: CentOS Stream 8 - OpenStack Xena state: present required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}" - name: "centos-stream-8-advanced-virtualization-development" @@ -654,9 +654,9 @@ stackhpc_pulp_distribution_rpm_production: required: "{{ stackhpc_pulp_sync_el_8 | bool }}" # Additional repositories required to build Kolla containers from local Pulp service. - - name: "centos-stream-8-openstack-wallaby-production" - base_path: "centos/8-stream/cloud/x86_64/openstack-wallaby/production" - repository: CentOS Stream 8 - OpenStack Wallaby + - name: "centos-stream-8-openstack-xena-production" + base_path: "centos/8-stream/cloud/x86_64/openstack-xena/production" + repository: CentOS Stream 8 - OpenStack Xena state: present required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}" - name: "centos-stream-8-advanced-virtualization-production" From ac7b697087bdff1d7c233a0e3d47625d14abab52 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Tue, 14 Feb 2023 14:40:42 +0000 Subject: [PATCH 207/292] Use same repo tags as Wallaby except Xena specific --- etc/kayobe/pulp-repo-versions.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/etc/kayobe/pulp-repo-versions.yml b/etc/kayobe/pulp-repo-versions.yml index 0e785c023..99ceb3ffd 100644 --- a/etc/kayobe/pulp-repo-versions.yml +++ b/etc/kayobe/pulp-repo-versions.yml @@ -5,7 +5,7 @@ stackhpc_pulp_repo_centos_stream_8_appstream_version: 20230201T025809 stackhpc_pulp_repo_centos_stream_8_baseos_version: 20230201T025809 stackhpc_pulp_repo_centos_stream_8_extras_version: 20220401T032901 stackhpc_pulp_repo_centos_stream_8_nfv_extras_version: 20220609T110556 -stackhpc_pulp_repo_centos_stream_8_nfv_openvswitch_version: 20230208T033911 +stackhpc_pulp_repo_centos_stream_8_nfv_openvswitch_version: 20230203T025251 stackhpc_pulp_repo_centos_stream_8_openstack_xena_version: 20230209T034107 stackhpc_pulp_repo_centos_stream_8_opstools_version: 20220617T100837 stackhpc_pulp_repo_centos_stream_8_powertools_version: 20230201T025809 @@ -14,10 +14,9 @@ stackhpc_pulp_repo_docker_ce_ubuntu_version: 20220708T132615 stackhpc_pulp_repo_docker_version: 20230203T025251 stackhpc_pulp_repo_elasticsearch_logstash_kibana_7_x_version: 20230203T025251 stackhpc_pulp_repo_epel_modular_version: 20220913T043117 -stackhpc_pulp_repo_epel_version: 20230210T031947 -stackhpc_pulp_repo_grafana_version: 20230209T034107 -stackhpc_pulp_repo_mariadb_10_5_centos8_version: 20230206T150339 -stackhpc_pulp_repo_rabbitmq_erlang_version: 20230210T031947 +stackhpc_pulp_repo_epel_version: 20230206T150339 +stackhpc_pulp_repo_grafana_version: 20230207T024123 +stackhpc_pulp_repo_rabbitmq_erlang_version: 20221229T025716 stackhpc_pulp_repo_rabbitmq_server_version: 20230201T025809 stackhpc_pulp_repo_treasuredata_4_version: 20221105T035018 stackhpc_pulp_repo_ubuntu_cloud_archive_version: 20220804T040153 From 648dbc49cd4d224b82fae073b3ac893ec4c30872 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Tue, 14 Feb 2023 14:54:41 +0000 Subject: [PATCH 208/292] Add accidentally removed mariadb version --- etc/kayobe/pulp-repo-versions.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/kayobe/pulp-repo-versions.yml b/etc/kayobe/pulp-repo-versions.yml index 99ceb3ffd..4c974b316 100644 --- a/etc/kayobe/pulp-repo-versions.yml +++ b/etc/kayobe/pulp-repo-versions.yml @@ -16,6 +16,7 @@ stackhpc_pulp_repo_elasticsearch_logstash_kibana_7_x_version: 20230203T025251 stackhpc_pulp_repo_epel_modular_version: 20220913T043117 stackhpc_pulp_repo_epel_version: 20230206T150339 stackhpc_pulp_repo_grafana_version: 20230207T024123 +stackhpc_pulp_repo_mariadb_10_5_centos8_version: 20230206T150339 stackhpc_pulp_repo_rabbitmq_erlang_version: 20221229T025716 stackhpc_pulp_repo_rabbitmq_server_version: 20230201T025809 stackhpc_pulp_repo_treasuredata_4_version: 20221105T035018 From c3676e00205bdd4f35766c59394d11c695c52a94 Mon Sep 17 00:00:00 2001 From: Dawud M <7688823+technowhizz@users.noreply.github.com> Date: Fri, 23 Dec 2022 10:14:38 +0000 Subject: [PATCH 209/292] Fix missing group in smart alerting rules (cherry picked from commit 02301f4f8420a33c0cf7e25c2a5ad3ee4c488988) --- etc/kayobe/kolla/config/prometheus/smart.rules | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/kolla/config/prometheus/smart.rules b/etc/kayobe/kolla/config/prometheus/smart.rules index 0b6552598..aea36bdf8 100644 --- a/etc/kayobe/kolla/config/prometheus/smart.rules +++ b/etc/kayobe/kolla/config/prometheus/smart.rules @@ -1,6 +1,10 @@ {% raw %} -- alert: DiskSmartStatusUnhealthy +groups: +- name: Smart + rules: + + - alert: DiskSmartStatusUnhealthy expr: smartmon_device_smart_healthy < 1 for: 10m labels: @@ -9,4 +13,4 @@ summary: "SMART monitor reports bad disk on (instance {{ $labels.instance }})" description: "{{ $labels.instance }} is reporting unhealthy for the disk at {{ $labels.disk }}. Disk serial number is: {{ $labels.serial_number }}" -{% endraw %} +{% endraw %} \ No newline at end of file From 9ccd872732f81f22cfd789a872a6143ff4bdb2b7 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 23 Jan 2023 16:57:10 +0000 Subject: [PATCH 210/292] alertmanager example and monitoring docs update (cherry picked from commit 452f6af91c47acd679c1fcc67d40089e28c3b031) --- doc/source/configuration/monitoring.rst | 55 ++++++++++++++++++- .../prometheus-alertmanager.yml.example | 47 ++++++++++++++++ 2 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.yml.example diff --git a/doc/source/configuration/monitoring.rst b/doc/source/configuration/monitoring.rst index 6045f3c29..c2384ba93 100644 --- a/doc/source/configuration/monitoring.rst +++ b/doc/source/configuration/monitoring.rst @@ -7,9 +7,20 @@ Monitoring Configuration StackHPC kayobe config includes a reference monitoring and alerting stack based on Prometheus, Alertmanager, Grafana, Fluentd, Elasticsearch & Kibana. These -services by default come enabled and configured. Central Elasticsearch cluster -collects OpenStack logs, with an option to receive operating system logs too. -In order to enable this, execute custom playbook after deployment: +services by default come enabled and configured. + +Monitoring hosts, usually the controllers, should be added to the monitoring +group. The group definition can be applied in various different places. For +example, this configuration could be added to etc/kayobe/inventory/groups: + +.. code-block:: yaml + + [monitoring:children] + controllers + +Central Elasticsearch cluster collects OpenStack logs, with an option to receive +operating system logs too. In order to enable this, execute custom playbook +after deployment: .. code-block:: console @@ -78,3 +89,41 @@ on the overcloud hosts: SMART reporting should now be enabled along with a Prometheus alert for unhealthy disks and a Grafana dashboard called ``Hardware Overview``. + +Alertmanager and Slack +====================== + +StackHPC Kayobe configuration comes bundled with an array of alerts but does not +enable any receivers for notifications by default. Various receivers can be +configured for Alertmanager. Slack is currently the most common. + +To set up a receiver, create a ``prometheus-alertmanager.yml`` file under +``etc/kayobe/kolla/config/prometheus/``. An example config is stored in this +directory. The example configuration uses two Slack channels. One channel +receives all alerts while the other only receives alerts tagged as critical. It +also adds a silence button to temporarily mute alerts. To use the example in a +deployment, you will need to generate two webhook URLs, one for each channel. + +To generate a slack webhook, `create a new app +`__ in the workspace you want to add alerts to. +From the Features page, toggle Activate incoming webhooks on. Click Add new +webhook to workspace. Pick a channel that the app will post to, then click +Authorise. You only need one app to generate both webhooks. + +Both URLs should be encrypted using ansible vault, as they give anyone access to +your slack channels. The standard practice is to store them in +``kayobe/secrets.yml`` as: + +.. code-block:: yaml + + secrets_slack_notification_channel_url: + secrets_slack_critical_notification_channel_url: + +These should then be set as the ``slack_api_url`` and ``api_url`` for the +regular and critical alerts channels respectively. Both slack channel names will +need to be set, and the proxy URL sould be set or removed. + +If you want to add an alerting rule, there are many good examples of alerts are +available `here `__. They simply +need to be added to one of the ``*.rules`` files in the prometheus configuration +directory. \ No newline at end of file diff --git a/etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.yml.example b/etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.yml.example new file mode 100644 index 000000000..435d63c09 --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.yml.example @@ -0,0 +1,47 @@ +--- +global: + resolve_timeout: 5m + smtp_require_tls: true + slack_api_url: 'https://hooks.slack.com/services/example/alerts/webhook' + +route: + receiver: 'slack-notifications' + group_by: [alertname] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h + + routes: + - matchers: + - severity=~"critical|alert" + receiver: 'slack-critical-notifications' + + +receivers: + - name: 'slack-notifications' + slack_configs: + - channel: '#notifications' + actions: + - type: button + text: 'Silence 🔕' + url: {{ '{% raw %}' }}{% raw %} '{{ template "__alert_silence_link" . }}' +{% endraw %}{{ '{% endraw %}' }} + send_resolved: true + http_config: + proxy_url: http://1.2.3.4:3128 + - name: 'slack-critical-notifications' + slack_configs: + - channel: '#notifications-critical' + actions: + - type: button + text: 'Silence 🔕' + url: {{ '{% raw %}' }}{% raw %} '{{ template "__alert_silence_link" . }}' +{% endraw %}{{ '{% endraw %}' }} + send_resolved: true + http_config: + proxy_url: http://1.2.3.4:3128 + api_url: 'https://hooks.slack.com/services/example/alerts/webhook-critical' + + +templates: + - '/etc/prometheus/*.tmpl' \ No newline at end of file From 2bf04bf81a5953239e51558c8c6a3f2bf1bea4c7 Mon Sep 17 00:00:00 2001 From: Dawud M <7688823+technowhizz@users.noreply.github.com> Date: Fri, 30 Dec 2022 23:11:55 +0000 Subject: [PATCH 211/292] Add drive temps to hardware dashboard This adds a column for the drive temperatures to the table on the hardware overview dashboard and changes the column ordering for better readability. (cherry picked from commit 7f48ed25c1fdd35582e7822f40861a51301ee399) --- .../openstack/hardware_overview.json | 156 ++++++++++++++---- 1 file changed, 122 insertions(+), 34 deletions(-) diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json index e4f78aee3..0c236dce4 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json @@ -25,6 +25,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, + "id": 1935501, "links": [], "liveNow": false, "panels": [ @@ -251,23 +252,7 @@ "filterable": false, "inspect": false }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "Failed" - }, - "1": { - "color": "dark-green", - "index": 0, - "text": "Ok" - } - }, - "type": "value" - } - ], + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ @@ -306,7 +291,7 @@ { "matcher": { "id": "byName", - "options": "Value" + "options": "Value #Health" }, "properties": [ { @@ -315,13 +300,37 @@ }, { "id": "custom.width" + }, + { + "id": "displayName", + "value": "Health" + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Bad" + }, + "1": { + "color": "green", + "index": 0, + "text": "Ok" + } + }, + "type": "value" + } + ] } ] }, { "matcher": { "id": "byName", - "options": "type" + "options": "type 1" }, "properties": [ { @@ -353,7 +362,7 @@ { "matcher": { "id": "byName", - "options": "Time" + "options": "Time 1" }, "properties": [ { @@ -365,7 +374,7 @@ { "matcher": { "id": "byName", - "options": "disk" + "options": "disk 1" }, "properties": [ { @@ -381,7 +390,7 @@ { "matcher": { "id": "byName", - "options": "instance" + "options": "instance 1" }, "properties": [ { @@ -397,7 +406,7 @@ { "matcher": { "id": "byName", - "options": "Type" + "options": "Hostname" }, "properties": [ { @@ -408,33 +417,60 @@ { "matcher": { "id": "byName", - "options": "Serial Number" + "options": "group 1" }, "properties": [ { - "id": "custom.width" + "id": "displayName", + "value": "Group" } ] }, { "matcher": { "id": "byName", - "options": "Hostname" + "options": "smart_id" }, "properties": [ { - "id": "custom.width" + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".* 2" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true } ] }, { "matcher": { "id": "byName", - "options": "Disk" + "options": "Value #Temp" }, "properties": [ { - "id": "custom.width" + "id": "displayName", + "value": "Temperature" + }, + { + "id": "unit", + "value": "celsius" + }, + { + "id": "noValue", + "value": "-" + }, + { + "id": "custom.displayMode", + "value": "color-text" } ] } @@ -442,7 +478,7 @@ }, "gridPos": { "h": 13, - "w": 18, + "w": 20, "x": 0, "y": 7 }, @@ -455,6 +491,7 @@ ], "show": false }, + "frameIndex": 1, "showHeader": true, "sortBy": [] }, @@ -481,13 +518,64 @@ "interval": "", "legendFormat": "", "range": false, - "refId": "A", + "refId": "Health", "units": "none", "valueHandler": "Number Threshold", "warn": 0 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "smartmon_temperature_case_raw_value", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "range": false, + "refId": "Temp" + } + ], + "title": "SMART Info", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "serial_number" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Time 1": 3, + "Time 2": 10, + "Value #Health": 8, + "Value #Temp": 9, + "__name__ 1": 4, + "__name__ 2": 11, + "disk 1": 5, + "disk 2": 12, + "group 1": 0, + "group 2": 13, + "instance 1": 1, + "instance 2": 14, + "job 1": 6, + "job 2": 15, + "serial_number": 7, + "smart_id": 16, + "type 1": 2, + "type 2": 17 + }, + "renameByName": {} + } } ], - "title": "Panel Title", "transparent": true, "type": "table" } @@ -511,7 +599,7 @@ }, { "current": { - "selected": true, + "selected": false, "text": "Prometheus", "value": "Prometheus" }, @@ -537,7 +625,7 @@ "timezone": "", "title": "Hardware Overview", "uid": "TCN51Y25P", - "version": 1, + "version": 3, "weekStart": "" } {% endraw %} \ No newline at end of file From b66e05cec0e2ee12f8557c9eb48a72b15a9a3108 Mon Sep 17 00:00:00 2001 From: Dawud M <7688823+technowhizz@users.noreply.github.com> Date: Wed, 4 Jan 2023 11:21:46 +0000 Subject: [PATCH 212/292] Update the hardware overview dashboard Adds the temperature of drives as a column to the table. Adds a timeseries to track the history of the temperature. Introduces a variable dropdown picker for nodes so that only information about disks on a specific node is shown. Change-Id: Ibff3037865e1891d8e07a945dbeedf08e1f8c0d2 (cherry picked from commit 4999f9c910204090d05b3338c5c60d73cf07b2dc) --- .../openstack/hardware_overview.json | 142 ++++++++++++++++-- 1 file changed, 133 insertions(+), 9 deletions(-) diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json index 0c236dce4..63bbf4af3 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json @@ -84,7 +84,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "count(smartmon_device_smart_healthy > 0)", + "expr": "count(smartmon_device_smart_healthy{instance=~\"$node\"} > 0 )", "format": "time_series", "instant": false, "interval": "", @@ -157,7 +157,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "count(smartmon_device_smart_healthy < 1) ", + "expr": "count(smartmon_device_smart_healthy{instance=~\"$node\"} < 1) ", "format": "time_series", "instant": false, "interval": "", @@ -225,7 +225,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "count(smartmon_device_smart_healthy)", + "expr": "count(smartmon_device_smart_healthy{instance=~\"$node\"})", "format": "time_series", "instant": false, "interval": "", @@ -477,7 +477,7 @@ ] }, "gridPos": { - "h": 13, + "h": 10, "w": 20, "x": 0, "y": 7 @@ -512,7 +512,7 @@ "displayValueWithAlias": "Never", "editorMode": "code", "exemplar": false, - "expr": "smartmon_device_smart_healthy", + "expr": "smartmon_device_smart_healthy{instance=~\"$node\"}", "format": "table", "instant": true, "interval": "", @@ -530,7 +530,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "smartmon_temperature_case_raw_value", + "expr": "smartmon_temperature_case_raw_value{instance=~\"$node\"}", "format": "table", "hide": false, "instant": true, @@ -578,6 +578,102 @@ ], "transparent": true, "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Temperature (°C)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 20, + "x": 0, + "y": 17 + }, + "hideTimeOverride": false, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg_over_time(smartmon_temperature_case_raw_value{instance=~\"$node\"}[1h])", + "instant": false, + "interval": "", + "legendFormat": "{{instance}} - {{disk}} - {{serial_number}}", + "range": true, + "refId": "A" + } + ], + "title": "Disk Temperatures", + "type": "timeseries" } ], "refresh": false, @@ -599,7 +695,7 @@ }, { "current": { - "selected": false, + "selected": true, "text": "Prometheus", "value": "Prometheus" }, @@ -614,18 +710,46 @@ "regex": "", "skipUrlSync": false, "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(node_cpu_seconds_total{job=\"node\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "Host:", + "multi": false, + "name": "node", + "options": [], + "query": { + "query": "label_values(node_cpu_seconds_total{job=\"node\"}, instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" } ] }, "time": { - "from": "now-1h", + "from": "now-24h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Hardware Overview", "uid": "TCN51Y25P", - "version": 3, + "version": 5, "weekStart": "" } {% endraw %} \ No newline at end of file From efc90d241935836835b30dc3e89bf72035b4b803 Mon Sep 17 00:00:00 2001 From: Dawud M <7688823+technowhizz@users.noreply.github.com> Date: Fri, 23 Dec 2022 12:05:35 +0000 Subject: [PATCH 213/292] Add release note for smart rules bugfix (cherry picked from commit 5e237eb2c525558db67f6e22914bdd2c302b9a84) --- releasenotes/notes/bugfix-smart-rules-53ff5400e7f681c8.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 releasenotes/notes/bugfix-smart-rules-53ff5400e7f681c8.yaml diff --git a/releasenotes/notes/bugfix-smart-rules-53ff5400e7f681c8.yaml b/releasenotes/notes/bugfix-smart-rules-53ff5400e7f681c8.yaml new file mode 100644 index 000000000..8da7f80fc --- /dev/null +++ b/releasenotes/notes/bugfix-smart-rules-53ff5400e7f681c8.yaml @@ -0,0 +1,6 @@ +--- + +fixes: + - | + Fixed a syntax error in Prometheus SMART monitoring rules + From be5e64d2ea6b350dd4c1742d7491cde2b1e976fc Mon Sep 17 00:00:00 2001 From: Dawud M <7688823+technowhizz@users.noreply.github.com> Date: Wed, 4 Jan 2023 17:26:21 +0000 Subject: [PATCH 214/292] Add awesome alerts alert for NTP Change-Id: I80610eeaa5cd7f83b0753af97d53cedbe12c2c86 (cherry picked from commit c82762c5691e3cbc27a5d0f4f664cf30c2c51548) --- .../kolla/config/prometheus/system.rules | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/etc/kayobe/kolla/config/prometheus/system.rules b/etc/kayobe/kolla/config/prometheus/system.rules index 532710fd3..be1343b82 100644 --- a/etc/kayobe/kolla/config/prometheus/system.rules +++ b/etc/kayobe/kolla/config/prometheus/system.rules @@ -78,6 +78,24 @@ groups: summary: "Host EDAC Uncorrectable Errors detected (instance {{ $labels.instance }})" description: "{{ $labels.instance }} has had {{ printf \"%.0f\" $value }} uncorrectable memory errors reported by EDAC in the last 5 minutes." + - alert: HostClockSkew + expr: (node_timex_offset_seconds > 0.05 and deriv(node_timex_offset_seconds[5m]) >= 0) or (node_timex_offset_seconds < -0.05 and deriv(node_timex_offset_seconds[5m]) <= 0) + for: 2m + labels: + severity: warning + annotations: + summary: Host clock skew (instance {{ $labels.instance }}) + description: "Clock skew detected. Clock is out of sync. Ensure NTP is configured correctly on this host." + + - alert: HostClockNotSynchronising + expr: min_over_time(node_timex_sync_status[1m]) == 0 and node_timex_maxerror_seconds >= 16 + for: 2m + labels: + severity: warning + annotations: + summary: Host clock not synchronising (instance {{ $labels.instance }}) + description: "Clock not synchronising. Ensure NTP is configured on this host." + - alert: HostConntrackLimit expr: node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8 for: 5m From 22a0a4ac1c66a7eb9e29318989a3567b68127818 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Wed, 15 Feb 2023 10:18:36 +0000 Subject: [PATCH 215/292] Change to new CentOS tags --- etc/kayobe/kolla.yml | 2 +- etc/kayobe/kolla/globals.yml | 15 ++++----------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index a9c1b1f1a..f4c70bc34 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -96,7 +96,7 @@ kolla_docker_registry_password: "{{ stackhpc_docker_registry_password }}" # Kolla OpenStack release version. This should be a Docker image tag. # Default is {{ openstack_release }}. -kolla_openstack_release: "{% if kolla_base_distro == 'centos' %}xena-20221010T103511{% else %}xena-20221213T204703{% endif %}" +kolla_openstack_release: "{% if kolla_base_distro == 'centos' %}xena-20230214T150257{% else %}xena-20221213T204703{% endif %}" # Docker tag applied to built container images. Default is # {{ kolla_openstack_release }}. diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 22e4fcc2a..f0d66d241 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -8,25 +8,15 @@ enable_docker_repo: {% raw %}"{{ 'overcloud' not in group_names or ansible_facts {% if kolla_base_distro == 'centos' %} -bifrost_tag: xena-20221128T101757 -cinder_tag: xena-20230206T165215 -etcd_tag: xena-20230201T095253 -glance_tag: xena-20230206T165215 -glance_tls_proxy_tag: "{% raw %}{{ haproxy_tag | default(openstack_tag) }}{% endraw %}" -magnum_tag: xena-20230131T144233 -neutron_tls_proxy_tag: "{% raw %}{{ haproxy_tag | default(openstack_tag) }}{% endraw %}" -nova_tag: xena-20230206T165215 -rabbitmq_tag: xena-20230105T141714 +bifrost_tag: xena-20230214T165534 {% else %} bifrost_tag: xena-20221213T224057 cinder_tag: xena-20230206T165220 etcd_tag: xena-20230201T131415 glance_tag: xena-20230206T165220 -glance_tls_proxy_tag: "{% raw %}{{ haproxy_tag | default(openstack_tag) }}{% endraw %}" kolla_toolbox_tag: xena-20230104T145414 magnum_tag: xena-20230131T151601 neutron_tag: xena-20230104T145414 -neutron_tls_proxy_tag: "{% raw %}{{ haproxy_tag | default(openstack_tag) }}{% endraw %}" nova_tag: xena-20230206T165220 octavia_tag: xena-20230104T145414 openvswitch_tag: xena-20230104T145414 @@ -34,6 +24,9 @@ ovn_tag: xena-20230104T145414 rabbitmq_tag: xena-20230105T145825 {% endif %} +glance_tls_proxy_tag: "{% raw %}{{ haproxy_tag | default(openstack_tag) }}{% endraw %}" +neutron_tls_proxy_tag: "{% raw %}{{ haproxy_tag | default(openstack_tag) }}{% endraw %}" + om_enable_rabbitmq_high_availability: true ############################################################################# From c2eb64fe631bbe6deb9aaa8649cd93d86835e95c Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Thu, 16 Feb 2023 10:31:19 +0000 Subject: [PATCH 216/292] Fix versions & releases in package refresh note --- ...02-2023-package-refresh-9de8049f8cb08a5e.yaml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/releasenotes/notes/release-train-02-2023-package-refresh-9de8049f8cb08a5e.yaml b/releasenotes/notes/release-train-02-2023-package-refresh-9de8049f8cb08a5e.yaml index 058cb3945..72a3318e0 100644 --- a/releasenotes/notes/release-train-02-2023-package-refresh-9de8049f8cb08a5e.yaml +++ b/releasenotes/notes/release-train-02-2023-package-refresh-9de8049f8cb08a5e.yaml @@ -1,16 +1,18 @@ --- features: - | - The package repositories have been refreshed. Key packages to note are: + Package repositories and container images for CentOS Stream based deployments have been updated. + Key packages to note are: + * Kernel * version: 4.18.0 - * release: 408.el8 + * release: 448.el8 * Libvirt - * version: 7.6.0 - * release: 6.el8s + * version: 8.0.0 + * release: 6.module_el8.7.0+1140+ff0772f9 * OVS * version: 2.17.0 - * release: 31.el8s + * release: 71.el8s * OVN - * version: 22.06.0 - * release: 24.el8s + * version: 22.09.0 + * release: 11.el8s From 9a03f40c2fc1137fa45be464ae936641a6709988 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 17 Feb 2023 14:32:35 +0100 Subject: [PATCH 217/292] Document how to override swap_device --- etc/kayobe/ansible/swap.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/swap.yml b/etc/kayobe/ansible/swap.yml index e47397212..97d7a5a0b 100644 --- a/etc/kayobe/ansible/swap.yml +++ b/etc/kayobe/ansible/swap.yml @@ -6,7 +6,9 @@ # * swap_group: Host pattern against which to target the playbook. Default is # 'overcloud'. # * swap_device: Name of the swap device to configure. Default is -# '/dev/rootvg/lv_swap'. +# '/dev/rootvg/lv_swap'. To use a different value, set an extra +# var (e.g. in $KAYOBE_CONFIG_PATH/swap.yml) because play vars +# take precedence over any inventory variable. - name: Configure swap hosts: "{{ swap_group | default('overcloud') }}" From eaa75644f0a7bfa902ba70fddb28c8b811b3a9f6 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Sun, 19 Feb 2023 15:09:13 +0100 Subject: [PATCH 218/292] Fix Ansible deprecation warning --- etc/kayobe/ansible.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/ansible.cfg b/etc/kayobe/ansible.cfg index 310c79499..515af8d32 100644 --- a/etc/kayobe/ansible.cfg +++ b/etc/kayobe/ansible.cfg @@ -7,7 +7,7 @@ bin_ansible_callbacks = True # Disable fact variable injection to improve performance. inject_facts_as_vars = False # Add timing information to output -callback_whitelist = ansible.posix.profile_tasks +callbacks_enabled = ansible.posix.profile_tasks [ssh_connection] pipelining = True From c6c5a01fff32237fa81fd3c03594a9eeb71e6735 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Mon, 20 Feb 2023 15:03:05 +0000 Subject: [PATCH 219/292] Refresh Ubuntu images Build logs: https://github.com/stackhpc/stackhpc-kayobe-config/actions/runs/4186064857 --- etc/kayobe/kolla.yml | 2 +- etc/kayobe/kolla/globals.yml | 13 +------------ 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index f4c70bc34..b328f0d94 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -96,7 +96,7 @@ kolla_docker_registry_password: "{{ stackhpc_docker_registry_password }}" # Kolla OpenStack release version. This should be a Docker image tag. # Default is {{ openstack_release }}. -kolla_openstack_release: "{% if kolla_base_distro == 'centos' %}xena-20230214T150257{% else %}xena-20221213T204703{% endif %}" +kolla_openstack_release: "{% if kolla_base_distro == 'centos' %}xena-20230214T150257{% else %}xena-20230215T164524{% endif %}" # Docker tag applied to built container images. Default is # {{ kolla_openstack_release }}. diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index f0d66d241..0b1c0d62c 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -10,18 +10,7 @@ enable_docker_repo: {% raw %}"{{ 'overcloud' not in group_names or ansible_facts {% if kolla_base_distro == 'centos' %} bifrost_tag: xena-20230214T165534 {% else %} -bifrost_tag: xena-20221213T224057 -cinder_tag: xena-20230206T165220 -etcd_tag: xena-20230201T131415 -glance_tag: xena-20230206T165220 -kolla_toolbox_tag: xena-20230104T145414 -magnum_tag: xena-20230131T151601 -neutron_tag: xena-20230104T145414 -nova_tag: xena-20230206T165220 -octavia_tag: xena-20230104T145414 -openvswitch_tag: xena-20230104T145414 -ovn_tag: xena-20230104T145414 -rabbitmq_tag: xena-20230105T145825 +bifrost_tag: xena-20230215T195824 {% endif %} glance_tls_proxy_tag: "{% raw %}{{ haproxy_tag | default(openstack_tag) }}{% endraw %}" From 7c6022db9a209b899d0f6f82206e0f42b4f53163 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Mon, 20 Feb 2023 15:39:54 +0000 Subject: [PATCH 220/292] Specify unified alerting to avoid migration error --- etc/kayobe/kolla/config/grafana.ini | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 etc/kayobe/kolla/config/grafana.ini diff --git a/etc/kayobe/kolla/config/grafana.ini b/etc/kayobe/kolla/config/grafana.ini new file mode 100644 index 000000000..ee8bc0301 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana.ini @@ -0,0 +1,2 @@ +[unified_alerting] +enabled = true From e0570cd56953d967c1057d7bee0ee5b5acd43a37 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 22 Feb 2023 11:22:44 +0000 Subject: [PATCH 221/292] Bump the default worker count Hoping this will help with the slowness when pulling container images across multiple hosts. Credit to @m-bull for noticing the option: https://github.com/pulp/pulp-oci-images/commit/29c29df5bf218bfed80a52316356fc0dc89d7bf1 I've bumped pulp to 3.22 so that we don't have to force pull 3.21 to get an image with this feature. --- etc/kayobe/containers/pulp/settings.py | 1 + etc/kayobe/seed.yml | 5 ++++- releasenotes/notes/update-pulp-3.22-aa485b7e619cd380.yaml | 7 +++++++ 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/update-pulp-3.22-aa485b7e619cd380.yaml diff --git a/etc/kayobe/containers/pulp/settings.py b/etc/kayobe/containers/pulp/settings.py index 90e64ed53..2afa6a08e 100644 --- a/etc/kayobe/containers/pulp/settings.py +++ b/etc/kayobe/containers/pulp/settings.py @@ -1,4 +1,5 @@ CONTENT_ORIGIN='{{ pulp_url }}' +ANALYTICS=False ANSIBLE_API_HOSTNAME='{{ pulp_url }}' ANSIBLE_CONTENT_HOSTNAME='{{ pulp_url }}/pulp/content' TOKEN_AUTH_DISABLED=True diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index 8f497b915..b575ab82a 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -106,11 +106,14 @@ seed_pulp_container: image: pulp/pulp pre: "{{ kayobe_config_path }}/containers/pulp/pre.yml" post: "{{ kayobe_config_path }}/containers/pulp/post.yml" - tag: "{{ '3.21-https' if pulp_enable_tls | bool else '3.21' }}" + tag: "{{ '3.22-https' if pulp_enable_tls | bool else '3.22' }}" network_mode: host # Override deploy_containers_defaults.init == true to ensure # s6-overlay-suexec starts as pid 1 init: false + env: + PULP_CONTENT_WORKERS: "{{ ansible_facts.processor_vcpus * 2 + 1 }}" + PULP_API_WORKERS: "{{ ansible_facts.processor_vcpus * 2 + 1 }}" volumes: - /opt/kayobe/containers/pulp:/etc/pulp - pulp_storage:/var/lib/pulp diff --git a/releasenotes/notes/update-pulp-3.22-aa485b7e619cd380.yaml b/releasenotes/notes/update-pulp-3.22-aa485b7e619cd380.yaml new file mode 100644 index 000000000..d1235e62f --- /dev/null +++ b/releasenotes/notes/update-pulp-3.22-aa485b7e619cd380.yaml @@ -0,0 +1,7 @@ +--- +features: + - Upgrades Pulp from ``3.21`` to ``3.22``. + - Disables Pulp analytics. + - | + Sets Pulp worker based on available CPU cores. This may improve performance + when pulling container images to many hosts simultaneously. From 096447b7a44f78b35b4676bb3914faaf493d0e33 Mon Sep 17 00:00:00 2001 From: Dawud <7688823+technowhizz@users.noreply.github.com> Date: Thu, 23 Feb 2023 09:26:33 +0000 Subject: [PATCH 222/292] Update Openstack Dashboard (#406) * Update Openstack dashboard Updates the Openstack dashboard to show http 300 status codes as green as this is the expected return code from some services. Change-Id: I19f62d5a1ec697536ad93cf80291930f72b70a42 * Add release note * Update releasenotes/notes/openstack-dash-ff7bf179e37c8e4b.yaml Co-authored-by: Mark Goddard --------- Co-authored-by: Mark Goddard --- .../kolla/config/grafana/dashboards/openstack/openstack.json | 2 +- releasenotes/notes/openstack-dash-ff7bf179e37c8e4b.yaml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/openstack-dash-ff7bf179e37c8e4b.yaml diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/openstack.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/openstack.json index 28a1fcb3f..c75cae2b2 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/openstack/openstack.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/openstack.json @@ -2168,7 +2168,7 @@ }, { "color": "#EAB839", - "value": 300 + "value": 301 }, { "color": "red", diff --git a/releasenotes/notes/openstack-dash-ff7bf179e37c8e4b.yaml b/releasenotes/notes/openstack-dash-ff7bf179e37c8e4b.yaml new file mode 100644 index 000000000..4a6324743 --- /dev/null +++ b/releasenotes/notes/openstack-dash-ff7bf179e37c8e4b.yaml @@ -0,0 +1,5 @@ +--- +other: + - | + Changes the Grafana OpenStack dashboard to show HTTP status 300 as green instead of + orange. From 5948cd958eb961520a76e3cb30545428f250dc4e Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Thu, 23 Feb 2023 09:42:31 +0000 Subject: [PATCH 223/292] Bump neutron containers Brings in the following networking-generic-switch-changes: - Adds support for DellOS10 - Adds support for trunk ports --- etc/kayobe/kolla/globals.yml | 2 ++ ...s-support-for-trunk-ports-7301a258dca0c469.yaml | 14 ++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 releasenotes/notes/adds-support-for-trunk-ports-7301a258dca0c469.yaml diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 0b1c0d62c..d2e1a4f0d 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -9,8 +9,10 @@ enable_docker_repo: {% raw %}"{{ 'overcloud' not in group_names or ansible_facts {% if kolla_base_distro == 'centos' %} bifrost_tag: xena-20230214T165534 +neutron_tag: xena-20230222T141338 {% else %} bifrost_tag: xena-20230215T195824 +neutron_tag: xena-20230222T150855 {% endif %} glance_tls_proxy_tag: "{% raw %}{{ haproxy_tag | default(openstack_tag) }}{% endraw %}" diff --git a/releasenotes/notes/adds-support-for-trunk-ports-7301a258dca0c469.yaml b/releasenotes/notes/adds-support-for-trunk-ports-7301a258dca0c469.yaml new file mode 100644 index 000000000..b604d0207 --- /dev/null +++ b/releasenotes/notes/adds-support-for-trunk-ports-7301a258dca0c469.yaml @@ -0,0 +1,14 @@ +--- +features: + - | + Updates neutron containers to contain a version of + networking-generic-switch with support for trunk ports when using DellOS 10 + or Cisco switches. See this `PR + `_ for more + details. + - | + Updates neutron containers to contain a version of + networking-generic-switch with support for DellOS 10. See this `PR + `_ for more + details. + From 136650097a0bc92069b9478a88821404975081f9 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Tue, 28 Feb 2023 06:50:43 +0100 Subject: [PATCH 224/292] Fix build of release notes Because we turn warnings into errors (sphinx-build -W), the following warning breaks the build: stackhpc-kayobe-config/releasenotes/source/xena.rst:3: WARNING: Duplicate explicit target name: "pr". This is caused by the use of a single underscore at the end of links using the same reference: in this case, "PR", resulting in the duplicate target name "pr". Switch to anonymous hyperlink references with two underscores instead of one [1]. [1] https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#anonymous-hyperlinks --- .../notes/adds-support-for-trunk-ports-7301a258dca0c469.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/releasenotes/notes/adds-support-for-trunk-ports-7301a258dca0c469.yaml b/releasenotes/notes/adds-support-for-trunk-ports-7301a258dca0c469.yaml index b604d0207..2146c460d 100644 --- a/releasenotes/notes/adds-support-for-trunk-ports-7301a258dca0c469.yaml +++ b/releasenotes/notes/adds-support-for-trunk-ports-7301a258dca0c469.yaml @@ -4,11 +4,10 @@ features: Updates neutron containers to contain a version of networking-generic-switch with support for trunk ports when using DellOS 10 or Cisco switches. See this `PR - `_ for more + `__ for more details. - | Updates neutron containers to contain a version of networking-generic-switch with support for DellOS 10. See this `PR - `_ for more + `__ for more details. - From 2f0f20c15e67626d237bf68f3555796726714132 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Tue, 28 Feb 2023 06:53:18 +0100 Subject: [PATCH 225/292] Remove extra new lines in release notes --- releasenotes/notes/bugfix-smart-rules-53ff5400e7f681c8.yaml | 4 +--- releasenotes/notes/kolla-local-builds-eca0277a969803fd.yaml | 1 - releasenotes/notes/smart-mon-db8fa642c3af74b1.yaml | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/releasenotes/notes/bugfix-smart-rules-53ff5400e7f681c8.yaml b/releasenotes/notes/bugfix-smart-rules-53ff5400e7f681c8.yaml index 8da7f80fc..50edfe142 100644 --- a/releasenotes/notes/bugfix-smart-rules-53ff5400e7f681c8.yaml +++ b/releasenotes/notes/bugfix-smart-rules-53ff5400e7f681c8.yaml @@ -1,6 +1,4 @@ --- - fixes: - | - Fixed a syntax error in Prometheus SMART monitoring rules - + Fixed a syntax error in Prometheus SMART monitoring rules. diff --git a/releasenotes/notes/kolla-local-builds-eca0277a969803fd.yaml b/releasenotes/notes/kolla-local-builds-eca0277a969803fd.yaml index a472ab6fa..810295354 100644 --- a/releasenotes/notes/kolla-local-builds-eca0277a969803fd.yaml +++ b/releasenotes/notes/kolla-local-builds-eca0277a969803fd.yaml @@ -5,4 +5,3 @@ features: when set to ``true``, configures the local Pulp server to sync all package repositories required for building kolla containers on a local kolla build host. - diff --git a/releasenotes/notes/smart-mon-db8fa642c3af74b1.yaml b/releasenotes/notes/smart-mon-db8fa642c3af74b1.yaml index feaec4dbe..0d4e0ace3 100644 --- a/releasenotes/notes/smart-mon-db8fa642c3af74b1.yaml +++ b/releasenotes/notes/smart-mon-db8fa642c3af74b1.yaml @@ -1,4 +1,3 @@ --- - features: - Enables SMART monitoring. Manual action is required, please see the monitoring documentation for the procedure. From b9cbe9f78766866d34ea8b2d23b0e122abe8e730 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 1 Mar 2023 14:22:41 +0000 Subject: [PATCH 226/292] Fix release train 02-2023 renos --- ...ase-train-02-2023-package-refresh-9de8049f8cb08a5e.yaml | 7 +++++++ ...in-02-2023-package-refresh-ubuntu-a9fe8a1c3c2f2796.yaml | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/releasenotes/notes/release-train-02-2023-package-refresh-9de8049f8cb08a5e.yaml b/releasenotes/notes/release-train-02-2023-package-refresh-9de8049f8cb08a5e.yaml index 72a3318e0..b8c79e5e2 100644 --- a/releasenotes/notes/release-train-02-2023-package-refresh-9de8049f8cb08a5e.yaml +++ b/releasenotes/notes/release-train-02-2023-package-refresh-9de8049f8cb08a5e.yaml @@ -5,14 +5,21 @@ features: Key packages to note are: * Kernel + * version: 4.18.0 * release: 448.el8 + * Libvirt + * version: 8.0.0 * release: 6.module_el8.7.0+1140+ff0772f9 + * OVS + * version: 2.17.0 * release: 71.el8s + * OVN + * version: 22.09.0 * release: 11.el8s diff --git a/releasenotes/notes/release-train-02-2023-package-refresh-ubuntu-a9fe8a1c3c2f2796.yaml b/releasenotes/notes/release-train-02-2023-package-refresh-ubuntu-a9fe8a1c3c2f2796.yaml index 722b68034..0ea0b4c9f 100644 --- a/releasenotes/notes/release-train-02-2023-package-refresh-ubuntu-a9fe8a1c3c2f2796.yaml +++ b/releasenotes/notes/release-train-02-2023-package-refresh-ubuntu-a9fe8a1c3c2f2796.yaml @@ -5,11 +5,16 @@ features: Key packages to note are: * Libvirt (unchanged since last container build) + * version: 6.0.0 * release: 0ubuntu8.16 + * OVS + * version: 2.17.3 * release: 0ubuntu0.22.04.1~cloud0 + * OVN (unchanged since last container build) + * version: 22.03.0 * release: 0ubuntu1~cloud0 From 0a2a0e15d9eccc113dad9b5a6d12bd0bac0e262b Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 1 Mar 2023 14:24:33 +0000 Subject: [PATCH 227/292] release notes: Remove default_branch config option This option seemed to cause a few problems: 1. The GitHub PR CI used the release branch, so does not test changes to release notes 2. Only tags that have not yet been merged to later release branches appeared in the release notes. Things look better after removing the option. --- releasenotes/config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/releasenotes/config.yaml b/releasenotes/config.yaml index 5dd9a157a..f42aaa466 100644 --- a/releasenotes/config.yaml +++ b/releasenotes/config.yaml @@ -1,4 +1,3 @@ --- # This needs to be updated to the latest release. -default_branch: stackhpc/xena release_tag_re: stackhpc/11\.\d+\.\d+\.\d From ac835014d23cf97a961dbe9dd732baa1dec8c37a Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 1 Mar 2023 14:30:49 +0000 Subject: [PATCH 228/292] Use yoga upper constraints for releasenotes tox env This allows building on Python 3.10, and is what we do for the docs env already. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index b4dd6e620..a34da4934 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ commands = allowlist_externals = rm skip_install = true deps = - -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/xena} + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/yoga} -r{toxinidir}/releasenotes/requirements.txt commands = rm -rf releasenotes/build/html From 0dc8f2de5a469bdbc2d0ce16a99e7d82378b8697 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Tue, 28 Feb 2023 09:53:26 +0000 Subject: [PATCH 229/292] support VMs as compute/controllers in multinode This change allows ansible facts to be used in group_vars/*/network-interfaces, which can be used to select the default network interface. This means different flavors and images can be used without needing to manually change the interface. It also reduces the MTU of all networks except external. --- etc/kayobe/ansible/configure-vxlan.yml | 2 +- .../inventory/group_vars/compute/network-interfaces | 2 +- .../group_vars/controllers/network-interfaces | 2 +- etc/kayobe/environments/ci-multinode/networks.yml | 12 ++++++------ .../add-vm-overcloud-support-92baa7c2ce87ac1d.yaml | 7 +++++++ 5 files changed, 16 insertions(+), 9 deletions(-) create mode 100644 releasenotes/notes/add-vm-overcloud-support-92baa7c2ce87ac1d.yaml diff --git a/etc/kayobe/ansible/configure-vxlan.yml b/etc/kayobe/ansible/configure-vxlan.yml index b43c8b65d..a801c5418 100644 --- a/etc/kayobe/ansible/configure-vxlan.yml +++ b/etc/kayobe/ansible/configure-vxlan.yml @@ -1,7 +1,7 @@ --- - name: Configure VXLAN hosts: controllers,compute,seed,storage - gather_facts: false + gather_facts: true vars: ansible_user: "{{ bootstrap_user }}" # We can't assume that a virtualenv exists at this point, so use the system diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces index aab2482b8..96c6f15c3 100644 --- a/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces @@ -2,7 +2,7 @@ ############################################################################### # Network interface definitions for the compute group. -admin_oc_interface: "eno1" +admin_oc_interface: "{{ ansible_facts.default_ipv4.interface }}" internal_interface: "{{ vxlan_interfaces[0].device }}.{{ internal_vlan }}" diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces index 2dab6c5a8..d02d17ead 100644 --- a/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces @@ -2,7 +2,7 @@ ############################################################################### # Network interface definitions for the controller group. -admin_oc_interface: "eno1" +admin_oc_interface: "{{ ansible_facts.default_ipv4.interface }}" internal_interface: "{{ vxlan_interfaces[0].device }}.{{ internal_vlan }}" diff --git a/etc/kayobe/environments/ci-multinode/networks.yml b/etc/kayobe/environments/ci-multinode/networks.yml index 86406bfad..6afa05758 100644 --- a/etc/kayobe/environments/ci-multinode/networks.yml +++ b/etc/kayobe/environments/ci-multinode/networks.yml @@ -73,7 +73,7 @@ storage_mgmt_net_name: storage_mgmt # Internal network internal_cidr: 192.168.37.0/24 -internal_mtu: 1450 +internal_mtu: 1392 internal_allocation_pool_start: 192.168.37.3 internal_allocation_pool_end: 192.168.37.254 internal_vip_address: 192.168.37.2 @@ -88,7 +88,7 @@ external_vlan: 102 # Public network public_cidr: 192.168.39.0/24 -public_mtu: 1450 +public_mtu: 1392 public_allocation_pool_start: 192.168.39.3 public_allocation_pool_end: 192.168.39.254 public_vip_address: 192.168.39.2 @@ -96,28 +96,28 @@ public_vlan: 103 # Tunnel network tunnel_cidr: 192.168.40.0/24 -tunnel_mtu: 1450 +tunnel_mtu: 1392 tunnel_allocation_pool_start: 192.168.40.3 tunnel_allocation_pool_end: 192.168.40.254 tunnel_vlan: 104 # Storage network storage_cidr: 192.168.41.0/24 -storage_mtu: 1450 +storage_mtu: 1392 storage_allocation_pool_start: 192.168.41.3 storage_allocation_pool_end: 192.168.41.254 storage_vlan: 105 # Storage management network storage_mgmt_cidr: 192.168.42.0/24 -storage_mgmt_mtu: 1450 +storage_mgmt_mtu: 1392 storage_mgmt_allocation_pool_start: 192.168.42.3 storage_mgmt_allocation_pool_end: 192.168.42.254 storage_mgmt_vlan: 106 # Provision overcloud network provision_oc_cidr: 192.168.33.0/24 -provision_oc_mtu: 1450 +provision_oc_mtu: 1392 provision_oc_allocation_pool_start: 192.168.33.128 provision_oc_allocation_pool_end: 192.168.33.254 provision_oc_vlan: 107 diff --git a/releasenotes/notes/add-vm-overcloud-support-92baa7c2ce87ac1d.yaml b/releasenotes/notes/add-vm-overcloud-support-92baa7c2ce87ac1d.yaml new file mode 100644 index 000000000..a1f18e10f --- /dev/null +++ b/releasenotes/notes/add-vm-overcloud-support-92baa7c2ce87ac1d.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Adds support for using a VMs as compute and controller nodes in the + multinode-ci environment by decreasing the MTU of the networks in + networks.yml to 1392 and removing the static definition of the network + interfaces for the compute and controller groups. From 67748f612a9bd201813df2c2f19d94e027f79fb9 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Tue, 28 Feb 2023 13:23:24 +0000 Subject: [PATCH 230/292] un-disable monitoring in the multinode environment --- etc/kayobe/environments/ci-multinode/kolla.yml | 4 ---- etc/kayobe/environments/ci-multinode/kolla/globals.yml | 3 +++ .../notes/add-monitoring-to-multinode-958603f026edbc7c.yaml | 4 ++++ 3 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 releasenotes/notes/add-monitoring-to-multinode-958603f026edbc7c.yaml diff --git a/etc/kayobe/environments/ci-multinode/kolla.yml b/etc/kayobe/environments/ci-multinode/kolla.yml index e645b971c..64c472fb4 100644 --- a/etc/kayobe/environments/ci-multinode/kolla.yml +++ b/etc/kayobe/environments/ci-multinode/kolla.yml @@ -9,7 +9,3 @@ kolla_enable_octavia: true # These are set to true within etc/kayobe/kolla.yml. # Leaving them set to true would require additional # configuration change. -kolla_enable_central_logging: false -kolla_enable_elasticsearch_curator: false -kolla_enable_grafana: false -kolla_enable_prometheus: false diff --git a/etc/kayobe/environments/ci-multinode/kolla/globals.yml b/etc/kayobe/environments/ci-multinode/kolla/globals.yml index 8d020e349..bd726ddc5 100644 --- a/etc/kayobe/environments/ci-multinode/kolla/globals.yml +++ b/etc/kayobe/environments/ci-multinode/kolla/globals.yml @@ -15,6 +15,9 @@ nova_backend_ceph: "yes" neutron_bridge_name: "{{ vxlan_interfaces[0].device }}-ovs" neutron_external_interface: "{{ vxlan_interfaces[0].device }}" +# Elasticsearch memory tuning +es_heap_size: 1g + # Octavia load balancer configuration octavia_auto_configure: "no" octavia_provider_drivers: "ovn:OVN provider" diff --git a/releasenotes/notes/add-monitoring-to-multinode-958603f026edbc7c.yaml b/releasenotes/notes/add-monitoring-to-multinode-958603f026edbc7c.yaml new file mode 100644 index 000000000..4f3b70639 --- /dev/null +++ b/releasenotes/notes/add-monitoring-to-multinode-958603f026edbc7c.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Set monitoring services be enabled by default in the multinode environment. From 1597744d1967cd9ac2d2a1aa8109781b7c405e8a Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Tue, 28 Feb 2023 13:54:10 +0000 Subject: [PATCH 231/292] fix VM internet connectivity in multinode This fix sets seed_enable_snat and fixes the external IP of the seed which (in conjunction with an update to the multinode openstack configuration to set the gateway on the external subnet) allows VMs to reach the internet. --- etc/kayobe/environments/ci-multinode/network-allocation.yml | 2 ++ etc/kayobe/environments/ci-multinode/seed.yml | 2 ++ ...-multinode-vm-internet-connectivity-49ab64cd75833f09.yaml | 5 +++++ 3 files changed, 9 insertions(+) create mode 100644 etc/kayobe/environments/ci-multinode/network-allocation.yml create mode 100644 releasenotes/notes/fix-multinode-vm-internet-connectivity-49ab64cd75833f09.yaml diff --git a/etc/kayobe/environments/ci-multinode/network-allocation.yml b/etc/kayobe/environments/ci-multinode/network-allocation.yml new file mode 100644 index 000000000..827597b84 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/network-allocation.yml @@ -0,0 +1,2 @@ +external_ips: + alex-mn-seed: 192.168.38.3 \ No newline at end of file diff --git a/etc/kayobe/environments/ci-multinode/seed.yml b/etc/kayobe/environments/ci-multinode/seed.yml index 1ae71f1a4..fe72e7c31 100644 --- a/etc/kayobe/environments/ci-multinode/seed.yml +++ b/etc/kayobe/environments/ci-multinode/seed.yml @@ -8,3 +8,5 @@ seed_lvm_groups: seed_extra_network_interfaces: - "external" - "public" + +seed_enable_snat: true diff --git a/releasenotes/notes/fix-multinode-vm-internet-connectivity-49ab64cd75833f09.yaml b/releasenotes/notes/fix-multinode-vm-internet-connectivity-49ab64cd75833f09.yaml new file mode 100644 index 000000000..7f34afca7 --- /dev/null +++ b/releasenotes/notes/fix-multinode-vm-internet-connectivity-49ab64cd75833f09.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes internet connectivity for VMs deployed in the ci-multinode + environment. From e6a057722b46105845e5143b2ce4601c28a92335 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Wed, 1 Mar 2023 14:19:50 +0000 Subject: [PATCH 232/292] make ci-multinode network MTUs dynamic --- etc/kayobe/environments/ci-multinode/kolla.yml | 5 ----- .../environments/ci-multinode/network-allocation.yml | 2 -- etc/kayobe/environments/ci-multinode/networks.yml | 12 ++++++------ etc/kayobe/environments/ci-multinode/seed.yml | 2 ++ ...add-monitoring-to-multinode-958603f026edbc7c.yaml | 3 ++- .../add-vm-overcloud-support-92baa7c2ce87ac1d.yaml | 4 ++-- ...de-vm-internet-connectivity-49ab64cd75833f09.yaml | 2 +- 7 files changed, 13 insertions(+), 17 deletions(-) delete mode 100644 etc/kayobe/environments/ci-multinode/network-allocation.yml diff --git a/etc/kayobe/environments/ci-multinode/kolla.yml b/etc/kayobe/environments/ci-multinode/kolla.yml index 64c472fb4..5c6c80dae 100644 --- a/etc/kayobe/environments/ci-multinode/kolla.yml +++ b/etc/kayobe/environments/ci-multinode/kolla.yml @@ -4,8 +4,3 @@ kolla_enable_cinder_backup: true kolla_enable_neutron_provider_networks: true kolla_enable_ovn: true kolla_enable_octavia: true - -# Override these from etc/kayobe/kolla.yml -# These are set to true within etc/kayobe/kolla.yml. -# Leaving them set to true would require additional -# configuration change. diff --git a/etc/kayobe/environments/ci-multinode/network-allocation.yml b/etc/kayobe/environments/ci-multinode/network-allocation.yml deleted file mode 100644 index 827597b84..000000000 --- a/etc/kayobe/environments/ci-multinode/network-allocation.yml +++ /dev/null @@ -1,2 +0,0 @@ -external_ips: - alex-mn-seed: 192.168.38.3 \ No newline at end of file diff --git a/etc/kayobe/environments/ci-multinode/networks.yml b/etc/kayobe/environments/ci-multinode/networks.yml index 6afa05758..8d1d93c2c 100644 --- a/etc/kayobe/environments/ci-multinode/networks.yml +++ b/etc/kayobe/environments/ci-multinode/networks.yml @@ -73,7 +73,7 @@ storage_mgmt_net_name: storage_mgmt # Internal network internal_cidr: 192.168.37.0/24 -internal_mtu: 1392 +internal_mtu: "{{ ansible_facts.default_ipv4.mtu - 50 }}" internal_allocation_pool_start: 192.168.37.3 internal_allocation_pool_end: 192.168.37.254 internal_vip_address: 192.168.37.2 @@ -88,7 +88,7 @@ external_vlan: 102 # Public network public_cidr: 192.168.39.0/24 -public_mtu: 1392 +public_mtu: "{{ ansible_facts.default_ipv4.mtu - 50 }}" public_allocation_pool_start: 192.168.39.3 public_allocation_pool_end: 192.168.39.254 public_vip_address: 192.168.39.2 @@ -96,28 +96,28 @@ public_vlan: 103 # Tunnel network tunnel_cidr: 192.168.40.0/24 -tunnel_mtu: 1392 +tunnel_mtu: "{{ ansible_facts.default_ipv4.mtu - 50 }}" tunnel_allocation_pool_start: 192.168.40.3 tunnel_allocation_pool_end: 192.168.40.254 tunnel_vlan: 104 # Storage network storage_cidr: 192.168.41.0/24 -storage_mtu: 1392 +storage_mtu: "{{ ansible_facts.default_ipv4.mtu - 50 }}" storage_allocation_pool_start: 192.168.41.3 storage_allocation_pool_end: 192.168.41.254 storage_vlan: 105 # Storage management network storage_mgmt_cidr: 192.168.42.0/24 -storage_mgmt_mtu: 1392 +storage_mgmt_mtu: "{{ ansible_facts.default_ipv4.mtu - 50 }}" storage_mgmt_allocation_pool_start: 192.168.42.3 storage_mgmt_allocation_pool_end: 192.168.42.254 storage_mgmt_vlan: 106 # Provision overcloud network provision_oc_cidr: 192.168.33.0/24 -provision_oc_mtu: 1392 +provision_oc_mtu: "{{ ansible_facts.default_ipv4.mtu - 50 }}" provision_oc_allocation_pool_start: 192.168.33.128 provision_oc_allocation_pool_end: 192.168.33.254 provision_oc_vlan: 107 diff --git a/etc/kayobe/environments/ci-multinode/seed.yml b/etc/kayobe/environments/ci-multinode/seed.yml index fe72e7c31..3eb7afa90 100644 --- a/etc/kayobe/environments/ci-multinode/seed.yml +++ b/etc/kayobe/environments/ci-multinode/seed.yml @@ -9,4 +9,6 @@ seed_extra_network_interfaces: - "external" - "public" +# Enable IP routing and source NAT on the seed, allowing it to be used as the +# external subnet gateway and provide internet access for VMs in the deployment. seed_enable_snat: true diff --git a/releasenotes/notes/add-monitoring-to-multinode-958603f026edbc7c.yaml b/releasenotes/notes/add-monitoring-to-multinode-958603f026edbc7c.yaml index 4f3b70639..0a985519c 100644 --- a/releasenotes/notes/add-monitoring-to-multinode-958603f026edbc7c.yaml +++ b/releasenotes/notes/add-monitoring-to-multinode-958603f026edbc7c.yaml @@ -1,4 +1,5 @@ --- features: - | - Set monitoring services be enabled by default in the multinode environment. + Set monitoring services be enabled by default in the ``ci-multinode`` + environment. diff --git a/releasenotes/notes/add-vm-overcloud-support-92baa7c2ce87ac1d.yaml b/releasenotes/notes/add-vm-overcloud-support-92baa7c2ce87ac1d.yaml index a1f18e10f..a3ebacc1f 100644 --- a/releasenotes/notes/add-vm-overcloud-support-92baa7c2ce87ac1d.yaml +++ b/releasenotes/notes/add-vm-overcloud-support-92baa7c2ce87ac1d.yaml @@ -2,6 +2,6 @@ features: - | Adds support for using a VMs as compute and controller nodes in the - multinode-ci environment by decreasing the MTU of the networks in - networks.yml to 1392 and removing the static definition of the network + ``ci-multinode`` environment by dynamically setting the MTU of the networks + in networks.yml and removing the static definition of the network interfaces for the compute and controller groups. diff --git a/releasenotes/notes/fix-multinode-vm-internet-connectivity-49ab64cd75833f09.yaml b/releasenotes/notes/fix-multinode-vm-internet-connectivity-49ab64cd75833f09.yaml index 7f34afca7..007de8a76 100644 --- a/releasenotes/notes/fix-multinode-vm-internet-connectivity-49ab64cd75833f09.yaml +++ b/releasenotes/notes/fix-multinode-vm-internet-connectivity-49ab64cd75833f09.yaml @@ -1,5 +1,5 @@ --- fixes: - | - Fixes internet connectivity for VMs deployed in the ci-multinode + Fixes internet connectivity for VMs deployed in the ``ci-multinode`` environment. From 3f2b4d469fbb74ac9c90554facd6519937bacc00 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 1 Mar 2023 14:54:16 +0000 Subject: [PATCH 233/292] Run doc8 and reno lint in pep8 tox env --- doc/source/configuration/host-images.rst | 2 +- doc/source/configuration/monitoring.rst | 2 +- doc/source/configuration/release-train.rst | 8 ++++---- doc/source/configuration/walled-garden.rst | 2 +- doc/source/contributor/environments/ci-multinode.rst | 8 ++++---- test-requirements.txt | 2 ++ tox.ini | 2 ++ 7 files changed, 15 insertions(+), 11 deletions(-) diff --git a/doc/source/configuration/host-images.rst b/doc/source/configuration/host-images.rst index cfa5fe678..ba35a244f 100644 --- a/doc/source/configuration/host-images.rst +++ b/doc/source/configuration/host-images.rst @@ -43,7 +43,7 @@ The 3 partitions are: * p0: EFI ESP bootloader * p1: EFI BSP * p2: LVM PV (``rootpv``) - + The LVM Logical Volumes are: ============== ================== ========= diff --git a/doc/source/configuration/monitoring.rst b/doc/source/configuration/monitoring.rst index c2384ba93..2b336c699 100644 --- a/doc/source/configuration/monitoring.rst +++ b/doc/source/configuration/monitoring.rst @@ -126,4 +126,4 @@ need to be set, and the proxy URL sould be set or removed. If you want to add an alerting rule, there are many good examples of alerts are available `here `__. They simply need to be added to one of the ``*.rules`` files in the prometheus configuration -directory. \ No newline at end of file +directory. diff --git a/doc/source/configuration/release-train.rst b/doc/source/configuration/release-train.rst index af1e7b3f9..d4757f54f 100644 --- a/doc/source/configuration/release-train.rst +++ b/doc/source/configuration/release-train.rst @@ -232,7 +232,7 @@ see this message when you later try to run ``pulp-container-sync.yml``: The issue is that pushing an image automatically creates a `container-push repository `__ which conflicts with the creation of a regular container repository of the same -name. You can resolve this conflict by deleting the distribution associated +name. You can resolve this conflict by deleting the distribution associated with the push repository using the pulp CLI: .. code-block:: console @@ -241,17 +241,17 @@ with the push repository using the pulp CLI: Started background task /pulp/api/v3/tasks/1f0a474a-b7c0-44b4-9ef4-ed633077f4d8/ .Done. -HTTP Error 404: Not Found +HTTP Error 404: Not Found ~~~~~~~~~~~~~~~~~~~~~~~~~ -If your login credentials are incorrect, or lack the required permissions, +If your login credentials are incorrect, or lack the required permissions, you will see a 404 error during ``pulp-repo-sync.yml``: .. code-block:: console TASK [stackhpc.pulp.pulp_repository : Sync RPM remotes into repositories] **************************************************************************************************************************************** An exception occurred during task execution. To see the full traceback, use -vvv. The error was: Exception: Task failed to complete. (failed; 404, message='Not Found', url=URL('https://ark.stackhpc.com/pulp/content/centos/8-stream/BaseOS/x86_64/os/20211122T102435')) - failed: [localhost] (item=centos-stream-8-baseos-development) => changed=false + failed: [localhost] (item=centos-stream-8-baseos-development) => changed=false ansible_loop_var: item item: name: centos-stream-8-baseos-development diff --git a/doc/source/configuration/walled-garden.rst b/doc/source/configuration/walled-garden.rst index 0df8ef8b5..9a45ea4db 100644 --- a/doc/source/configuration/walled-garden.rst +++ b/doc/source/configuration/walled-garden.rst @@ -49,7 +49,7 @@ In some environments we have found that squid’s preference for IPv6 can cause problems. It can be forced to prefer IPv4, by adding the following in ``etc/kayobe/containers/squid_proxy/squid.conf``: -.. code:: none +.. code:: dns_v4_first on diff --git a/doc/source/contributor/environments/ci-multinode.rst b/doc/source/contributor/environments/ci-multinode.rst index 72aecc3e7..0dd787140 100644 --- a/doc/source/contributor/environments/ci-multinode.rst +++ b/doc/source/contributor/environments/ci-multinode.rst @@ -1,5 +1,5 @@ ========================== -Multinode Test Environment +Multinode Test Environment ========================== Set up hosts @@ -21,7 +21,7 @@ Basic Kayobe Setup 2. ``sudo dnf install -y python3-virtualenv`` 3. ``mkdir src`` and ``cd src`` 4. Clone https://github.com/stackhpc/stackhpc-kayobe-config.git, then checkout - commit f31df6256f1b1fea99c84547d44f06c4cb74b161 + commit f31df6256f1b1fea99c84547d44f06c4cb74b161 5. ``cd ..`` and ``mkdir venvs`` 6. ``virtualenv venvs/kayobe`` and source ``venvs/kayobe/bin/activate`` 7. ``pip install -U pip`` @@ -39,8 +39,8 @@ Config changes 4. Also under vxlan_interfaces, check vxlan_dstport is not 4789 (this causes conflicts, change to 4790) 5. In /etc/kayobe/environments/ci-multinode/tf-networks.yml, edit admin_ips so - that the compute and controller IPs line up with the - instances that were created earlier, remove the other IPs for seed and + that the compute and controller IPs line up with the + instances that were created earlier, remove the other IPs for seed and cephOSD 6. In /etc/kayobe/environments/ci-multinode/network-allocation.yml, remove all the entries and just assign ``aio_ips:`` an empty set ``[]`` diff --git a/test-requirements.txt b/test-requirements.txt index 333d31836..d7f05bb0d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,3 +3,5 @@ # process, which may cause wedges in the gate later. yamllint # GPLv3 +doc8 # Apache-2.0 +reno>=3.4.0 # Apache-2.0 diff --git a/tox.ini b/tox.ini index b4dd6e620..7778e421d 100644 --- a/tox.ini +++ b/tox.ini @@ -12,6 +12,8 @@ deps = [testenv:pep8] commands = yamllint etc/kayobe + reno lint + doc8 README.rst doc/source --ignore D001 # StackHPC Kayobe configuration release notes: [testenv:releasenotes] From f52fe3fb0f6539013165c1e1ebd2074adfa6927f Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Fri, 3 Feb 2023 18:06:28 +0000 Subject: [PATCH 234/292] Reduce concurrency --- .automation.conf/config.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.automation.conf/config.sh b/.automation.conf/config.sh index e8b88728b..bfb98f9b0 100644 --- a/.automation.conf/config.sh +++ b/.automation.conf/config.sh @@ -19,10 +19,15 @@ if [ ! -z ${KAYOBE_ENVIRONMENT:+x} ]; then # Seem to get servers failing to spawn with higher concurrency export TEMPEST_CONCURRENCY=1 fi + if [[ "$KAYOBE_ENVIRONMENT" =~ "ci-multinode" ]]; then - export KAYOBE_AUTOMATION_TEMPEST_LOADLIST=tempest-full - export KAYOBE_AUTOMATION_TEMPEST_SKIPLIST=ci-multinode + # SMSLab is currently running with 1G switches. This causes tests using volumes and images to fail if + # the concurrency is set too high. + export TEMPEST_CONCURRENCY=2 + export KAYOBE_AUTOMATION_TEMPEST_LOADLIST=tempest-full + export KAYOBE_AUTOMATION_TEMPEST_SKIPLIST=ci-multinode fi + fi if [[ -z "${KAYOBE_AUTOMATION_TEMPEST_CONF_OVERRIDES:+x}" ]] || [[ ! -e "${KAYOBE_AUTOMATION_TEMPEST_CONF_OVERRIDES}" ]]; then From b1a3bb0fdaf8bbfd9cca7ceb46d2362ea69fcb43 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 8 Feb 2023 09:55:02 +0000 Subject: [PATCH 235/292] Configure tunnel mtu in neutron This determines the default mtu that neutron will use when networks are created. --- etc/kayobe/environments/ci-multinode/kolla/config/neutron.conf | 3 +++ .../ci-multinode/kolla/config/neutron/ml2_conf.ini | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 etc/kayobe/environments/ci-multinode/kolla/config/neutron.conf create mode 100644 etc/kayobe/environments/ci-multinode/kolla/config/neutron/ml2_conf.ini diff --git a/etc/kayobe/environments/ci-multinode/kolla/config/neutron.conf b/etc/kayobe/environments/ci-multinode/kolla/config/neutron.conf new file mode 100644 index 000000000..265f67b26 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/kolla/config/neutron.conf @@ -0,0 +1,3 @@ +[DEFAULT] +global_physnet_mtu = {{ tunnel_net_name | net_mtu }} + diff --git a/etc/kayobe/environments/ci-multinode/kolla/config/neutron/ml2_conf.ini b/etc/kayobe/environments/ci-multinode/kolla/config/neutron/ml2_conf.ini new file mode 100644 index 000000000..ff31f09a5 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/kolla/config/neutron/ml2_conf.ini @@ -0,0 +1,3 @@ +[ml2] +path_mtu = {{ tunnel_net_name | net_mtu }} + From 54c8d2835f2fd71faf7072030a0f8909cc3198f3 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 8 Feb 2023 12:08:12 +0000 Subject: [PATCH 236/292] Reduce tempest concurrency to 1 --- .automation.conf/config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.automation.conf/config.sh b/.automation.conf/config.sh index bfb98f9b0..4ca6b3b78 100644 --- a/.automation.conf/config.sh +++ b/.automation.conf/config.sh @@ -23,7 +23,7 @@ if [ ! -z ${KAYOBE_ENVIRONMENT:+x} ]; then if [[ "$KAYOBE_ENVIRONMENT" =~ "ci-multinode" ]]; then # SMSLab is currently running with 1G switches. This causes tests using volumes and images to fail if # the concurrency is set too high. - export TEMPEST_CONCURRENCY=2 + export TEMPEST_CONCURRENCY=1 export KAYOBE_AUTOMATION_TEMPEST_LOADLIST=tempest-full export KAYOBE_AUTOMATION_TEMPEST_SKIPLIST=ci-multinode fi From f0c6d4af00cb1bab6329f78dd599e458cd9e81c2 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 8 Feb 2023 12:08:37 +0000 Subject: [PATCH 237/292] Bump timeouts to account for ceph being slow --- .automation.conf/tempest/tempest-ci-multinode.overrides.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.automation.conf/tempest/tempest-ci-multinode.overrides.conf b/.automation.conf/tempest/tempest-ci-multinode.overrides.conf index 5cd61d03b..0007e5d97 100644 --- a/.automation.conf/tempest/tempest-ci-multinode.overrides.conf +++ b/.automation.conf/tempest/tempest-ci-multinode.overrides.conf @@ -23,8 +23,12 @@ volume_backed_live_migration = true console_output = true [volume] +build_timeout = 600 min_microversion = 3.0 max_microversion = 3.66 +[image] +build_timeout = 600 + [dashboard] dashboard_url = http://192.168.39.2 From 3c3c3664db3026676b5bef4b54ab53b173e312f2 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 8 Feb 2023 12:09:36 +0000 Subject: [PATCH 238/292] Bump MTU on external network We need this to match the global_physical_network_mtu set in neutron.conf for some tests to pass correctly. --- etc/kayobe/environments/ci-multinode/networks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/environments/ci-multinode/networks.yml b/etc/kayobe/environments/ci-multinode/networks.yml index 86406bfad..e6ef17fad 100644 --- a/etc/kayobe/environments/ci-multinode/networks.yml +++ b/etc/kayobe/environments/ci-multinode/networks.yml @@ -81,7 +81,7 @@ internal_vlan: 101 # External network external_cidr: 192.168.38.0/24 -external_mtu: 1350 +external_mtu: 1450 external_allocation_pool_start: 192.168.38.3 external_allocation_pool_end: 192.168.38.128 external_vlan: 102 From 210c3c01492af0c46b52d98866f7cbefeb0a922a Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Fri, 10 Feb 2023 18:36:23 +0000 Subject: [PATCH 239/292] Set storage protocol Resolves a few tempest failures: ``` 2023-02-10 18:34:25.310 590 INFO default [-] {1} tempest.api.volume.admin.test_volume_types.VolumeTypesTest.test_volume_crud_with_volume_type_and_extra_specs ... success [4.531s] 2023-02-10 18:34:46.163 590 INFO default [-] {1} tempest.api.volume.test_volume_delete_cascade.VolumesDeleteCascade.test_volume_from_snapshot_cascade_delete ... skip [0.000s]: b'Skip because of Bug#1677525' 2023-02-10 18:34:48.844 590 INFO default [-] {1} tempest.api.volume.test_volumes_backup.VolumesBackupsTest.test_volume_backup_create_get_detailed_list_restore_delete ... skip [0.000s]: b'ceph does not support arbitrary container names' ``` --- .automation.conf/tempest/tempest-ci-multinode.overrides.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/.automation.conf/tempest/tempest-ci-multinode.overrides.conf b/.automation.conf/tempest/tempest-ci-multinode.overrides.conf index 0007e5d97..9c6398074 100644 --- a/.automation.conf/tempest/tempest-ci-multinode.overrides.conf +++ b/.automation.conf/tempest/tempest-ci-multinode.overrides.conf @@ -23,6 +23,7 @@ volume_backed_live_migration = true console_output = true [volume] +storage_protocol = ceph build_timeout = 600 min_microversion = 3.0 max_microversion = 3.66 From af2dcdb148eeef621a843f4f07539f4f38850739 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Wed, 1 Mar 2023 15:53:29 +0000 Subject: [PATCH 240/292] Clarify which exchanges should remain after the forced reset of RabbitMQ --- doc/source/operations/rabbitmq.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/operations/rabbitmq.rst b/doc/source/operations/rabbitmq.rst index e678fbed9..dc9c3404b 100644 --- a/doc/source/operations/rabbitmq.rst +++ b/doc/source/operations/rabbitmq.rst @@ -128,6 +128,8 @@ Check to see if all OpenStack queues and exchanges have been removed from the Ra kayobe overcloud host command run --limit controllers --show-output --command 'docker exec rabbitmq rabbitmqctl list_queues name' kayobe overcloud host command run --limit controllers --show-output --command 'docker exec rabbitmq rabbitmqctl list_exchanges name' +There should be no queus listed, and the only exchanges listed should start with `amq.`. + Start the OpenStack services which use RabbitMQ. Note that this will start all matching services, even if they weren't running prior to starting this procedure. From d8ac5c749601be1ab3bca06d929f988bf996cace Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Wed, 1 Mar 2023 15:09:26 +0000 Subject: [PATCH 241/292] Change fix-networking.yml so it doesn't overwrite all of /etc/hosts --- etc/kayobe/ansible/fix-networking.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/etc/kayobe/ansible/fix-networking.yml b/etc/kayobe/ansible/fix-networking.yml index f24be3862..0a1736702 100644 --- a/etc/kayobe/ansible/fix-networking.yml +++ b/etc/kayobe/ansible/fix-networking.yml @@ -11,11 +11,10 @@ ansible_ssh_common_args: "-o StrictHostKeyChecking=no" tasks: - name: Ensure `hosts` file contains pulp entries - ansible.builtin.copy: - content: | - 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 - ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + blockinfile: + path: /etc/hosts + marker_begin: BEGIN Kayobe Pulp entries + block: | 10.0.0.34 pelican pelican.service.compute.sms-lab.cloud 10.205.3.187 pulp-server pulp-server.internal.sms-cloud - dest: /etc/hosts become: true From 51017f8cdfd08c06c2b5a8ef85d1ecbfd2a29380 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Wed, 1 Mar 2023 16:02:08 +0000 Subject: [PATCH 242/292] Update doc/source/operations/rabbitmq.rst Co-authored-by: Mark Goddard --- doc/source/operations/rabbitmq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/operations/rabbitmq.rst b/doc/source/operations/rabbitmq.rst index dc9c3404b..12fd5a789 100644 --- a/doc/source/operations/rabbitmq.rst +++ b/doc/source/operations/rabbitmq.rst @@ -128,7 +128,7 @@ Check to see if all OpenStack queues and exchanges have been removed from the Ra kayobe overcloud host command run --limit controllers --show-output --command 'docker exec rabbitmq rabbitmqctl list_queues name' kayobe overcloud host command run --limit controllers --show-output --command 'docker exec rabbitmq rabbitmqctl list_exchanges name' -There should be no queus listed, and the only exchanges listed should start with `amq.`. +There should be no queues listed, and the only exchanges listed should start with `amq.`. Start the OpenStack services which use RabbitMQ. Note that this will start all matching services, even if they weren't running prior to starting this From 79bfa3b7bba46052272149703890c9fb70f8b8ce Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Wed, 1 Mar 2023 16:23:55 +0000 Subject: [PATCH 243/292] Correct marker --- etc/kayobe/ansible/fix-networking.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/fix-networking.yml b/etc/kayobe/ansible/fix-networking.yml index 0a1736702..c992be181 100644 --- a/etc/kayobe/ansible/fix-networking.yml +++ b/etc/kayobe/ansible/fix-networking.yml @@ -13,7 +13,7 @@ - name: Ensure `hosts` file contains pulp entries blockinfile: path: /etc/hosts - marker_begin: BEGIN Kayobe Pulp entries + marker: "# {mark} Kayobe Pulp entries" block: | 10.0.0.34 pelican pelican.service.compute.sms-lab.cloud 10.205.3.187 pulp-server pulp-server.internal.sms-cloud From b1f78d34b5fc6602d94a15969bf24c5bd83f0bf9 Mon Sep 17 00:00:00 2001 From: Dawud <7688823+technowhizz@users.noreply.github.com> Date: Wed, 1 Mar 2023 19:19:18 +0000 Subject: [PATCH 244/292] Fix InstanceDown alert (#419) Fixes the InstanceDown alerting rule wait time to be consistent with the alert message. The alert message says "for 5 minutes" but the rule was set to wait for 1 minute. --- etc/kayobe/kolla/config/prometheus/system.rules | 2 +- .../notes/fixes-InstanceDown-Alert-570a295e3d5006f7.yaml | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/fixes-InstanceDown-Alert-570a295e3d5006f7.yaml diff --git a/etc/kayobe/kolla/config/prometheus/system.rules b/etc/kayobe/kolla/config/prometheus/system.rules index be1343b82..f693a250c 100644 --- a/etc/kayobe/kolla/config/prometheus/system.rules +++ b/etc/kayobe/kolla/config/prometheus/system.rules @@ -53,7 +53,7 @@ groups: - alert: InstanceDown expr: up{job="node"} == 0 - for: 1m + for: 5m labels: severity: alert annotations: diff --git a/releasenotes/notes/fixes-InstanceDown-Alert-570a295e3d5006f7.yaml b/releasenotes/notes/fixes-InstanceDown-Alert-570a295e3d5006f7.yaml new file mode 100644 index 000000000..60be7d77b --- /dev/null +++ b/releasenotes/notes/fixes-InstanceDown-Alert-570a295e3d5006f7.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Fixes the InstanceDown alerting rule wait time to be consistent with + the alert message. The alert message says "for 5 minutes" but the rule + was set to wait for 1 minute. + From 388c3811ac1923bb52b8fc78f0f7e220d69f03f5 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Tue, 7 Mar 2023 21:41:32 +0100 Subject: [PATCH 245/292] Bump Neutron to include IPv6 metadata workaround --- etc/kayobe/kolla/globals.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index d2e1a4f0d..6a38da4ab 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -9,10 +9,10 @@ enable_docker_repo: {% raw %}"{{ 'overcloud' not in group_names or ansible_facts {% if kolla_base_distro == 'centos' %} bifrost_tag: xena-20230214T165534 -neutron_tag: xena-20230222T141338 +neutron_tag: xena-20230307T142413 {% else %} bifrost_tag: xena-20230215T195824 -neutron_tag: xena-20230222T150855 +neutron_tag: xena-20230307T142414 {% endif %} glance_tls_proxy_tag: "{% raw %}{{ haproxy_tag | default(openstack_tag) }}{% endraw %}" From c88c90e4c04156ff28bcdce341452b4a1093cb23 Mon Sep 17 00:00:00 2001 From: Piotr Parczewski Date: Wed, 8 Mar 2023 12:53:43 +0100 Subject: [PATCH 246/292] Use newer liblasso package in Ubuntu Fixes https://dev.entrouvert.org/issues/25640 --- etc/kayobe/kolla.yml | 8 ++++++++ .../notes/fix-liblasso-netiq-issue-afec5b2ee7de2a1e.yaml | 5 +++++ 2 files changed, 13 insertions(+) create mode 100644 releasenotes/notes/fix-liblasso-netiq-issue-afec5b2ee7de2a1e.yaml diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 6c8f27ca2..5cf57682c 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -275,6 +275,14 @@ kolla_build_blocks: && grafana-cli plugins install grafana-piechart-panel ironic_inspector_header: | ADD additions-archive / + keystone_base_footer: | + {% raw %} + {% if base_package_type == 'deb' %} + RUN curl -sLO https://github.com/stackhpc/lasso/releases/download/applied%2F2.7.0-2build3/liblasso3_2.7.0-2build3_amd64.deb && \ + sudo dpkg -i liblasso3_2.7.0-2build3_amd64.deb && \ + rm -f liblasso3_2.7.0-2build3_amd64.deb + {% endif %} + {% endraw %} nova_base_footer: | # Fix for https://bugs.launchpad.net/nova/+bug/1955035, i.e. # https://bugzilla.redhat.com/show_bug.cgi?id=2090752 on c8s diff --git a/releasenotes/notes/fix-liblasso-netiq-issue-afec5b2ee7de2a1e.yaml b/releasenotes/notes/fix-liblasso-netiq-issue-afec5b2ee7de2a1e.yaml new file mode 100644 index 000000000..4c71ad67d --- /dev/null +++ b/releasenotes/notes/fix-liblasso-netiq-issue-afec5b2ee7de2a1e.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes the `issue `__ with using + SAML2 federation in Keystone against NetIQ IdP. From f09071e3f5ac637df6db7d9230dbf9c5d3d27e35 Mon Sep 17 00:00:00 2001 From: Piotr Parczewski Date: Wed, 8 Mar 2023 14:10:08 +0100 Subject: [PATCH 247/292] Bump up Keystone's image tag for Ubuntu --- etc/kayobe/kolla/globals.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 6a38da4ab..67c5b1bb6 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -1,8 +1,8 @@ # yamllint disable-file --- -# To work around issue of trying to install docker from -# empty pulp server, use upstream docker dnf repo on +# To work around issue of trying to install docker from +# empty pulp server, use upstream docker dnf repo on # non-overcloud hosts enable_docker_repo: {% raw %}"{{ 'overcloud' not in group_names or ansible_facts.os_family == 'Debian' }}"{% endraw %} @@ -12,6 +12,7 @@ bifrost_tag: xena-20230214T165534 neutron_tag: xena-20230307T142413 {% else %} bifrost_tag: xena-20230215T195824 +keystone_tag: xena-20230308T120251 neutron_tag: xena-20230307T142414 {% endif %} From 5bea347bd89dae1513e5feb36f61f2155e1d381d Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Thu, 9 Mar 2023 15:57:34 +0000 Subject: [PATCH 248/292] fix virtualenv ci failure (cherry picked from commit e36a6fb384e20d6d3d3f2c6634fca80f71cd149f) --- etc/kayobe/ansible/configure-aio-resources.yml | 9 ++++++++- etc/kayobe/environments/ci-aio/kolla/globals.yml | 3 +++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/configure-aio-resources.yml b/etc/kayobe/ansible/configure-aio-resources.yml index 78450eaeb..243472d60 100644 --- a/etc/kayobe/ansible/configure-aio-resources.yml +++ b/etc/kayobe/ansible/configure-aio-resources.yml @@ -16,7 +16,14 @@ state: present become: true - - name: Install python openstack client + - name: Ensure latest version of pip is installed + pip: + name: pip + state: latest + virtualenv: '{{ venv }}' + virtualenv_command: "/usr/bin/python3 -m venv" + + - name: Ensure python openstack client is installed pip: name: python-openstackclient virtualenv: '{{ venv }}' diff --git a/etc/kayobe/environments/ci-aio/kolla/globals.yml b/etc/kayobe/environments/ci-aio/kolla/globals.yml index 843acf785..626ecae08 100644 --- a/etc/kayobe/environments/ci-aio/kolla/globals.yml +++ b/etc/kayobe/environments/ci-aio/kolla/globals.yml @@ -14,3 +14,6 @@ docker_yum_gpgkey: "https://download.docker.com/linux/centos/gpg" # Elasticsearch memory tuning es_heap_size: 1g + +# Increase Grafana timeout +grafana_start_first_node_retries: 20 From 958114d5beff45915ca2952748ca19b8f9278954 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 13 Mar 2023 14:47:03 +0000 Subject: [PATCH 249/292] CI: Install python3-venv on Ubuntu in configure-aio-resources.yml --- etc/kayobe/ansible/configure-aio-resources.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/etc/kayobe/ansible/configure-aio-resources.yml b/etc/kayobe/ansible/configure-aio-resources.yml index 243472d60..48263c641 100644 --- a/etc/kayobe/ansible/configure-aio-resources.yml +++ b/etc/kayobe/ansible/configure-aio-resources.yml @@ -8,13 +8,16 @@ tasks: - name: Install python build dependencies package: - name: - - "{% if ansible_facts.os_family == 'RedHat' %}gcc{% else %}build-essential{% endif %}" - - "python3-dev{% if ansible_facts.os_family == 'RedHat' %}el{% endif %}" + name: "{{ packages | select | list }}" cache_valid_time: "{{ apt_cache_valid_time if ansible_facts.os_family == 'Debian' else omit }}" update_cache: "{{ True if ansible_facts.os_family == 'Debian' else omit }}" state: present become: true + vars: + packages: + - "{% if ansible_facts.os_family == 'RedHat' %}gcc{% else %}build-essential{% endif %}" + - "python3-dev{% if ansible_facts.os_family == 'RedHat' %}el{% endif %}" + - "{% if ansible_facts.os_family == 'Debian' %}python3-venv{% endif %}" - name: Ensure latest version of pip is installed pip: From 7748d7b61492efb24edb799ca0866fcc5bc64608 Mon Sep 17 00:00:00 2001 From: Dawud <7688823+technowhizz@users.noreply.github.com> Date: Tue, 14 Mar 2023 10:35:56 +0000 Subject: [PATCH 250/292] Add unit (GiB) to LowMemory alert (#437) Add unit (GiB) to LowMemory alert --- etc/kayobe/kolla/config/prometheus/system.rules | 2 +- .../notes/low-memory-alert-units-a6fde380ff9b7839.yaml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/low-memory-alert-units-a6fde380ff9b7839.yaml diff --git a/etc/kayobe/kolla/config/prometheus/system.rules b/etc/kayobe/kolla/config/prometheus/system.rules index f693a250c..c82bed16e 100644 --- a/etc/kayobe/kolla/config/prometheus/system.rules +++ b/etc/kayobe/kolla/config/prometheus/system.rules @@ -22,7 +22,7 @@ groups: severity: alert annotations: summary: "Prometheus exporter at {{ $labels.instance }} reports low memory" - description: "Available memory is {{ $value }}." + description: "Available memory is {{ $value }} GiB." - alert: HostOomKillDetected expr: increase(node_vmstat_oom_kill[5m]) > 0 diff --git a/releasenotes/notes/low-memory-alert-units-a6fde380ff9b7839.yaml b/releasenotes/notes/low-memory-alert-units-a6fde380ff9b7839.yaml new file mode 100644 index 000000000..4732bb759 --- /dev/null +++ b/releasenotes/notes/low-memory-alert-units-a6fde380ff9b7839.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Add unit to LowMemory alert description. From 992263081b4831f7a4d8ba0728bd635f5c73da7b Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 15 Mar 2023 13:15:30 +0000 Subject: [PATCH 251/292] Add tags for Blazar and Caso images --- etc/kayobe/kolla/globals.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 67c5b1bb6..81c231d6d 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -9,9 +9,13 @@ enable_docker_repo: {% raw %}"{{ 'overcloud' not in group_names or ansible_facts {% if kolla_base_distro == 'centos' %} bifrost_tag: xena-20230214T165534 +blazar_tag: xena-20230315T122920 +caso_tag: xena-20230315T122920 neutron_tag: xena-20230307T142413 {% else %} bifrost_tag: xena-20230215T195824 +blazar_tag: xena-20230315T122918 +caso_tag: xena-20230315T122918 keystone_tag: xena-20230308T120251 neutron_tag: xena-20230307T142414 {% endif %} From edf5c1a91d946f53da3269f3c26cfd61631e00c7 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Tue, 28 Feb 2023 16:05:28 +0100 Subject: [PATCH 252/292] Bump node exporter to 1.5.0 This fixes a crash seen on HPE DL385 nodes [1]: panic: "node_rapl_package-0-die-0_joules_total" is not a valid metric name [1] https://github.com/prometheus/node_exporter/issues/2299 --- etc/kayobe/kolla.yml | 2 +- .../notes/node-exporter-1.5.0-5e3764790a5a0190.yaml | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/node-exporter-1.5.0-5e3764790a5a0190.yaml diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 5cf57682c..72404e9c6 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -362,7 +362,7 @@ kolla_build_args: haproxy_exporter_version: "0.13.0" memcached_exporter_version: "0.9.0" mysqld_exporter_version: "0.13.0" - node_exporter_version: "1.3.1" + node_exporter_version: "1.5.0" prometheus_version: "2.35.0" prometheus_alertmanager_version: "0.24.0" prometheus_cadvisor_sha256sum: "8e3df91eee70c72ac3f3184b9fab1229b037c2e850ff1593103b2bd9028c57c0" diff --git a/releasenotes/notes/node-exporter-1.5.0-5e3764790a5a0190.yaml b/releasenotes/notes/node-exporter-1.5.0-5e3764790a5a0190.yaml new file mode 100644 index 000000000..2683e94b4 --- /dev/null +++ b/releasenotes/notes/node-exporter-1.5.0-5e3764790a5a0190.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + Updates Prometheus Node exporter to version 1.5.0. +fixes: + - | + Fixes a `Prometheus Node exporter crash + `__ which may + affect nodes with AMD processors (first seen on HPE DL385). From ee98ed99bf52a8536934307cb5e8255e933ddc73 Mon Sep 17 00:00:00 2001 From: Dawud M <7688823+technowhizz@users.noreply.github.com> Date: Wed, 15 Mar 2023 17:06:00 +0000 Subject: [PATCH 253/292] Update node exporter container image tags --- etc/kayobe/kolla/globals.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 81c231d6d..64f9e9d25 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -12,12 +12,14 @@ bifrost_tag: xena-20230214T165534 blazar_tag: xena-20230315T122920 caso_tag: xena-20230315T122920 neutron_tag: xena-20230307T142413 +prometheus_node_exporter_tag: xena-20230310T170439 {% else %} bifrost_tag: xena-20230215T195824 blazar_tag: xena-20230315T122918 caso_tag: xena-20230315T122918 keystone_tag: xena-20230308T120251 neutron_tag: xena-20230307T142414 +prometheus_node_exporter_tag: xena-20230315T164024 {% endif %} glance_tls_proxy_tag: "{% raw %}{{ haproxy_tag | default(openstack_tag) }}{% endraw %}" From 24b31d923469bb7d16b214dae99c1107264ae3da Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Fri, 17 Mar 2023 09:53:08 +0000 Subject: [PATCH 254/292] Add new Ironic tags --- etc/kayobe/kolla/globals.yml | 2 ++ .../notes/ironic-fix-online-upgrades-36e86b3ab3ac081c.yaml | 6 ++++++ 2 files changed, 8 insertions(+) create mode 100644 releasenotes/notes/ironic-fix-online-upgrades-36e86b3ab3ac081c.yaml diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 64f9e9d25..3db61ffcd 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -11,12 +11,14 @@ enable_docker_repo: {% raw %}"{{ 'overcloud' not in group_names or ansible_facts bifrost_tag: xena-20230214T165534 blazar_tag: xena-20230315T122920 caso_tag: xena-20230315T122920 +ironic_tag: xena-20230316T162305 neutron_tag: xena-20230307T142413 prometheus_node_exporter_tag: xena-20230310T170439 {% else %} bifrost_tag: xena-20230215T195824 blazar_tag: xena-20230315T122918 caso_tag: xena-20230315T122918 +ironic_tag: xena-20230317T090705 keystone_tag: xena-20230308T120251 neutron_tag: xena-20230307T142414 prometheus_node_exporter_tag: xena-20230315T164024 diff --git a/releasenotes/notes/ironic-fix-online-upgrades-36e86b3ab3ac081c.yaml b/releasenotes/notes/ironic-fix-online-upgrades-36e86b3ab3ac081c.yaml new file mode 100644 index 000000000..b7e0f9de7 --- /dev/null +++ b/releasenotes/notes/ironic-fix-online-upgrades-36e86b3ab3ac081c.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Adds Ironic images tags to fix a bug with online upgrades for + Bios/Traits. See patch: + https://review.opendev.org/c/openstack/ironic/+/877409 From c01f39e6752543e9ae2a832d7bd4b8269c187c00 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Fri, 17 Mar 2023 17:18:11 +0000 Subject: [PATCH 255/292] Specify ironic_dnsmasq tag The ironic_dnsmasq container was missed during the ironic container builds. As such, ironic_dnsmasq_tag is explicilty set to the same as the operating-system-equivalent kolla_openstack_release. --- etc/kayobe/kolla/globals.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 3db61ffcd..129c99e6c 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -12,6 +12,7 @@ bifrost_tag: xena-20230214T165534 blazar_tag: xena-20230315T122920 caso_tag: xena-20230315T122920 ironic_tag: xena-20230316T162305 +ironic_dnsmasq_tag: xena-20230214T150257 neutron_tag: xena-20230307T142413 prometheus_node_exporter_tag: xena-20230310T170439 {% else %} @@ -19,6 +20,7 @@ bifrost_tag: xena-20230215T195824 blazar_tag: xena-20230315T122918 caso_tag: xena-20230315T122918 ironic_tag: xena-20230317T090705 +ironic_dnsmasq_tag: xena-20230215T164524 keystone_tag: xena-20230308T120251 neutron_tag: xena-20230307T142414 prometheus_node_exporter_tag: xena-20230315T164024 From a2c3d1f26dcbc3b9846f56136e2d940873644d07 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Tue, 21 Mar 2023 09:25:12 +0000 Subject: [PATCH 256/292] Update neutron tag --- etc/kayobe/kolla/globals.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 05afd7c27..5ac963f76 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -13,7 +13,7 @@ blazar_tag: xena-20230315T122920 caso_tag: xena-20230315T122920 ironic_tag: xena-20230316T162305 ironic_dnsmasq_tag: xena-20230214T150257 -neutron_tag: FIXME-BUILD-NEW-IMAGE +neutron_tag: xena-20230320T143013 prometheus_node_exporter_tag: xena-20230310T170439 {% else %} bifrost_tag: xena-20230215T195824 @@ -22,7 +22,7 @@ caso_tag: xena-20230315T122918 ironic_tag: xena-20230317T090705 ironic_dnsmasq_tag: xena-20230215T164524 keystone_tag: xena-20230308T120251 -neutron_tag: FIXME-BUILD-NEW-IMAGE +neutron_tag: xena-20230320T143007 prometheus_node_exporter_tag: xena-20230315T164024 {% endif %} From 6fc586733145cdeb76f2e1fbd2f3c0a61cc6f548 Mon Sep 17 00:00:00 2001 From: Dawud M <7688823+technowhizz@users.noreply.github.com> Date: Tue, 21 Mar 2023 19:09:14 +0000 Subject: [PATCH 257/292] Fix smartmon script Fixes the smartmon script to detect SMART info values regardless of their case. Previously, this would lead to wrongly formatted .prom files resulting in node_exporter failing to parse them. --- etc/kayobe/ansible/scripts/smartmon.sh | 3 ++- .../notes/fix-smartmon-script-db2a38df1b245e58.yaml | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/fix-smartmon-script-db2a38df1b245e58.yaml diff --git a/etc/kayobe/ansible/scripts/smartmon.sh b/etc/kayobe/ansible/scripts/smartmon.sh index 32b3ea81c..c08c46e60 100644 --- a/etc/kayobe/ansible/scripts/smartmon.sh +++ b/etc/kayobe/ansible/scripts/smartmon.sh @@ -106,6 +106,7 @@ parse_smartctl_scsi_attributes() { } parse_smartctl_info() { + shopt -s nocasematch local -i smart_available=0 smart_enabled=0 smart_healthy= local disk="$1" disk_type="$2" local model_family='' device_model='' serial_number='' fw_version='' vendor='' product='' revision='' lun_id='' @@ -199,4 +200,4 @@ for device in ${device_list}; do exit ;; esac -done | format_output \ No newline at end of file +done | format_output diff --git a/releasenotes/notes/fix-smartmon-script-db2a38df1b245e58.yaml b/releasenotes/notes/fix-smartmon-script-db2a38df1b245e58.yaml new file mode 100644 index 000000000..b71a5c439 --- /dev/null +++ b/releasenotes/notes/fix-smartmon-script-db2a38df1b245e58.yaml @@ -0,0 +1,9 @@ +--- +fixes: + - | + Fixes the smartmon script to be case insensitive when checking for + the inital SMART info. This is to ensure that the script works + correctly on systems where the output of `smartctl -i` is not + capitalised as previously expected by the script. This leads to + badly formatted .prom files which lead to node_exporter failing to + scrape the file. From 204610f44d06547eccfef123cb290c4500c62db4 Mon Sep 17 00:00:00 2001 From: Dawud M <7688823+technowhizz@users.noreply.github.com> Date: Mon, 27 Mar 2023 10:47:29 +0000 Subject: [PATCH 258/292] Update HAProxy alerting rules HAProxy alerting rules have been updated to use the server name that is down, rather than the name of the instance that reported the down server --- etc/kayobe/kolla/config/prometheus/haproxy.rules | 4 ++-- .../update-haproxy-alerting-rules-5594547666659c0f.yaml | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/update-haproxy-alerting-rules-5594547666659c0f.yaml diff --git a/etc/kayobe/kolla/config/prometheus/haproxy.rules b/etc/kayobe/kolla/config/prometheus/haproxy.rules index 090f6e37a..274ebf4eb 100644 --- a/etc/kayobe/kolla/config/prometheus/haproxy.rules +++ b/etc/kayobe/kolla/config/prometheus/haproxy.rules @@ -30,7 +30,7 @@ groups: labels: severity: critical annotations: - summary: "HAProxy server down (instance {{ $labels.instance }})" - description: "HAProxy server is down" + summary: "HAProxy server down at {{ $labels.server }}" + description: "HAProxy server for {{ $labels.backend }} is down at {{ $labels.server }}" {% endraw %} diff --git a/releasenotes/notes/update-haproxy-alerting-rules-5594547666659c0f.yaml b/releasenotes/notes/update-haproxy-alerting-rules-5594547666659c0f.yaml new file mode 100644 index 000000000..e327a45b1 --- /dev/null +++ b/releasenotes/notes/update-haproxy-alerting-rules-5594547666659c0f.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + HAProxy alerting rules have been updated to use the server name that is + down, rather than the name of the instance that reported the down server. From 1a61db89659749787c7f2a004240cfbf1423704f Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Tue, 28 Mar 2023 12:15:50 +0100 Subject: [PATCH 259/292] Add support for manila in multinode Supports Manila in the ci-multinode environment using the CephFS native backend. Configuration is automatic based on the Kolla variables to enable manila and the backend. Also moves cephadm-commands to run after other tasks so that the commands can reference objects created earlier on. --- etc/kayobe/ansible/cephadm.yml | 2 +- .../environments/ci-multinode/cephadm.yml | 17 +++++++++++++++++ .../group_vars/seed/network-interfaces | 4 ++++ .../environments/ci-multinode/kolla.yml | 4 ++++ .../ci-multinode/kolla/globals.yml | 3 +++ etc/kayobe/environments/ci-multinode/seed.yml | 19 ++++++++++++++++++- ...manila-for-multinode-df2186276c6b434a.yaml | 8 ++++++++ 7 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/manila-for-multinode-df2186276c6b434a.yaml diff --git a/etc/kayobe/ansible/cephadm.yml b/etc/kayobe/ansible/cephadm.yml index 1cb4d56e9..de5f9fac4 100644 --- a/etc/kayobe/ansible/cephadm.yml +++ b/etc/kayobe/ansible/cephadm.yml @@ -1,8 +1,8 @@ --- # Deploy Ceph via Cephadm. Create EC profiles, CRUSH rules, pools and keys. - import_playbook: cephadm-deploy.yml -- import_playbook: cephadm-commands.yml - import_playbook: cephadm-ec-profiles.yml - import_playbook: cephadm-crush-rules.yml - import_playbook: cephadm-pools.yml - import_playbook: cephadm-keys.yml +- import_playbook: cephadm-commands.yml diff --git a/etc/kayobe/environments/ci-multinode/cephadm.yml b/etc/kayobe/environments/ci-multinode/cephadm.yml index addb37ddf..1c7a0bdf6 100644 --- a/etc/kayobe/environments/ci-multinode/cephadm.yml +++ b/etc/kayobe/environments/ci-multinode/cephadm.yml @@ -35,6 +35,12 @@ cephadm_pools: - name: vms application: rbd state: present + - name: cephfs_data + application: cephfs + state: "{{ 'present' if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else 'absent' }}" + - name: cephfs_metadata + application: cephfs + state: "{{ 'present' if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else 'absent' }}" # List of Cephx keys. See stackhpc.cephadm.keys role for format. cephadm_keys: @@ -56,3 +62,14 @@ cephadm_keys: osd: "profile rbd pool=images" mgr: "profile rbd pool=images" state: present + - name: client.manila + caps: + mon: "allow r" + mgr: "allow rw" + state: "{{ 'present' if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else 'absent' }}" + +# List of Cephadm commands to run. See stackhpc.cephadm.commands role for format. +cephadm_commands: "{{ cephadm_commands_manila_cephfs_native if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else [] }}" +cephadm_commands_manila_cephfs_native: + - "fs new manila-cephfs cephfs_metadata cephfs_data" + - "orch apply mds manila-cephfs" diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces index 0c868b110..a918946eb 100644 --- a/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces @@ -10,6 +10,10 @@ external_interface: "{{ vxlan_interfaces[0].device }}.{{ external_vlan }}" public_interface: "{{ vxlan_interfaces[0].device }}.{{ public_vlan }}" +# The storage interface is required for routing manila traffic between VMs and +# the Storage nodes. +storage_interface: "{{ vxlan_interfaces[0].device }}.{{ storage_vlan }}" + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/environments/ci-multinode/kolla.yml b/etc/kayobe/environments/ci-multinode/kolla.yml index 5c6c80dae..a087ad66f 100644 --- a/etc/kayobe/environments/ci-multinode/kolla.yml +++ b/etc/kayobe/environments/ci-multinode/kolla.yml @@ -4,3 +4,7 @@ kolla_enable_cinder_backup: true kolla_enable_neutron_provider_networks: true kolla_enable_ovn: true kolla_enable_octavia: true + +# The multinode environment supports Manila but it is not enabled by default. +# kolla_enable_manila: true +# kolla_enable_manila_backend_cephfs_native: true diff --git a/etc/kayobe/environments/ci-multinode/kolla/globals.yml b/etc/kayobe/environments/ci-multinode/kolla/globals.yml index bd726ddc5..e0766e7fe 100644 --- a/etc/kayobe/environments/ci-multinode/kolla/globals.yml +++ b/etc/kayobe/environments/ci-multinode/kolla/globals.yml @@ -22,3 +22,6 @@ es_heap_size: 1g octavia_auto_configure: "no" octavia_provider_drivers: "ovn:OVN provider" octavia_provider_agents: "ovn" + +# Manila CephFS configuration +manila_cephfs_filesystem_name: manila-cephfs diff --git a/etc/kayobe/environments/ci-multinode/seed.yml b/etc/kayobe/environments/ci-multinode/seed.yml index 3eb7afa90..749c398d1 100644 --- a/etc/kayobe/environments/ci-multinode/seed.yml +++ b/etc/kayobe/environments/ci-multinode/seed.yml @@ -3,12 +3,29 @@ seed_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' else 'cl seed_lvm_groups: - "{{ stackhpc_lvm_group_rootvg }}" +seed_extra_network_interfaces: > + "{{ seed_extra_network_interfaces_external + + (seed_extra_network_interfaces_manila if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else []) }}" + # Seed has been provided an external interface # for tempest tests and SSH access to machines. -seed_extra_network_interfaces: +seed_extra_network_interfaces_external: - "external" - "public" +# Seed requires access to the storage network for manila-cephfs. +seed_extra_network_interfaces_manila: + - "storage" + # Enable IP routing and source NAT on the seed, allowing it to be used as the # external subnet gateway and provide internet access for VMs in the deployment. seed_enable_snat: true + +snat_rules_default: + - interface: "{{ ansible_facts.default_ipv4.interface }}" + source_ip: "{{ ansible_facts.default_ipv4.address }}" +snat_rules_manila: + - interface: "{{ storage_interface }}" + source_ip: "{{ ansible_facts[storage_interface].ipv4.address }}" +# Only add the storage snat rule if we are using manila-cephfs. +snat_rules: "{{ snat_rules_default + snat_rules_manila if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else snat_rules_default }}" diff --git a/releasenotes/notes/manila-for-multinode-df2186276c6b434a.yaml b/releasenotes/notes/manila-for-multinode-df2186276c6b434a.yaml new file mode 100644 index 000000000..6952fe9bc --- /dev/null +++ b/releasenotes/notes/manila-for-multinode-df2186276c6b434a.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Adds support for Manila in the ci-multinode environment using the CephFS + native backend. This is disabled by default, but can be enabled by setting + the following variables in the kayobe configuration: + `kolla_enable_manila: true` + `kolla_enable_manila_backend_cephfs_native: true` From 37d5671bf3abc08afe0bab78ddff1d2f9f386994 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Wed, 29 Mar 2023 15:09:57 +0100 Subject: [PATCH 260/292] Add documentation for multinode manila --- .../contributor/environments/ci-multinode.rst | 150 ++++++++++++++++++ 1 file changed, 150 insertions(+) diff --git a/doc/source/contributor/environments/ci-multinode.rst b/doc/source/contributor/environments/ci-multinode.rst index 0dd787140..e256281d4 100644 --- a/doc/source/contributor/environments/ci-multinode.rst +++ b/doc/source/contributor/environments/ci-multinode.rst @@ -53,3 +53,153 @@ Final steps 1. ``source kayobe-env --environment ci-aio`` 2. Run ``kayobe overcloud host configure`` 3. Run ``kayobe overcloud service deploy`` + + +Manila +====== +The Multinode environment supports Manila with the CephFS native backend, but it +is not enabled by default. To enable it, set the following in +``/etc/kayobe/environments/ci-multinode/kolla.yml``: + +.. code-block:: yaml + + kolla_enable_manila: true + kolla_enable_manila_backend_cephfs: true + +And re-run ``kayobe overcloud service deploy`` if you are working on an existing +deployment. + +To test it, you will need two virtual machines. Cirros does not support the Ceph +kernel client, so you will need to use a different image. Any regular Linux +distribution should work. As an example, we will use Ubuntu 20.04. + +Download the image locally: + +.. code-block:: bash + + wget http://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img + +Upload the image to Glance: + +.. code-block:: bash + + openstack image create --container-format bare --disk-format qcow2 --file focal-server-cloudimg-amd64.img Ubuntu-20.04 --progress + +Create a keypair: + +.. code-block:: bash + + openstack keypair create --private-key ~/.ssh/id_rsa id_rsa + +Create two virtual machines from the image: + +.. code-block:: bash + + openstack server create --flavor m1.small --image Ubuntu-20.04 --key-name id_rsa --network admin-tenant ubuntu-client-1 + openstack server create --flavor m1.small --image Ubuntu-20.04 --key-name id_rsa --network admin-tenant ubuntu-client-2 + +Wait until the instances are active. It is worth noting that this process can +take a while, especially if the overcloud is deployed to virtual machines. You +can monitor the progress with the following command: + +.. code-block:: bash + + watch openstack server list + +Once they are active, create two floating IPs: + +.. code-block:: bash + + openstack floating ip create external + openstack floating ip create external + +Associate the floating IPs to the instances: + +.. code-block:: bash + + openstack server add floating ip ubuntu-client-1 + openstack server add floating ip ubuntu-client-2 + + +Then SSH into each instance and install the Ceph client: + +.. code-block:: bash + + sudo apt update + sudo apt install -y ceph-common + + +Back on the host, install the Manila client: + +.. code-block:: bash + + sudo dnf install -y python-manilaclient + +Then create a share type and share: + +.. code-block:: bash + + manila type-create cephfs-type false --is_public true + manila type-key cephfs-type set vendor_name=Ceph storage_protocol=CEPHFS + manila create --name test-share --share-type cephfs-type CephFS 2 + +Wait until the share is available: + +.. code-block:: bash + + manila list + +Then allow access to the shares to two users: + +.. code-block:: bash + + manila access-allow test-share cephx alice + manila access-allow test-share cephx bob + +Show the access list to make sure the state of both entries is ``active`` and +take note of the access keys: + +.. code-block:: bash + + manila access-list test-share + +And take note of the path to the share: + +.. code-block:: bash + + manila share-export-location-list test-share + +SSH into the first instance, create a directory for the share, and mount it: + +.. code-block:: bash + + mkdir testdir + sudo mount -t ceph {path} -o name=alice,secret='{access_key}' testdir + +Where the path is the path to the share from the previous step, and the secret +is the access key for the user alice. + +Then create a file in the share: + +.. code-block:: bash + + sudo touch testdir/testfile + +SSH into the second instance, create a directory for the share, and mount it: + +.. code-block:: bash + + mkdir testdir + sudo mount -t ceph {path} -o name=bob,secret='{access_key}' testdir + +Where the path is the same as before, and the secret is the access key for the +user bob. + +Then check that the file created in the first instance is visible in the second +instance: + +.. code-block:: bash + + ls testdir + +If it shows the test file then the share is working correctly. From 2aa47037cd9540596f1f394da8d20d008a24cf36 Mon Sep 17 00:00:00 2001 From: Dawud M <7688823+technowhizz@users.noreply.github.com> Date: Tue, 21 Mar 2023 18:36:19 +0000 Subject: [PATCH 261/292] Fix temperature graph on hardware overview dash Fixes the hardware overview dashboard to use the correct metric for displaying drive temps. Now uses an `or` to display whichever metric is compatible with the drives in the system. The two metrics are temperature_case_raw_value and temperature_celsius_raw_value. --- .../grafana/dashboards/openstack/hardware_overview.json | 6 +++--- ...-hardware-overview-dashboard-temp-ecc047ba46305668.yaml | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 releasenotes/notes/fix-hardware-overview-dashboard-temp-ecc047ba46305668.yaml diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json index 63bbf4af3..60649ff28 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json @@ -530,7 +530,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "smartmon_temperature_case_raw_value{instance=~\"$node\"}", + "expr": "smartmon_temperature_case_raw_value{instance=~\"$node\"} or smartmon_temperature_celsius_raw_value{instance=~\"$node\"}", "format": "table", "hide": false, "instant": true, @@ -664,7 +664,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "avg_over_time(smartmon_temperature_case_raw_value{instance=~\"$node\"}[1h])", + "expr": "avg_over_time(smartmon_temperature_case_raw_value{instance=~\"$node\"}[1h]) or avg_over_time(smartmon_temperature_celsius_raw_value{instance=~\"$node\"}[1h])", "instant": false, "interval": "", "legendFormat": "{{instance}} - {{disk}} - {{serial_number}}", @@ -752,4 +752,4 @@ "version": 5, "weekStart": "" } -{% endraw %} \ No newline at end of file +{% endraw %} diff --git a/releasenotes/notes/fix-hardware-overview-dashboard-temp-ecc047ba46305668.yaml b/releasenotes/notes/fix-hardware-overview-dashboard-temp-ecc047ba46305668.yaml new file mode 100644 index 000000000..6a395888b --- /dev/null +++ b/releasenotes/notes/fix-hardware-overview-dashboard-temp-ecc047ba46305668.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Fixes the hardware overview dashboard to use the correct metric for + displaying drive temps. Now uses an `or` to display whichever metric + is compatible with the drives in the system. The two metrics are + temperature_case_raw_value and temperature_celsius_raw_value. From 92a30f0ad1ac8e69e9a1cb1f77e2e032b2c02082 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Thu, 30 Mar 2023 10:16:47 +0100 Subject: [PATCH 262/292] Split cephadm_commands into pre and post Split `cephadm_commands` into `cephadm_commands_pre` and `cephadm_commands_post` commands. This allows the user to run commands that must be executed before the rest of the post-deployment configuration, as well as commands that rely on resources created by the post-deployment config. --- doc/source/configuration/cephadm.rst | 15 +++++++++++---- .../contributor/environments/ci-multinode.rst | 4 ++-- ...adm-commands.yml => cephadm-commands-post.yml} | 4 +++- etc/kayobe/ansible/cephadm-commands-pre.yml | 13 +++++++++++++ etc/kayobe/ansible/cephadm.yml | 3 ++- etc/kayobe/cephadm.yml | 9 ++++++--- etc/kayobe/environments/ci-multinode/cephadm.yml | 4 +++- ...ephadm-commands-pre-post-bddd25b8bd171ab0.yaml | 8 ++++++++ 8 files changed, 48 insertions(+), 12 deletions(-) rename etc/kayobe/ansible/{cephadm-commands.yml => cephadm-commands-post.yml} (52%) create mode 100644 etc/kayobe/ansible/cephadm-commands-pre.yml create mode 100644 releasenotes/notes/split-cephadm-commands-pre-post-bddd25b8bd171ab0.yaml diff --git a/doc/source/configuration/cephadm.rst b/doc/source/configuration/cephadm.rst index 0322482c0..dfab2b748 100644 --- a/doc/source/configuration/cephadm.rst +++ b/doc/source/configuration/cephadm.rst @@ -242,19 +242,26 @@ for Cinder, Cinder backup, Glance, and Nova in Kolla Ansible. Ceph Commands ~~~~~~~~~~~~~ -It is possible to run an arbitrary list of commands against the cluster after deployment -by setting the ``cephadm_commands`` variable. ``cephadm_commands`` should be a list of commands -to pass to ``cephadm shell -- ceph``. For example: +It is possible to run an arbitrary list of commands against the cluster after +deployment by setting the ``cephadm_commands_pre`` and ``cephadm_commands_post`` +variables. Each should be a list of commands to pass to ``cephadm shell -- +ceph``. For example: .. code:: yaml # A list of commands to pass to cephadm shell -- ceph. See stackhpc.cephadm.commands # for format. - cephadm_commands: + cephadm_commands_pre: # Configure Prometheus exporter to listen on a specific interface. The default # is to listen on all interfaces. - "config set mgr mgr/prometheus/server_addr 10.0.0.1" +Both variables have the same format, however commands in the +``cephadm_commands_pre`` list are executed before the rest of the Ceph +post-deployment configuration is applied. Commands in the +``cephadm_commands_post`` list are executed after the rest of the Ceph +post-deployment configuration is applied. + Deployment ========== diff --git a/doc/source/contributor/environments/ci-multinode.rst b/doc/source/contributor/environments/ci-multinode.rst index e256281d4..e42d61807 100644 --- a/doc/source/contributor/environments/ci-multinode.rst +++ b/doc/source/contributor/environments/ci-multinode.rst @@ -70,7 +70,7 @@ And re-run ``kayobe overcloud service deploy`` if you are working on an existing deployment. To test it, you will need two virtual machines. Cirros does not support the Ceph -kernel client, so you will need to use a different image. Any regular Linux +kernel client, so you will need to use a different image. Any regular Linux distribution should work. As an example, we will use Ubuntu 20.04. Download the image locally: @@ -124,7 +124,7 @@ Associate the floating IPs to the instances: Then SSH into each instance and install the Ceph client: .. code-block:: bash - + sudo apt update sudo apt install -y ceph-common diff --git a/etc/kayobe/ansible/cephadm-commands.yml b/etc/kayobe/ansible/cephadm-commands-post.yml similarity index 52% rename from etc/kayobe/ansible/cephadm-commands.yml rename to etc/kayobe/ansible/cephadm-commands-post.yml index 32742a343..fbca30dfd 100644 --- a/etc/kayobe/ansible/cephadm-commands.yml +++ b/etc/kayobe/ansible/cephadm-commands-post.yml @@ -1,5 +1,5 @@ --- -- name: Ensure additional Ceph commands are run +- name: Ensure additional Ceph commands are run after post-deployment configuration gather_facts: false hosts: mons become: true @@ -9,3 +9,5 @@ tasks: - import_role: name: stackhpc.cephadm.commands + vars: + cephadm_commands: "{{ cephadm_commands_post }}" diff --git a/etc/kayobe/ansible/cephadm-commands-pre.yml b/etc/kayobe/ansible/cephadm-commands-pre.yml new file mode 100644 index 000000000..08047cc51 --- /dev/null +++ b/etc/kayobe/ansible/cephadm-commands-pre.yml @@ -0,0 +1,13 @@ +--- +- name: Ensure additional Ceph commands are run before post-deployment configuration + gather_facts: false + hosts: mons + become: true + tags: + - cephadm + - cephadm-commands + tasks: + - import_role: + name: stackhpc.cephadm.commands + vars: + cephadm_commands: "{{ cephadm_commands_pre }}" diff --git a/etc/kayobe/ansible/cephadm.yml b/etc/kayobe/ansible/cephadm.yml index de5f9fac4..b699df153 100644 --- a/etc/kayobe/ansible/cephadm.yml +++ b/etc/kayobe/ansible/cephadm.yml @@ -1,8 +1,9 @@ --- # Deploy Ceph via Cephadm. Create EC profiles, CRUSH rules, pools and keys. - import_playbook: cephadm-deploy.yml +- import_playbook: cephadm-commands-pre.yml - import_playbook: cephadm-ec-profiles.yml - import_playbook: cephadm-crush-rules.yml - import_playbook: cephadm-pools.yml - import_playbook: cephadm-keys.yml -- import_playbook: cephadm-commands.yml +- import_playbook: cephadm-commands-post.yml diff --git a/etc/kayobe/cephadm.yml b/etc/kayobe/cephadm.yml index 4f5f0ed7c..ae06880ca 100644 --- a/etc/kayobe/cephadm.yml +++ b/etc/kayobe/cephadm.yml @@ -60,9 +60,12 @@ cephadm_cluster_network: "{{ storage_mgmt_net_name | net_cidr }}" # List of Cephx keys. See stackhpc.cephadm.keys role for format. #cephadm_keys: -# A list of commands to pass to cephadm shell -- ceph. See stackhpc.cephadm.commands -# for format. -#cephadm_commands: +# A list of commands to pass to cephadm shell -- ceph. See +# stackhpc.cephadm.commands for format. Pre commands run before the rest of the +# post-deployment configuration, post commands run after the rest of the +# post-deployment configuration. +#cephadm_commands_pre: +#cephadm_commands_post: ############################################################################### # Kolla Ceph auto-configuration. diff --git a/etc/kayobe/environments/ci-multinode/cephadm.yml b/etc/kayobe/environments/ci-multinode/cephadm.yml index 1c7a0bdf6..7885a5735 100644 --- a/etc/kayobe/environments/ci-multinode/cephadm.yml +++ b/etc/kayobe/environments/ci-multinode/cephadm.yml @@ -69,7 +69,9 @@ cephadm_keys: state: "{{ 'present' if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else 'absent' }}" # List of Cephadm commands to run. See stackhpc.cephadm.commands role for format. -cephadm_commands: "{{ cephadm_commands_manila_cephfs_native if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else [] }}" +cephadm_commands_pre: [] + +cephadm_commands_post: "{{ cephadm_commands_manila_cephfs_native if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else [] }}" cephadm_commands_manila_cephfs_native: - "fs new manila-cephfs cephfs_metadata cephfs_data" - "orch apply mds manila-cephfs" diff --git a/releasenotes/notes/split-cephadm-commands-pre-post-bddd25b8bd171ab0.yaml b/releasenotes/notes/split-cephadm-commands-pre-post-bddd25b8bd171ab0.yaml new file mode 100644 index 000000000..c39ffa43a --- /dev/null +++ b/releasenotes/notes/split-cephadm-commands-pre-post-bddd25b8bd171ab0.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Split `cephadm_commands` into `cephadm_commands_pre` and + `cephadm_commands_post` commands. This allows the user to run commands that + must be run before the rest of the post-deployment configuration, as well + as commands that rely on resources created by the post-deployment config. + From ff6abe95192cb97d9cb20eac9e6b9dc3b21eaf57 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Thu, 30 Mar 2023 11:43:46 +0100 Subject: [PATCH 263/292] Update documentation for Manila and Ceph --- doc/source/configuration/cephadm.rst | 44 +++++++++++++++++++ .../contributor/environments/ci-multinode.rst | 10 ++--- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/doc/source/configuration/cephadm.rst b/doc/source/configuration/cephadm.rst index dfab2b748..8908b351f 100644 --- a/doc/source/configuration/cephadm.rst +++ b/doc/source/configuration/cephadm.rst @@ -262,6 +262,50 @@ post-deployment configuration is applied. Commands in the ``cephadm_commands_post`` list are executed after the rest of the Ceph post-deployment configuration is applied. +Manila & CephFS +~~~~~~~~~~~~~~~ + +Using Manila with the CephFS backend requires the configuration of additional +resources. + +A Manila key should be added to cephadm_keys: + +.. code:: yaml + + # Append the following to cephadm_keys: + - name: client.manila + caps: + mon: "allow r" + mgr: "allow rw" + state: present + +A CephFS filesystem requires two pools, one for metadata and one for data: + +.. code:: yaml + + # Append the following to cephadm_pools: + - name: cephfs_data + application: cephfs + state: present + - name: cephfs_metadata + application: cephfs + state: present + +Finally, the CephFS filesystem itself should be created: + +.. code:: yaml + + # Append the following to cephadm_commands_post: + - "fs new manila-cephfs cephfs_metadata cephfs_data" + - "orch apply mds manila-cephfs" + +In this example, the filesystem is named ``manila-cephfs``. This name +should be used in the Kolla Manila configuration e.g.: + +.. code:: yaml + + manila_cephfs_filesystem_name: manila-cephfs + Deployment ========== diff --git a/doc/source/contributor/environments/ci-multinode.rst b/doc/source/contributor/environments/ci-multinode.rst index e42d61807..93341113a 100644 --- a/doc/source/contributor/environments/ci-multinode.rst +++ b/doc/source/contributor/environments/ci-multinode.rst @@ -38,11 +38,11 @@ Config changes 3. Also under vxlan_interfaces, add vni:x where x is between 500 and 1000 4. Also under vxlan_interfaces, check vxlan_dstport is not 4789 (this causes conflicts, change to 4790) -5. In /etc/kayobe/environments/ci-multinode/tf-networks.yml, edit admin_ips so +5. In etc/kayobe/environments/ci-multinode/tf-networks.yml, edit admin_ips so that the compute and controller IPs line up with the instances that were created earlier, remove the other IPs for seed and cephOSD -6. In /etc/kayobe/environments/ci-multinode/network-allocation.yml, remove all +6. In etc/kayobe/environments/ci-multinode/network-allocation.yml, remove all the entries and just assign ``aio_ips:`` an empty set ``[]`` 7. In etc/kayobe/environments/ci-multinode/inventory/hosts, remove the seed 8. run stackhpc-kayobe-config/etc/kayobe/ansible/growroot.yml (if this fails, @@ -59,7 +59,7 @@ Manila ====== The Multinode environment supports Manila with the CephFS native backend, but it is not enabled by default. To enable it, set the following in -``/etc/kayobe/environments/ci-multinode/kolla.yml``: +``etc/kayobe/environments/ci-multinode/kolla.yml``: .. code-block:: yaml @@ -71,7 +71,7 @@ deployment. To test it, you will need two virtual machines. Cirros does not support the Ceph kernel client, so you will need to use a different image. Any regular Linux -distribution should work. As an example, we will use Ubuntu 20.04. +distribution should work. As an example, this guide will use Ubuntu 20.04. Download the image locally: @@ -133,7 +133,7 @@ Back on the host, install the Manila client: .. code-block:: bash - sudo dnf install -y python-manilaclient + pip install python-manilaclient Then create a share type and share: From 8ce3ed00ef32996d674d8d24fd9a8dfedd20b215 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Fri, 31 Mar 2023 12:36:32 +0100 Subject: [PATCH 264/292] Add docs describing the package update process --- doc/source/contributor/index.rst | 1 + doc/source/contributor/package-updates.rst | 202 +++++++++++++++++++++ 2 files changed, 203 insertions(+) create mode 100644 doc/source/contributor/package-updates.rst diff --git a/doc/source/contributor/index.rst b/doc/source/contributor/index.rst index 1e1bdd2d6..346e67989 100644 --- a/doc/source/contributor/index.rst +++ b/doc/source/contributor/index.rst @@ -10,3 +10,4 @@ This guide is for contributors of the StackHPC Kayobe configuration project. documentation release-notes environments/index + package-updates diff --git a/doc/source/contributor/package-updates.rst b/doc/source/contributor/package-updates.rst new file mode 100644 index 000000000..a4ab80b8c --- /dev/null +++ b/doc/source/contributor/package-updates.rst @@ -0,0 +1,202 @@ +============================= +Package and Container Updates +============================= + +Preparations +============ + +1. Before building images, you should check for any outstanding PRs into the “base branch“. Below are the links for the Wallaby branches. + + kayobe-config: https://github.com/stackhpc/stackhpc-kayobe-config/pulls?q=is%3Apr+is%3Aopen+base%3Astackhpc%2Fwallaby + + kolla: https://github.com/stackhpc/kolla/pulls?q=is%3Apr+is%3Aopen+base%3Astackhpc%2Fwallaby + + kolla-ansible: https://github.com/stackhpc/kolla-ansible/pulls?q=is%3Apr+is%3Aopen+base%3Astackhpc%2Fwallaby + + You should also check any referenced source trees in etc/kayobe/kolla.yml. + + e.g: https://github.com/stackhpc/stackhpc-kayobe-config/blob/stackhpc/wallaby/etc/kayobe/kolla.yml#L112-L158 + +2. Follow the workflows documented `here `_. Sync the package repositories. Then, for each release: update the Kayobe package repository versions, build and push Kolla container images, open a draft PR with the updated container image tags. The rest of this document describes the stage "Test". + +Testing +======= + +The following steps describe the process to test the new package and container repositories. See the subsections beneath for further explanations. + +1. Build two multinode environments for OVS and OVN, both on the "base branch". + +2. Run tests on current package versions as a baseline. + +3. Upgrade host packages. + +4. Upgrade containers. + +5. Run tests again with the new packages. + +6. Request reviews for your proposed PR to bring in the new packages. + +7. Promote these packages before merging the PR, but after the CI checks have passed. + +8. Upgrade OpenStack to the next release. + +9. Repeat steps 2 and 4-7. (Step 3 is skipped as the host packages will be shared across these releases.) + +10. Repeat 8 and 9 for any further releases. + +Creating the multinode environments +----------------------------------- + +There is a comprehensive guide to setting up a multinode environment with Terraform, found here: https://github.com/stackhpc/terraform-kayobe-multinode. There are some things to note: + +* OVN is enabled by default, you should override it under ``etc/kayobe/environments/ci-multinode/kolla.yml kolla_enable_ovn: false`` for the OVS multinode environment. + +* Remember to set different vxlan_vnis for each. + +* Before running deploy-openstack.sh, run distro sync on each host to ensure you are using the same snapshots as in the release train. + +* The tempest tests run automatically at the end of deploy-openstack.sh. If you have the time, it is worth fixing any failing tests you can so that there is greater coverage for the package updates. (Also remember to propose these fixes in the relevant repos where applicable.) + +Upgrading host packages +----------------------- + +Bump the snapshot versions in /etc/yum/repos.d with + +.. code-block:: console + + kayobe overcloud host configure -t dnf -kt none + +Install new packages: + +.. code-block:: console + + kayobe overcloud host package update --packages "*" + +Perform a rolling reboot of hosts: + +.. code-block:: console + + export ANSIBLE_SERIAL=1 + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/reboot.yml --limit controllers + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/reboot.yml --limit compute[0] + + # Test live migration + openstack server create --image cirros --flavor m1.tiny --network external --hypervisor-hostname wallaby-pkg-refresh-ovs-compute-02.novalocal --os-compute-api-version 2.74 server1 + openstack server migrate --live-migration server1 + watch openstack server show server1 + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/reboot.yml --limit compute[1] + + # Try and migrate back + openstack server migrate --live-migration server1 + watch openstack server show server1 + +Upgrading containers within a release +------------------------------------- + +Deploy the services, once the new tags are set in the kayobe_config: + +.. code-block:: console + + kayobe overcloud service deploy + +Upgrading OpenStack to the next release in a multinode environment +------------------------------------------------------------------ + +As this is not a full production system, only a reduced number of steps need to be followed to upgrade to a new release. Below describes these steps, with Wallaby as the base branch: + +.. code-block:: console + + cd /src/kayobe-config/ + git fetch + git checkout -b xena_uber_merge + git merge origin/stackhpc/xena + + source /venvs/kayobe/bin/activate + cd /src/kayobe + git checkout stackhpc/xena + git fetch + pip install -U ~/src/kayobe + + kayobe control host upgrade + kayobe overcloud host upgrade + + kayobe overcloud container image pull + + ---Optional + kayobe overcloud service configuration save --output-dir config/wallaby + kayobe overcloud service configuration generate --node-config-dir /tmp/kolla-xena-config + kayobe overcloud service configuration save --output-dir config/xena --node-config-dir /tmp/kolla-xena-config + kayobe overcloud host command run --command 'rm -rf /tmp/kolla-xena-config' --become + --- + + kayobe overcloud service upgrade + +Tests +----- + +Tempest +####### + +Run tempest, you can then perform the other tests while it runs. Once complete, check if any tests are failing. + +As of February 2023, only one test was expected to fail. This may no longer be the case, so any additional failures are worth exploring. + +.. code-block:: console + + tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_port_security_macspoofing_port + +Poke around horizon +################### + +Perform some basic operations such as spawning VMs or attaching/detaching volumes and check that each page works correctly. + +Monitoring +########## + +Check for any ERROR log messages in Kibana. + +Check that the Grafana dashboards are all populated with data. + +Check that there are no active alerts. + +Check that there are no flapping alerts. + +Octavia (OVN only) +################## + +You will need to add an Ubuntu image and create a keypair. + +.. code-block:: console + + wget http://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img + + openstack image create \ + --progress \ + --container-format bare \ + --disk-format qcow2 \ + --file focal-server-cloudimg-amd64.img \ + Ubuntu-20.04 + + openstack keypair create --private-key ~/.ssh/os-admin os-admin + +Then run Octavia test script: + +https://gist.github.com/MoteHue/ee5990bddea0677f54d8bb93d307aa71#file-octavia_test-sh + + +Attempt to build OFED against the latest kernel +############################################### + +Note that this only needs to be performed once. + +.. code-block:: console + + kayobe seed host configure -t dnf -kt none + kayobe seed host package update --packages "*" + +Then run the OFED test script: + +https://gist.github.com/cityofships/b4883ee19f75d14534f04115892b8465 + + From 6176037d74a32c21c714044bbc015603494e0cd6 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Tue, 4 Apr 2023 13:03:52 +0200 Subject: [PATCH 265/292] docs: fix typo --- doc/source/configuration/lvm.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/configuration/lvm.rst b/doc/source/configuration/lvm.rst index 65b892aa8..a96ca8db9 100644 --- a/doc/source/configuration/lvm.rst +++ b/doc/source/configuration/lvm.rst @@ -76,7 +76,7 @@ another LV needs to be grown at a later date. Growroot playbook ================= -A ``growroot.yml`` custom playbook is provied that can be used to grow the +A ``growroot.yml`` custom playbook is provided that can be used to grow the partition and LVM Physical Volume (PV) of the root Volume Group (VG). This allows for expansion of Logical Volumes (LVs) in that VG. From d1b376bca3d746f96ac2c09d6473709556fc350e Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Tue, 11 Apr 2023 10:29:29 +0100 Subject: [PATCH 266/292] Update docs based on review --- doc/source/contributor/package-updates.rst | 52 +++++++++++++++++++--- 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/doc/source/contributor/package-updates.rst b/doc/source/contributor/package-updates.rst index a4ab80b8c..746266150 100644 --- a/doc/source/contributor/package-updates.rst +++ b/doc/source/contributor/package-updates.rst @@ -2,10 +2,12 @@ Package and Container Updates ============================= +This section describes the Release Train process of creating new package repository snapshots and updating the container images to use these new packages. It includes instructions on the recommended way to test these updates, using the multinode test environment. + Preparations ============ -1. Before building images, you should check for any outstanding PRs into the “base branch“. Below are the links for the Wallaby branches. +1. Before building images, you should check for any outstanding PRs into the earliest supported release. Below are the links for the Wallaby branches. kayobe-config: https://github.com/stackhpc/stackhpc-kayobe-config/pulls?q=is%3Apr+is%3Aopen+base%3Astackhpc%2Fwallaby @@ -15,16 +17,30 @@ Preparations You should also check any referenced source trees in etc/kayobe/kolla.yml. - e.g: https://github.com/stackhpc/stackhpc-kayobe-config/blob/stackhpc/wallaby/etc/kayobe/kolla.yml#L112-L158 + e.g: https://github.com/stackhpc/stackhpc-kayobe-config/blob/320ba8b28da879917beec0ce7c4c90c478aaeaf7/etc/kayobe/kolla.yml#L112-L158 + +2. Follow the workflows documented `here `_. Sync the package repositories. Then, for each release: + +* Make sure the nightly sync of package repositories has succeeded. + +* Update the Kayobe package repository versions. + +* Build and push Kolla container images. + +* Open a draft PR into ``stackhpc/kayobe-config`` with the updated container image tags. -2. Follow the workflows documented `here `_. Sync the package repositories. Then, for each release: update the Kayobe package repository versions, build and push Kolla container images, open a draft PR with the updated container image tags. The rest of this document describes the stage "Test". +* Test. + +* Review the PR. Once approved, promote the container images before then merging the PR. + +The rest of this document describes the final "test" stage. Testing ======= The following steps describe the process to test the new package and container repositories. See the subsections beneath for further explanations. -1. Build two multinode environments for OVS and OVN, both on the "base branch". +1. Build two multinode environments for OVS and OVN, both on the earliest supported release. 2. Run tests on current package versions as a baseline. @@ -53,14 +69,27 @@ There is a comprehensive guide to setting up a multinode environment with Terraf * Remember to set different vxlan_vnis for each. -* Before running deploy-openstack.sh, run distro sync on each host to ensure you are using the same snapshots as in the release train. +* Before starting any tests, run ``dnf distro-sync`` on each host to ensure you are using the same snapshots as in the release train. You can do this using the following commands: + +.. code-block:: console + + kayobe seed host command run -b --command "dnf distro-sync" + kayobe overcloud host command run -b --command "dnf distro-sync" * The tempest tests run automatically at the end of deploy-openstack.sh. If you have the time, it is worth fixing any failing tests you can so that there is greater coverage for the package updates. (Also remember to propose these fixes in the relevant repos where applicable.) Upgrading host packages ----------------------- -Bump the snapshot versions in /etc/yum/repos.d with +Checkout the new kayobe-config branch (from the draft PR): + +.. code-block:: console + + cd $KAYOBE_CONFIG_ROOT + git fetch + git checkout + +Bump the snapshot versions in /etc/yum/repos.d with: .. code-block:: console @@ -94,6 +123,14 @@ Perform a rolling reboot of hosts: Upgrading containers within a release ------------------------------------- +Checkout the new kayobe-config branch (from the draft PR): + +.. code-block:: console + + cd $KAYOBE_CONFIG_ROOT + git fetch + git checkout + Deploy the services, once the new tags are set in the kayobe_config: .. code-block:: console @@ -103,7 +140,7 @@ Deploy the services, once the new tags are set in the kayobe_config: Upgrading OpenStack to the next release in a multinode environment ------------------------------------------------------------------ -As this is not a full production system, only a reduced number of steps need to be followed to upgrade to a new release. Below describes these steps, with Wallaby as the base branch: +As this is not a full production system, only a reduced number of steps need to be followed to upgrade to a new release. Below describes these steps, with ``stackhpc/wallaby`` as the starting branch: .. code-block:: console @@ -128,6 +165,7 @@ As this is not a full production system, only a reduced number of steps need to kayobe overcloud service configuration generate --node-config-dir /tmp/kolla-xena-config kayobe overcloud service configuration save --output-dir config/xena --node-config-dir /tmp/kolla-xena-config kayobe overcloud host command run --command 'rm -rf /tmp/kolla-xena-config' --become + # Check the diff between the old and new configs --- kayobe overcloud service upgrade From a8bb5ed0b838a928fb0ccc5f191db9480c204960 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Wed, 12 Apr 2023 09:42:06 +0100 Subject: [PATCH 267/292] Add check to package update docs for new kernel --- doc/source/contributor/package-updates.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/source/contributor/package-updates.rst b/doc/source/contributor/package-updates.rst index 746266150..55fcd19be 100644 --- a/doc/source/contributor/package-updates.rst +++ b/doc/source/contributor/package-updates.rst @@ -76,6 +76,15 @@ There is a comprehensive guide to setting up a multinode environment with Terraf kayobe seed host command run -b --command "dnf distro-sync" kayobe overcloud host command run -b --command "dnf distro-sync" +* This may have installed a new kernel version. If so, you will need to reboot the overcloud hosts. You can check the installed kernels and the currently running kernel with the following commands. If the latest listed version is not running, you will need to reboot. + +.. code-block:: console + + kayobe seed host command run -b --show-output --command "dnf list installed kernel" + kayobe seed host command run -b --show-output --command "uname -a" + + kayobe playbook run --limit seed,overcloud $KAYOBE_CONFIG_PATH/ansible/reboot.yml + * The tempest tests run automatically at the end of deploy-openstack.sh. If you have the time, it is worth fixing any failing tests you can so that there is greater coverage for the package updates. (Also remember to propose these fixes in the relevant repos where applicable.) Upgrading host packages From 9d6e39ad3594701be9e47f5f2723fd95eb52e248 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Wed, 12 Apr 2023 13:54:01 +0100 Subject: [PATCH 268/292] Apply suggestions from code review Co-authored-by: Mark Goddard --- doc/source/contributor/package-updates.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/source/contributor/package-updates.rst b/doc/source/contributor/package-updates.rst index 55fcd19be..9a6c74589 100644 --- a/doc/source/contributor/package-updates.rst +++ b/doc/source/contributor/package-updates.rst @@ -27,7 +27,7 @@ Preparations * Build and push Kolla container images. -* Open a draft PR into ``stackhpc/kayobe-config`` with the updated container image tags. +* Open a draft PR into ``stackhpc-kayobe-config`` with the updated container image tags. * Test. @@ -71,10 +71,10 @@ There is a comprehensive guide to setting up a multinode environment with Terraf * Before starting any tests, run ``dnf distro-sync`` on each host to ensure you are using the same snapshots as in the release train. You can do this using the following commands: -.. code-block:: console + .. code-block:: console - kayobe seed host command run -b --command "dnf distro-sync" - kayobe overcloud host command run -b --command "dnf distro-sync" + kayobe seed host command run -b --command "dnf distro-sync" + kayobe overcloud host command run -b --command "dnf distro-sync" * This may have installed a new kernel version. If so, you will need to reboot the overcloud hosts. You can check the installed kernels and the currently running kernel with the following commands. If the latest listed version is not running, you will need to reboot. From 8bfb7bdb5adbb342c7c0c6dc8e0ec6fe7bb91e6b Mon Sep 17 00:00:00 2001 From: Grzegorz Bialas <93606484+GregWhiteyBialas@users.noreply.github.com> Date: Thu, 13 Apr 2023 11:29:23 +0200 Subject: [PATCH 269/292] Add wazuh (#318) * add Wazuh deployment playbook and docs Co-authored-by: Mark Goddard Co-authored-by: Jack Hodgkiss Co-authored-by: Rafal Lewandowski --- .gitignore | 5 + doc/source/configuration/index.rst | 1 + doc/source/configuration/wazuh.rst | 330 ++++++++++++++++++ etc/kayobe/ansible/requirements.yml | 3 + .../ansible/templates/wazuh-secrets.yml.j2 | 14 + etc/kayobe/ansible/wazuh-agent.yml | 7 + etc/kayobe/ansible/wazuh-manager.yml | 70 ++++ etc/kayobe/ansible/wazuh-secrets.yml | 27 ++ .../group_vars/wazuh-agent/wazuh-agent | 26 ++ .../group_vars/wazuh-manager/wazuh-manager | 149 ++++++++ etc/kayobe/inventory/groups | 7 + .../notes/add_wazuh-be2999448db5cb7c.yaml | 5 + 12 files changed, 644 insertions(+) create mode 100644 doc/source/configuration/wazuh.rst create mode 100644 etc/kayobe/ansible/templates/wazuh-secrets.yml.j2 create mode 100644 etc/kayobe/ansible/wazuh-agent.yml create mode 100644 etc/kayobe/ansible/wazuh-manager.yml create mode 100644 etc/kayobe/ansible/wazuh-secrets.yml create mode 100644 etc/kayobe/inventory/group_vars/wazuh-agent/wazuh-agent create mode 100644 etc/kayobe/inventory/group_vars/wazuh-manager/wazuh-manager create mode 100644 releasenotes/notes/add_wazuh-be2999448db5cb7c.yaml diff --git a/.gitignore b/.gitignore index 17cbc4fe2..0eb758954 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,8 @@ etc/kayobe/inventory/group_vars/seed/ansible-host # Ignore kolla configuration. etc/kolla + +# Ignore wazuh +etc/kayobe/ansible/roles/wazuh-ansible/ +etc/kayobe/ansible/wazuh/certificates/* +etc/kayobe/ansible/wazuh/certificates/custom_certificates/* diff --git a/doc/source/configuration/index.rst b/doc/source/configuration/index.rst index ebaab804e..16bba800e 100644 --- a/doc/source/configuration/index.rst +++ b/doc/source/configuration/index.rst @@ -15,3 +15,4 @@ the various features provided. swap cephadm monitoring + wazuh diff --git a/doc/source/configuration/wazuh.rst b/doc/source/configuration/wazuh.rst new file mode 100644 index 000000000..8020d154a --- /dev/null +++ b/doc/source/configuration/wazuh.rst @@ -0,0 +1,330 @@ +===== +Wazuh +===== + +Wazuh Manager +============= + +Provision using infra-vms +------------------------- + +Provisioning an infra VM for Wazuh Manager. + +From Xena, Kayobe supports :kayobe-doc:`provisioning infra VMs `. The StackHPC fork of Kayobe has backported this to Wallaby. +The following configuration may be used as a guide. Config for infra VMs is documented :kayobe-doc:`here `. + + +Set the python interpreter in +``etc/kayobe/inventory/group_vars/infra-vms/ansible-python-interpreter``: + + +.. code-block:: console + + --- + # Use a virtual environment for remote operations. + ansible_python_interpreter: "{{ virtualenv_path }}/kayobe/bin/python" + + +Define VM sizing in ``etc/kayobe/inventory/group_vars/wazuh-manager/infra-vms``: + +.. code-block:: console + + --- + # Memory in MB. + infra_vm_memory_mb: 16384 + + + # Number of vCPUs. + infra_vm_vcpus: 8 + + + # Capacity of the infra VM data volume. + infra_vm_data_capacity: "200G" + + +Optional: define LVM volumes ``etc/kayobe/inventory/group_vars/wazuh-manager/lvm``: + +.. code-block:: console + + # List of infra VM volume groups. See mrlesmithjr.manage-lvm role for + # format. + infra_vm_lvm_groups: + - vgname: "data" + disks: + - "/dev/vdb" + create: true + lvnames: + - lvname: "data" + size: "100%VG" + filesystem: "ext4" + mount: true + mntp: “/var/lib/elasticsearch” + create: true + + +Define network interfaces ``etc/kayobe/inventory/group_vars/wazuh-manager/network-interfaces``: + +(The following is an example - the names will depend on your particular network configuration.) + +.. code-block:: console + + --- + # Overcloud provisioning network IP information. + provision_oc_net_interface: "ens3" + + +The Wazuh manager may need to be exposed externally, in which case it may require another interface. +This can be done as follows in ``etc/kayobe/inventory/group_vars/wazuh-manager/network-interfaces`` , +with the network defined in network.yml as usual. + +.. code-block:: console + + infra_vm_extra_network_interfaces: + - "extra_net" + + # External network connectivity on ens2 + extra_net_interface: "ens2" + + +Follow the Kayobe instructions to :kayobe-doc:`provision the VM ` and configure the host. + + +Network Setup +------------- + +Your wazuh-manager VM needs to have network connection with servers which will have the wazuh-agent installed, preferably it should be in the `provision_oc_net`. + + +Required ports +-------------- + +Several services are used for the communication of Wazuh components. Below is the list of default ports used by these services. + ++-----------------+-----------+----------------+------------------------------------------------+ +| Component | Port | Protocol | Purpose | ++=================+===========+================+================================================+ +| | 1514 | TCP (default) | Agent connection service | ++ +-----------+----------------+------------------------------------------------+ +| | 1514 | UDP (optional) | Agent connection service (disabled by default) | ++ +-----------+----------------+------------------------------------------------+ +| Wazuh server | 1515 | TCP | Agent enrollment service | ++ +-----------+----------------+------------------------------------------------+ +| | 1516 | TCP | Wazuh cluster daemon | ++ +-----------+----------------+------------------------------------------------+ +| | 514 | UDP (default) | Wazuh Syslog collector (disabled by default) | ++ +-----------+----------------+------------------------------------------------+ +| | 514 | TCP (optional) | Wazuh Syslog collector (disabled by default) | ++ +-----------+----------------+------------------------------------------------+ +| | 55000 | TCP | Wazuh server RESTful API | ++-----------------+-----------+----------------+------------------------------------------------+ +| | 9200 | TCP | Wazuh indexer RESTful API | ++ Wazuh indexer +-----------+----------------+------------------------------------------------+ +| | 9300-9400 | TCP | Wazuh indexer cluster communication | ++-----------------+-----------+----------------+------------------------------------------------+ +| Wazuh dashboard | 443 | TCP | Wazuh web user interface | ++-----------------+-----------+----------------+------------------------------------------------+ + + +Manually provisioned VM +----------------------- + +In case where you can’t use infra-vms to deploy your wazuh-manager VM but you want to configure +host using kayobe, there are some tips (note that depending on your setup this don’t have to always apply): + +* Depending on preferences host have to be part of some group in inventory. ``infra-vms`` group still seems as best choice + You can use ``kayobe infra vm host configure`` to configure host in this case. + Bellow tips are based on assumption that infra-vm will be used. +* user ``stack`` with password less sudo and accessible with ssh keys needs to be present on host. + It can be achieved in many different ways, depending on your setup. +* lvm configuration should be placed in ``host_vars/`` +* wazuh-manager host have to be part of ``infra-vms`` group (directly or as child) +* network used on host needs to be defined in ``networks.yml`` and + if you have pre-alocated IP, it should be added to ``network-allocation.yml``. + For example, if using host with IP 10.10.224.5 in network 10.10.224.0/24 one have to add: + + +``networks.yml``: + +.. code-block:: console + + undercloud_admin_net_cidr: 10.10.224.0/24 + undercloud_admin_net_allocation_pool_start: 10.10.224.3 + undercloud_admin_net_allocation_pool_end: 10.10.224.200 + undercloud_admin_net_gateway: 10.10.224.254 + + +``network-allocation.yml``: + +.. code-block:: console + + undercloud_admin_net_ips: + nesmetprd01: 10.10.224.5 + +Note that in this example network name is ``undercloud`` to demonstrate that this network isn't "standard" kayobe network. + + +Deploying Wazuh Manager services +================================ + +Setup +----- + +To install specific version modify wazuh-ansible entry in ``etc/kayobe/ansible/requirements.yml``: + +.. code-block:: console + + roles: + - name: wazuh-ansible + src: https://github.com/stackhpc/wazuh-ansible + version: stackhpc + +Version above was tested and verified, but there is no reason to use not different one. + +Install the role: + +``kayobe control host bootstrap`` + + +Edit the playbook and variables to your needs: + +Wazuh manager configuration +--------------------------- + +Wazuh manager playbook is located in ``etc/kayobe/ansible/wazuh-manager.yml``. +Running this playbook will: + +* generate certificates for wazuh-manager +* setup and deploy filebeat on wazuh-manager vm +* setup and deploy wazuh-indexer on wazuh-manager vm +* setup and deploy wazuh-manager on wazuh-manager vm +* setup and deploy wazuh-dashboard on wazuh-manager vm +* copy certificates over to wazuh-manager vm + +Wazuh manager variables file is located in ``etc/kayobe/inventory/group_vars/wazuh-manager/wazuh-manager``. + +You may need to modify some of the variables, including: + +* wazuh_manager_ip + + +.. note:: + + NOTE: + If you are using multiple environments, and you need to customise Wazuh in each environement, create override files in an appropriate directory, + for example `etc/kayobe/environments/production/inventory/group_vars/` + Files which values can be overridden (in context of Wazuh): + - etc/kayobe/inventory/group_vars/wazuh/wazuh-manager/wazuh-manager + - etc/kayobe/wazuh-manager.yml + - etc/kayobe/inventory/group_vars/wazuh/wazuh-agent/wazuh-agent + +Secrets +------- + +Wazuh secrets playbook is located in ``etc/kayobe/ansible/wazuh-secrets.yml``. +Running this playbook will generate and put pertinent security items into secrets +vault file which will be placed in ``$KAYOBE_CONFIG_PATH/wazuh-secrets.yml``. +If using environments it ends up in ``$KAYOBE_CONFIG_PATH/environments//wazuh-secrets.yml`` +Remember to encrypt! + +Wazuh secrets template is located in ``etc/kayobe/ansible/templates/wazuh-secrets.yml.j2``. +It will be used by wazuh secrets playbook to generate wazuh secrets vault file. + + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-secrets.yml + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/inventory/group_vars/wazuh/wazuh-manager/wazuh-secrets + + +TLS (optional) +-------------- + +You can generate your own TLS certificates, otherwise skip this section. +By default, Wazuh Ansible uses `wazuh-cert-tool.sh `__ +to automatically +generate certificates for wazuh-indexer (previously Elasticsearch and opendistro) +and wazuh-dashbooard (previously Kibana) using a local CA. +If the certificates directory ``etc/kayobe/ansible/wazuh/certificates`` +does not exist, it will generate the following certificates in ``etc/kayobe/ansible/wazuh/certificates/certs/`` +(here os-wazuh is set as ``elasticsearch_node_name`` and ``kibana_node_name``: + + +* Admin certificate for opendistro security + * admin-key.pem, admin.pem +* Node certificate + * os-wazuh-key.pem, os-wazuh.pem +* HTTP certificate for wazuh-dashboard (port 5601) & wazuh-indexer (port 9200) + * os-wazuh_http.key, os-wazuh_http.pem +* Root CA certificate + * root-ca.key root-ca.pem + + + +It is also possible to use externally generated certificates for wazuh-dashboard. root-ca.pem should contain the CA chain. +Those certificates can be uploaded to ``etc/kayobe/ansible/wazuh/custom_certificates``, +and will replace certificates generated by wazuh. +Certificates should have the same name scheme as those generated by wazuh (typicaly .pem) +The key for the external certificate should be in PKCS#8 format +(in its header it may have BEGIN PRIVATE KEY instead of BEGIN RSA PRIVATE KEY or BEGIN OPENSSH PRIVATE KEY). + +Example OpenSSL rune to convert to PKCS#8: + +``openssl pkcs8 -topk8 -nocrypt -in wazuh.key -out wazuh.key.pkcs8`` + +TODO: document how to use a local certificate. Do we need to override all certificates? + +Deploy +------ + +Deploy Wazuh manager: + +``kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-manager.yml`` + +If you are using the wazuh generated certificates, +this will result in the creation of some certificates and keys (in case of custom certs adjust path to it). +Encrypt the keys (and remember to commit to git): + + +``ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/ansible/wazuh/certificates/certs/*.key`` + +Verification +============== + +The Wazuh portal should be accessible on port 443 of the Wazuh +manager’s IPs (using HTTPS, with the root CA cert in ``etc/kayobe/ansible/wazuh/certificates/wazuh-certificates/root-ca.pem``). +The first login should be as the admin user, +with the opendistro_admin_password password in ``$KAYOBE_CONFIG_PATH/wazuh-secrets.yml``. +This will create the necessary indices. + +Troubleshooting + +Logs are in ``/var/log/wazuh-indexer/wazuh.log``. There are also logs in the journal. + +============ +Wazuh agents +============ + + +Wazuh agent playbook is located in ``etc/kayobe/ansible/wazuh-agent.yml``. + +Wazuh agent variables file is located in ``etc/kayobe/inventory/group_vars/wazuh-agent/wazuh-agent``. + +You may need to modify some variables, including: + +* wazuh_manager_address + +Deploy the Wazuh agents: + +``kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-agent.yml`` + +Verification +============= + +The Wazuh agents should register with the Wazuh manager. This can be verified via the agents page in Wazuh Portal. +Check CIS benchmark output in agent section. + +Additional resources: +===================== + +For times when you need to upgrade wazuh with elasticsearch to version with opensearch or you just need to deinstall all wazuh components: +Wazuh purge script: https://github.com/stackhpc/wazuh-server-purge diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index ad3c290dc..5ddf980a0 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -6,3 +6,6 @@ collections: version: 0.4.1 roles: - src: stackhpc.vxlan + - name: wazuh-ansible + src: https://github.com/stackhpc/wazuh-ansible + version: stackhpc diff --git a/etc/kayobe/ansible/templates/wazuh-secrets.yml.j2 b/etc/kayobe/ansible/templates/wazuh-secrets.yml.j2 new file mode 100644 index 000000000..73d3ab36d --- /dev/null +++ b/etc/kayobe/ansible/templates/wazuh-secrets.yml.j2 @@ -0,0 +1,14 @@ +--- +# Secrets used by Wazuh managers and agents +# Store these securely and use lookups here +secrets_wazuh: + # Wazuh agent authd pass + authd_pass: "{{ secrets_wazuh.authd_pass | default(lookup('password', '/dev/null'), true) }}" + # Strengthen default wazuh api user pass + wazuh_api_users: + - username: "wazuh" + password: "{{ secrets_wazuh.wazuh_api_users[0].password | default(lookup('password', '/dev/null length=30' ), true) }}" + # OpenSearch 'admin' user pass + opendistro_admin_password: "{{ secrets_wazuh.opendistro_admin_password | default(lookup('password', '/dev/null'), true) }}" + # OpenSearch 'kibanaserver' user pass + opendistro_kibana_password: "{{ secrets_wazuh.opendistro_kibana_password | default(lookup('password', '/dev/null'), true) }}" diff --git a/etc/kayobe/ansible/wazuh-agent.yml b/etc/kayobe/ansible/wazuh-agent.yml new file mode 100644 index 000000000..c14616fb9 --- /dev/null +++ b/etc/kayobe/ansible/wazuh-agent.yml @@ -0,0 +1,7 @@ +--- +- name: Deploy Wazuh agent + hosts: wazuh-agent + become: yes + tasks: + - import_role: + name: "wazuh-ansible/wazuh-ansible/roles/wazuh/ansible-wazuh-agent" diff --git a/etc/kayobe/ansible/wazuh-manager.yml b/etc/kayobe/ansible/wazuh-manager.yml new file mode 100644 index 000000000..1e669fbb6 --- /dev/null +++ b/etc/kayobe/ansible/wazuh-manager.yml @@ -0,0 +1,70 @@ +--- +# Certificates generation +- hosts: localhost + roles: + - role: "{{ playbook_dir }}/roles/wazuh-ansible/wazuh-ansible/roles/wazuh/wazuh-indexer" + perform_installation: false + become: no + tags: + - generate-certs +# Single node +- hosts: wazuh-manager + become: yes + become_user: root + roles: + - role: "{{ playbook_dir }}/roles/wazuh-ansible/wazuh-ansible/roles/wazuh/wazuh-indexer" + - role: "{{ playbook_dir }}/roles/wazuh-ansible/wazuh-ansible/roles/wazuh/ansible-wazuh-manager" + - role: "{{ playbook_dir }}/roles/wazuh-ansible/wazuh-ansible/roles/wazuh/ansible-filebeat-oss" + - role: "{{ playbook_dir }}/roles/wazuh-ansible/wazuh-ansible/roles/wazuh/wazuh-dashboard" + post_tasks: + - name: Set http/s_proxy vars in ossec-init.conf for vulnerability detector + blockinfile: + path: "/var/ossec/etc/ossec.conf" + state: present + owner: root + group: ossec + block: | + HTTPS_PROXY={{ http_proxy_url }} + HTTP_PROXY={{ http_proxy_url }} + backup: yes + when: http_proxy_url is defined + notify: + - Restart wazuh + + - name: Check if custom certificates exists + stat: + path: "{{ local_custom_certs_path }}" + register: custom_certificates_folder + delegate_to: localhost + become: no + + - name: Copy the node & admin certificates to Wazuh dashboard + copy: + src: "{{ local_custom_certs_path }}/{{ item }}" + dest: /etc/wazuh-dashboard/certs/ + owner: wazuh-dashboard + group: wazuh-dashboard + mode: 0400 + with_items: + - "{{ indexer_node_name }}-key.pem" + - "{{ indexer_node_name }}.pem" + when: custom_certificates_folder.stat.exists + notify: + - Restart wazuh-dashboard + + - name: Perform health check against filebeat + command: filebeat test output + changed_when: false + become: true + retries: 2 + + handlers: + - name: Restart wazuh + service: + name: wazuh-manager + state: restarted + + - name: Restart wazuh-dashboard + service: + name: wazuh-dashboard + state: restarted diff --git a/etc/kayobe/ansible/wazuh-secrets.yml b/etc/kayobe/ansible/wazuh-secrets.yml new file mode 100644 index 000000000..8b2064e71 --- /dev/null +++ b/etc/kayobe/ansible/wazuh-secrets.yml @@ -0,0 +1,27 @@ +--- +- hosts: localhost + gather_facts: false + vars: + wazuh_secrets_path: "{{ kayobe_env_config_path }}/wazuh-secrets.yml" + tasks: + - name: install passlib[bcrypt] + pip: + name: passlib[bcrypt] + virtualenv: "{{ ansible_playbook_python | dirname | dirname }}" + + - name: Ensure secrets directory exists + file: + path: "{{ wazuh_secrets_path | dirname }}" + state: directory + + - name: Template new secrets + template: + src: wazuh-secrets.yml.j2 + dest: "{{ wazuh_secrets_path }}" + notify: Please encrypt keys + + handlers: + - name: Please encrypt keys + debug: + msg: >- + Please encrypt the keys using Ansible Vault. diff --git a/etc/kayobe/inventory/group_vars/wazuh-agent/wazuh-agent b/etc/kayobe/inventory/group_vars/wazuh-agent/wazuh-agent new file mode 100644 index 000000000..e464b3385 --- /dev/null +++ b/etc/kayobe/inventory/group_vars/wazuh-agent/wazuh-agent @@ -0,0 +1,26 @@ +--- +# Wazuh-Agent role configuration +# Reference: https://documentation.wazuh.com/4.3/deploying-with-ansible/reference.html#wazuh-agent +# Defaults: https://github.com/wazuh/wazuh-ansible/blob/4.3/roles/wazuh/ansible-wazuh-agent/defaults/main.yml + +# Wazuh-Manager IP address +# Convenience var not used by wazuh-agent role +wazuh_manager_address: "{{ admin_oc_net_name | net_ip(groups['wazuh-manager'][0]) }}" + +# Wazuh-Manager API config +wazuh_managers: + - address: "{{ wazuh_manager_address }}" + port: 1514 + protocol: tcp + api_port: 55000 + +# Wazuh-Agent authd config +wazuh_agent_authd: + registration_address: "{{ wazuh_manager_address }}" + enable: true + port: 1515 + ssl_agent_ca: null + ssl_auto_negotiate: 'no' + +# Wazuh-Agent authd password +authd_pass: "{{ secrets_wazuh.authd_pass }}" diff --git a/etc/kayobe/inventory/group_vars/wazuh-manager/wazuh-manager b/etc/kayobe/inventory/group_vars/wazuh-manager/wazuh-manager new file mode 100644 index 000000000..9449683d9 --- /dev/null +++ b/etc/kayobe/inventory/group_vars/wazuh-manager/wazuh-manager @@ -0,0 +1,149 @@ +--- +# Wazuh deployment details and documetation: +# https://documentation.wazuh.com/current/deployment-options/deploying-with-ansible/index.html +# +# Defaults that are modified by this file: +# https://github.com/wazuh/wazuh-ansible/blob/4.3/roles/wazuh/ansible-filebeat-oss/defaults/main.yml +# https://github.com/wazuh/wazuh-ansible/blob/4.3/roles/wazuh/ansible-wazuh-manager/defaults/main.yml +# https://github.com/wazuh/wazuh-ansible/blob/4.3/roles/wazuh/wazuh-dashboard/defaults/main.yml +# https://github.com/wazuh/wazuh-ansible/blob/4.3/roles/wazuh/wazuh-indexer/defaults/main.yml +# https://github.com/wazuh/wazuh-ansible/blob/4.3/roles/wazuh/ansible-wazuh-manager/defaults/main.yml + +# Wazuh IP accessible from nodes running agents +# Change network as needed +wazuh_manager_ip: "{{ provision_oc_net_name | net_ip }}" + +# Single node setup +single_node: true +minimum_master_nodes: 1 +indexer_node_master: true + +# Ansible control host certificate directory +local_certs_path: "{{ playbook_dir }}/wazuh/certificates" + +# Ansible control host custom certificates directory +local_custom_certs_path: "{{ playbook_dir }}/wazuh/custom_certificates" + +# Indexer variables +indexer_node_name: "{{ inventory_hostname }}" + +# IP to which indexer will bind on host. +indexer_network_host: "{{ provision_oc_net_name | net_ip }}" + +# Even in a single node setup this must be defined. If not defaults to 127.0.0.1 +indexer_cluster_nodes: + - "{{ indexer_network_host }}" + +instances: + node1: + name: "{{ groups['wazuh-manager'][0] }}" + ip: "{{ admin_oc_net_name | net_ip(groups['wazuh-manager'][0]) }}" + role: indexer + +# Filebeat variables +filebeat_node_name: "{{ indexer_node_name }}" + +filebeat_output_indexer_hosts: + - "{{ indexer_network_host }}" + +# Dashboard variables +dashboard_node_name: "{{ indexer_node_name }}" + +dashboard_server_host: "{{ indexer_network_host }}" + +dashboard_server_name: "{{ indexer_node_name }}" + +indexer_admin_password: "{{ secrets_wazuh.opendistro_admin_password }}" + +dashboard_user: kibanaserver + +# TODO refactor creation and usage of secrets_wazuh +dashboard_password: "{{ secrets_wazuh.opendistro_kibana_password }}" + +# Indexer filebeat user +indexer_security_user: admin + +# Indexer filebeat password +indexer_security_password: "{{ secrets_wazuh.opendistro_admin_password }}" + +# Used to override the default Wazuh api user +wazuh_api_users: "{{ secrets_wazuh.wazuh_api_users }}" + +# Dashboard to Wazuh api credentials +wazuh_api_credentials: + - id: "default" + url: "https://{{ indexer_network_host }}" + port: 55000 + username: "wazuh" + password: "{{ secrets_wazuh.wazuh_api_users | selectattr('username', 'match', 'wazuh') | map(attribute='password') | first }}" + +# Perform hash_behaviour=merge at role runtime +wazuh_manager_config_overlay: true + +# JSON formatted alerts to /var/ossec/logs/alerts/alerts.json +wazuh_manager_json_output: 'yes' + +# Plain text alerts to /var/ossec/logs/alerts/alerts.log +wazuh_manager_alerts_log: 'yes' + +# Store events that do not trip rules +wazuh_manager_logall: 'no' + +# Email alert notifications +wazuh_manager_email_notification: 'no' + +# Log format of /var/ossec/logs/ossec.{log|json} +wazuh_manager_log_format: 'plain' + +authd_pass: "{{ secrets_wazuh.authd_pass }}" + +# Wazuh-Manager listener settings +wazuh_manager_connection: + - type: 'secure' + port: '1514' + protocol: 'tcp' + queue_size: 131072 + +# Wazuh-Manager authd settings +# TODO, enable SSL/TLS +wazuh_manager_authd: + enable: true + port: 1515 + use_source_ip: 'no' + force: + enabled: 'no' + key_mismatch: 'no' + disconnected_time: '0' + after_registration_time: '0' + purge: 'no' + use_password: 'yes' + ciphers: 'HIGH:!ADH:!EXP:!MD5:!RC4:!3DES:!CAMELLIA:@STRENGTH' + ssl_agent_ca: null + ssl_verify_host: 'no' + ssl_manager_cert: null + ssl_manager_key: null + ssl_auto_negotiate: 'no' + +# Wazuh-Manager Vulnerability Detector settings +wazuh_manager_vulnerability_detector: + enabled: 'yes' + interval: '5m' + ignore_time: '6h' + run_on_start: 'yes' + providers: + - enabled: 'yes' + os: + - 'bionic' + - 'focal' + update_interval: '1h' + name: '"canonical"' + - enabled: 'yes' + os: + - '7' + - '8' + update_interval: '1h' + name: '"redhat"' + - enabled: 'yes' + update_from_year: '2010' + update_interval: '1h' + name: '"nvd"' diff --git a/etc/kayobe/inventory/groups b/etc/kayobe/inventory/groups index e51195309..fa0a55a1c 100644 --- a/etc/kayobe/inventory/groups +++ b/etc/kayobe/inventory/groups @@ -24,6 +24,13 @@ seed-hypervisor [infra-vms] # Empty group to provide declaration of infra-vms group. +[wazuh-manager] +# Empty group to provide declaration of wazuh-manager group. + +[wazuh-agent:children] +seed +overcloud + ############################################################################### # Overcloud groups. diff --git a/releasenotes/notes/add_wazuh-be2999448db5cb7c.yaml b/releasenotes/notes/add_wazuh-be2999448db5cb7c.yaml new file mode 100644 index 000000000..571bc9d28 --- /dev/null +++ b/releasenotes/notes/add_wazuh-be2999448db5cb7c.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add Wazuh deployment playbook. + From 1cfefbd1a12f18c414546f957b9afb43fb1aa18d Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 13 Apr 2023 11:27:15 +0100 Subject: [PATCH 270/292] Drop Nova fork Our downstream patch for OSSA-2023-002 has landed upstream. --- etc/kayobe/kolla.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 89dd03d3b..b24e68d9f 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -145,10 +145,6 @@ kolla_sources: type: git location: https://github.com/stackhpc/networking-generic-switch.git reference: stackhpc/xena - nova-base: - type: git - location: https://github.com/stackhpc/nova.git - reference: stackhpc/{{ openstack_release }} ############################################################################### # Kolla image build configuration. From 6e6c9c4861bc0c9d559587efb377b8c37a6b3992 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 14 Apr 2023 17:31:01 +0200 Subject: [PATCH 271/292] Add defaults for cephadm_commands_{pre,post} Without a default value we get: 'cephadm_commands_pre' is undefined --- etc/kayobe/ansible/cephadm-commands-post.yml | 2 +- etc/kayobe/ansible/cephadm-commands-pre.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/ansible/cephadm-commands-post.yml b/etc/kayobe/ansible/cephadm-commands-post.yml index fbca30dfd..d60a135a3 100644 --- a/etc/kayobe/ansible/cephadm-commands-post.yml +++ b/etc/kayobe/ansible/cephadm-commands-post.yml @@ -10,4 +10,4 @@ - import_role: name: stackhpc.cephadm.commands vars: - cephadm_commands: "{{ cephadm_commands_post }}" + cephadm_commands: "{{ cephadm_commands_post | default([]) }}" diff --git a/etc/kayobe/ansible/cephadm-commands-pre.yml b/etc/kayobe/ansible/cephadm-commands-pre.yml index 08047cc51..abe58caa5 100644 --- a/etc/kayobe/ansible/cephadm-commands-pre.yml +++ b/etc/kayobe/ansible/cephadm-commands-pre.yml @@ -10,4 +10,4 @@ - import_role: name: stackhpc.cephadm.commands vars: - cephadm_commands: "{{ cephadm_commands_pre }}" + cephadm_commands: "{{ cephadm_commands_pre | default([]) }}" From fcb0b893ea3278096fca5997be02ad75a76b1644 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 3 Apr 2023 16:14:28 +0100 Subject: [PATCH 272/292] Add documentation for Magnum in a Multinode --- .../contributor/environments/ci-multinode.rst | 121 ++++++++++++++++++ ...ultinode-magnum-docs-00607f139dc2b53e.yaml | 5 + 2 files changed, 126 insertions(+) create mode 100644 releasenotes/notes/multinode-magnum-docs-00607f139dc2b53e.yaml diff --git a/doc/source/contributor/environments/ci-multinode.rst b/doc/source/contributor/environments/ci-multinode.rst index 93341113a..9f266d97b 100644 --- a/doc/source/contributor/environments/ci-multinode.rst +++ b/doc/source/contributor/environments/ci-multinode.rst @@ -203,3 +203,124 @@ instance: ls testdir If it shows the test file then the share is working correctly. + +Magnum +====== + +The Multinode environment has Magnum enabled by default. To test it, you will +need to create a Kubernetes cluster. It is recommended that you use the +specified Fedora 35 image, as others may not work. Download the image locally, +then extract it and upload it to glance: + +.. code-block:: bash + + wget https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/35.20220410.3.1/x86_64/fedora-coreos-35.20220410.3.1-openstack.x86_64.qcow2.xz + unxz fedora-coreos-35.20220410.3.1-openstack.x86_64.qcow2.xz + openstack image create --container-format bare --disk-format qcow2 --property os_distro='fedora-coreos' --property os_version='35' --file fedora-coreos-35.20220410.3.1-openstack.x86_64.qcow2 fedora-coreos-35 --progress + +Create a keypair: + +.. code-block:: bash + + openstack keypair create --private-key ~/.ssh/id_rsa id_rsa + +Install the Magnum, Heat, and Octavia clients: + +.. code-block:: bash + + pip install python-magnumclient + pip install python-heatclient + pip install python-octaviaclient + +Create a cluster template: + +.. code-block:: bash + + openstack coe cluster template create test-template --image fedora-coreos-35 --external-network external --labels etcd_volume_size=8,boot_volume_size=50,cloud_provider_enabled=true,heat_container_agent_tag=wallaby-stable-1,kube_tag=v1.23.6,cloud_provider_tag=v1.23.1,monitoring_enabled=true,auto_scaling_enabled=true,auto_healing_enabled=true,auto_healing_controller=magnum-auto-healer,magnum_auto_healer_tag=v1.23.0.1-shpc,etcd_tag=v3.5.4,master_lb_floating_ip_enabled=true,cinder_csi_enabled=true,container_infra_prefix=ghcr.io/stackhpc/,min_node_count=1,max_node_count=50,octavia_lb_algorithm=SOURCE_IP_PORT,octavia_provider=ovn --dns-nameserver 8.8.8.8 --flavor m1.medium --master-flavor m1.medium --network-driver calico --volume-driver cinder --docker-storage-driver overlay2 --floating-ip-enabled --master-lb-enabled --coe kubernetes + +Create a cluster: + +.. code-block:: bash + + openstack coe cluster create --keypair id_rsa --master-count 1 --node-count 1 --floating-ip-enabled test-cluster + +This command will take a while to complete. You can monitor the progress with +the following command: + +.. code-block:: bash + + watch "openstack --insecure coe cluster list ; openstack --insecure stack list ; openstack --insecure server list" + +Once the cluster is created, you can SSH into the master node and check that +there are no failed containers: + +.. code-block:: bash + + ssh core@{master-ip} + +List the podman and docker containers: + +.. code-block:: bash + + sudo docker ps + sudo podman ps + +If there are any failed containers, you can check the logs with the following +commands: + +.. code-block:: bash + + sudo docker logs {container-id} + sudo podman logs {container-id} + +Or look at the logs under ``/var/log``. In particular, pay close attention to +``/var/log/heat-config`` on the master and +``/var/log/kolla/{magnum,heat,neutron}/*`` on the controllers. + +Otherwise, the ``state`` of the cluster should eventually become +``CREATE_COMPLETE`` and the ``health_status`` should be ``HEALTHY``. + +You can interact with the cluster using ``kubectl``. The instructions for +installing ``kubectl`` are available `here +`_. You can then +configure ``kubectl`` to use the cluster, and check that the pods are all +running: + +.. code-block:: bash + + openstack coe cluster config test-cluster --dir $PWD + export KUBECONFIG=$PWD/config + kubectl get pods -A + +Finally, you can optionally use sonobuoy to run a complete set of Kubernetes +conformance tests. + +Find the latest release of sonobuoy on their `github releases page +`_. Then download it with wget, e.g.: + +.. code-block:: bash + + wget https://github.com/vmware-tanzu/sonobuoy/releases/download/v0.56.16/sonobuoy_0.56.16_linux_amd64.tar.gz + +Extract it with tar: + +.. code-block:: bash + + tar -xvf sonobuoy_0.56.16_linux_amd64.tar.gz + +And run it: + +.. code-block:: bash + + ./sonobuoy run --wait + +This will take a while to complete. Once it is done you can check the results +with: + +.. code-block:: bash + + results=$(./sonobuoy retrieve) + ./sonobuoy results $results + +There are various other options for sonobuoy, see the `documentation +`_ for more details. diff --git a/releasenotes/notes/multinode-magnum-docs-00607f139dc2b53e.yaml b/releasenotes/notes/multinode-magnum-docs-00607f139dc2b53e.yaml new file mode 100644 index 000000000..004e3b1f0 --- /dev/null +++ b/releasenotes/notes/multinode-magnum-docs-00607f139dc2b53e.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Updated the documentation for the ci-multinode to include instructions on + how to set up and test Magnum. From e3b942676e482b1fd1486bd522f3ca7b67a354f5 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 17 Apr 2023 12:15:59 +0100 Subject: [PATCH 273/292] Add geneve to multinode if ovn is enabled --- etc/kayobe/environments/ci-multinode/neutron.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/etc/kayobe/environments/ci-multinode/neutron.yml b/etc/kayobe/environments/ci-multinode/neutron.yml index 1a7fd8255..44f50d00c 100644 --- a/etc/kayobe/environments/ci-multinode/neutron.yml +++ b/etc/kayobe/environments/ci-multinode/neutron.yml @@ -1,3 +1,12 @@ --- kolla_neutron_ml2_network_vlan_ranges: - physical_network: "physnet1" + +kolla_neutron_ml2_type_drivers: + - flat + - vlan + - "{{ 'geneve' if kolla_enable_ovn | bool else 'vxlan' }}" + +kolla_neutron_ml2_tenant_network_types: + - vlan + - "{{ 'geneve' if kolla_enable_ovn | bool else 'vxlan' }}" From 54b79534cfd314db58015f9ddf1722de062f7815 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 17 Apr 2023 16:24:46 +0100 Subject: [PATCH 274/292] Fix multinode failures from seed misconfiguration --- etc/kayobe/environments/ci-multinode/seed.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/etc/kayobe/environments/ci-multinode/seed.yml b/etc/kayobe/environments/ci-multinode/seed.yml index 749c398d1..5e3f8bffe 100644 --- a/etc/kayobe/environments/ci-multinode/seed.yml +++ b/etc/kayobe/environments/ci-multinode/seed.yml @@ -3,9 +3,7 @@ seed_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' else 'cl seed_lvm_groups: - "{{ stackhpc_lvm_group_rootvg }}" -seed_extra_network_interfaces: > - "{{ seed_extra_network_interfaces_external + - (seed_extra_network_interfaces_manila if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else []) }}" +seed_extra_network_interfaces: "{{ seed_extra_network_interfaces_external + seed_extra_network_interfaces_manila if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else [] }}" # Seed has been provided an external interface # for tempest tests and SSH access to machines. @@ -26,6 +24,6 @@ snat_rules_default: source_ip: "{{ ansible_facts.default_ipv4.address }}" snat_rules_manila: - interface: "{{ storage_interface }}" - source_ip: "{{ ansible_facts[storage_interface].ipv4.address }}" + source_ip: "{{ ansible_facts[storage_interface].ipv4.address | default }}" # Only add the storage snat rule if we are using manila-cephfs. snat_rules: "{{ snat_rules_default + snat_rules_manila if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else snat_rules_default }}" From cc1a8637212a2a893205ea8c6be0d2559e9f82c8 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Tue, 25 Apr 2023 10:02:31 +0100 Subject: [PATCH 275/292] Fix seed_extra_network_interfaces without Manila --- etc/kayobe/environments/ci-multinode/seed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/environments/ci-multinode/seed.yml b/etc/kayobe/environments/ci-multinode/seed.yml index 5e3f8bffe..bb9e3c6bf 100644 --- a/etc/kayobe/environments/ci-multinode/seed.yml +++ b/etc/kayobe/environments/ci-multinode/seed.yml @@ -3,7 +3,7 @@ seed_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' else 'cl seed_lvm_groups: - "{{ stackhpc_lvm_group_rootvg }}" -seed_extra_network_interfaces: "{{ seed_extra_network_interfaces_external + seed_extra_network_interfaces_manila if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else [] }}" +seed_extra_network_interfaces: "{{ seed_extra_network_interfaces_external + (seed_extra_network_interfaces_manila if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else []) }}" # Seed has been provided an external interface # for tempest tests and SSH access to machines. From 7e502ff063052467ed1c7dfda190cb623990b52f Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Tue, 25 Apr 2023 11:03:46 +0100 Subject: [PATCH 276/292] fix typo in multinode-magnum docs --- doc/source/contributor/environments/ci-multinode.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/contributor/environments/ci-multinode.rst b/doc/source/contributor/environments/ci-multinode.rst index 9f266d97b..4bef5fd39 100644 --- a/doc/source/contributor/environments/ci-multinode.rst +++ b/doc/source/contributor/environments/ci-multinode.rst @@ -242,7 +242,7 @@ Create a cluster: .. code-block:: bash - openstack coe cluster create --keypair id_rsa --master-count 1 --node-count 1 --floating-ip-enabled test-cluster + openstack coe cluster create --cluster-template test-template --keypair id_rsa --master-count 1 --node-count 1 --floating-ip-enabled test-cluster This command will take a while to complete. You can monitor the progress with the following command: From b3d767fe57d5ac3000d5da775cadf0c4a7a0aefa Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Tue, 2 May 2023 18:24:26 +0200 Subject: [PATCH 277/292] Update Pulp to 3.23 --- etc/kayobe/seed.yml | 2 +- releasenotes/notes/update-pulp-3.23-b4193a417a219d26.yaml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/update-pulp-3.23-b4193a417a219d26.yaml diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index b575ab82a..e900a587b 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -106,7 +106,7 @@ seed_pulp_container: image: pulp/pulp pre: "{{ kayobe_config_path }}/containers/pulp/pre.yml" post: "{{ kayobe_config_path }}/containers/pulp/post.yml" - tag: "{{ '3.22-https' if pulp_enable_tls | bool else '3.22' }}" + tag: "{{ '3.23-https' if pulp_enable_tls | bool else '3.23' }}" network_mode: host # Override deploy_containers_defaults.init == true to ensure # s6-overlay-suexec starts as pid 1 diff --git a/releasenotes/notes/update-pulp-3.23-b4193a417a219d26.yaml b/releasenotes/notes/update-pulp-3.23-b4193a417a219d26.yaml new file mode 100644 index 000000000..f3019561c --- /dev/null +++ b/releasenotes/notes/update-pulp-3.23-b4193a417a219d26.yaml @@ -0,0 +1,3 @@ +--- +features: + - Upgrades Pulp from ``3.22`` to ``3.23``. From 741be7c94659e082a0440ffb9cc9f8fd6249f3c6 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Wed, 3 May 2023 10:41:22 +0100 Subject: [PATCH 278/292] Bump stackhpc cephadm ansible collection version --- etc/kayobe/ansible/requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index 5ddf980a0..b0c89d3d1 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -1,7 +1,7 @@ --- collections: - name: stackhpc.cephadm - version: 1.12.2 + version: 1.13.1 - name: stackhpc.pulp version: 0.4.1 roles: From f2f389fd399375bd4d0bbf57314b1e7b6de2d48d Mon Sep 17 00:00:00 2001 From: Alex-Welsh <112560678+Alex-Welsh@users.noreply.github.com> Date: Wed, 10 May 2023 12:23:56 +0100 Subject: [PATCH 279/292] Add workflows to build and promote overcloud host images (#509) * CI: Add overcloud-host-image-build workflow * CI: Workaround lack of internal DNS in SMS lab Use an etc-hosts DIB element to add the test Pulp server to /etc/hosts in the image. * Draft layout for pulp image hosting * upload-image removing placeholder vars * Renaming, and various fixes for, pulp_upload_image * Image upload functioning with workarounds * renaming pulp-upload-image, removing workarounds * Finalised pulp-upload-image and workflow * updated etc hosts workaround * Adding overcloud host image sources * fix ubuntu image url typo, add pulp url with auth * move image sources to separate file & small fixes * Add image promotion workflow * Rename pulp overcloud host image playbooks Rename ansible playbooks handling overcloud host image promotion * Revert changes to pull pulp overcloud host images * Correct host image promote workflow input syntax Co-authored-by: Mark Goddard --------- Co-authored-by: Mark Goddard --- .../workflows/overcloud-host-image-build.yml | 177 ++++++++++++++++++ .../overcloud-host-image-promote.yml | 121 ++++++++++++ .../ansible/pulp-host-image-promote.yml | 40 ++++ etc/kayobe/ansible/pulp-host-image-upload.yml | 109 +++++++++++ .../environments/ci-builder/overcloud-dib.yml | 20 ++ etc/kayobe/overcloud-dib.yml | 2 +- etc/kayobe/stackhpc-overcloud-dib.yml | 4 + 7 files changed, 472 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/overcloud-host-image-build.yml create mode 100644 .github/workflows/overcloud-host-image-promote.yml create mode 100644 etc/kayobe/ansible/pulp-host-image-promote.yml create mode 100644 etc/kayobe/ansible/pulp-host-image-upload.yml create mode 100644 etc/kayobe/environments/ci-builder/overcloud-dib.yml diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml new file mode 100644 index 000000000..cf9b62806 --- /dev/null +++ b/.github/workflows/overcloud-host-image-build.yml @@ -0,0 +1,177 @@ +--- +name: Build overcloud host images +on: + workflow_dispatch: + inputs: + centos: + description: Build CentOS Stream + type: boolean + default: true + rocky: + description: Build Rocky Linux + type: boolean + default: true + ubuntu: + description: Build Ubuntu + type: boolean + default: true + +env: + ANSIBLE_FORCE_COLOR: True +jobs: + overcloud-host-image-build: + name: Build overcloud host images + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder] + steps: + - uses: actions/checkout@v3 + with: + path: src/kayobe-config + + - name: Clone StackHPC Kayobe repository + uses: actions/checkout@v3 + with: + repository: stackhpc/kayobe + ref: refs/heads/stackhpc/xena + path: src/kayobe + + # FIXME: Failed in kolla-ansible : Ensure the latest version of pip is installed + - name: Install dependencies + run: | + sudo dnf -y install python3-virtualenv + + - name: Setup networking + run: | + if ! ip l show breth1 >/dev/null 2>&1; then + sudo ip l add breth1 type bridge + fi + sudo ip l set breth1 up + if ! ip a show breth1 | grep 192.168.33.3/24; then + sudo ip a add 192.168.33.3/24 dev breth1 + fi + if ! ip l show dummy1 >/dev/null 2>&1; then + sudo ip l add dummy1 type dummy + fi + sudo ip l set dummy1 up + sudo ip l set dummy1 master breth1 + + - name: Install Kayobe + run: | + mkdir -p venvs && + pushd venvs && + python3 -m venv kayobe && + source kayobe/bin/activate && + pip install -U pip && + pip install ../src/kayobe + + - name: Bootstrap the control host + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe control host bootstrap + + - name: Configure the seed host + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host configure + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + + - name: Create bifrost_httpboot Docker volume + run: | + if [[ $(sudo docker volume ls -f Name=bifrost_httpboot -q | wc -l) = 0 ]]; then + sudo docker volume create bifrost_httpboot + fi + + - name: Clean any previous build artifact + run: | + rm -f /tmp/updated_images.txt + + - name: Build a CentOS Stream 8 overcloud host image + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe overcloud host image build --force-rebuild \ + -e os_distribution=centos \ + -e stackhpc_overcloud_dib_name=overcloud-centos-stream-8 + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: inputs.centos + + - name: Upload CentOS Stream 8 overcloud host image artifact + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \ + -e image_path='/opt/kayobe/images/overcloud-centos-stream-8' \ + -e os_distribution='centos' + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: inputs.centos + + - name: Build a Rocky Linux 8 overcloud host image + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe overcloud host image build --force-rebuild \ + -e os_distribution=rocky \ + -e stackhpc_overcloud_dib_name=overcloud-rocky-linux-8 + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: inputs.rocky + + - name: Upload Rocky Linux 8 overcloud host image artifact + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \ + -e image_path='/opt/kayobe/images/overcloud-rocky-linux-8' \ + -e os_distribution='rocky' + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: inputs.rocky + + # FIXME: Need EPEL on CentOS for debootstrap. It is disabled by default. + # Do this via config? + - name: Enable EPEL repository + run: | + sudo dnf config-manager --set-enabled epel + + - name: Build a Ubuntu Focal 20.04 overcloud host image + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe overcloud host image build --force-rebuild \ + -e os_distribution=ubuntu \ + -e stackhpc_overcloud_dib_name=overcloud-ubuntu-focal + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: inputs.ubuntu + + - name: Upload Ubuntu Focal 20.04 overcloud host image artifact + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \ + -e image_path='/opt/kayobe/images/overcloud-ubuntu-focal' \ + -e os_distribution='ubuntu' + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: inputs.ubuntu + + - name: Upload updated images artifact + uses: actions/upload-artifact@v3 + with: + name: Updated images list + path: /tmp/updated_images.txt + retention-days: 7 + if: always() + + - name: Clean up old images + run: | + sudo rm -rf /opt/kayobe/images/ + if: always() diff --git a/.github/workflows/overcloud-host-image-promote.yml b/.github/workflows/overcloud-host-image-promote.yml new file mode 100644 index 000000000..cacbd575e --- /dev/null +++ b/.github/workflows/overcloud-host-image-promote.yml @@ -0,0 +1,121 @@ +--- +name: Promote overcloud host image +on: + workflow_dispatch: + inputs: + os_image: + description: Image to promote + type: choice + required: true + default: 'CentOS Stream 8' + options: + - 'CentOS Stream 8' + - 'Rocky Linux 8' + - 'Ubuntu Focal 20.04' + image_tag: + description: Tag to promote + type: string + required: true +env: + ANSIBLE_FORCE_COLOR: True +jobs: + overcloud-host-image-promote: + name: Promote overcloud host image + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder] + steps: + - uses: actions/checkout@v3 + with: + path: src/kayobe-config + + - name: Clone StackHPC Kayobe repository + uses: actions/checkout@v3 + with: + repository: stackhpc/kayobe + ref: refs/heads/stackhpc/xena + path: src/kayobe + + # FIXME: Failed in kolla-ansible : Ensure the latest version of pip is installed + - name: Install dependencies + run: | + sudo dnf -y install python3-virtualenv + + - name: Setup networking + run: | + if ! ip l show breth1 >/dev/null 2>&1; then + sudo ip l add breth1 type bridge + fi + sudo ip l set breth1 up + if ! ip a show breth1 | grep 192.168.33.3/24; then + sudo ip a add 192.168.33.3/24 dev breth1 + fi + if ! ip l show dummy1 >/dev/null 2>&1; then + sudo ip l add dummy1 type dummy + fi + sudo ip l set dummy1 up + sudo ip l set dummy1 master breth1 + + - name: Install Kayobe + run: | + mkdir -p venvs && + pushd venvs && + python3 -m venv kayobe && + source kayobe/bin/activate && + pip install -U pip && + pip install ../src/kayobe + + - name: Bootstrap the control host + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe control host bootstrap + + - name: Configure the seed host + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host configure + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + + - name: Promote CentOS Stream 8 overcloud host image artifact + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-host-image-promote.yml \ + -e image_path='/opt/kayobe/images/overcloud-centos-stream-8' \ + -e os_distribution='centos' \ + -e os_release='8-stream' + env: + OVERCLOUD_HOST_IMAGE_TAG: ${{ inputs.image_tag }} + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: os_image == 'CentOS Stream 8' + + - name: Promote Rocky Linux 8 overcloud host image artifact + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-host-image-promote.yml \ + -e image_path='/opt/kayobe/images/overcloud-rocky-linux-8' \ + -e os_distribution='rocky' \ + -e os_release='8' + env: + OVERCLOUD_HOST_IMAGE_TAG: ${{ inputs.image_tag }} + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: os_image == 'Rocky Linux 8' + + - name: Promote Ubuntu Focal 20.04 overcloud host image artifact + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-host-image-promote.yml \ + -e image_path='/opt/kayobe/images/overcloud-ubuntu-focal' \ + -e os_distribution='ubuntu' \ + -e os_release='focal' + env: + OVERCLOUD_HOST_IMAGE_TAG: ${{ inputs.image_tag }} + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: os_image == 'Ubuntu Focal 20.04' diff --git a/etc/kayobe/ansible/pulp-host-image-promote.yml b/etc/kayobe/ansible/pulp-host-image-promote.yml new file mode 100644 index 000000000..d93d71d51 --- /dev/null +++ b/etc/kayobe/ansible/pulp-host-image-promote.yml @@ -0,0 +1,40 @@ +--- +# Tag is the version of the image to promote, stored in an env variable named OVERCLOUD_HOST_IMAGE_TAG +- name: Promote an image to production + hosts: localhost + vars: + remote_pulp_url: "{{ stackhpc_release_pulp_url }}" + remote_pulp_username: "{{ stackhpc_image_repository_username }}" + remote_pulp_password: "{{ stackhpc_image_repository_password }}" + repository_name: "kayobe-images-{{ openstack_release }}-{{ os_distribution }}-{{ os_release }}" + base_path: "kayobe-images/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}" + promotion_tag: "{{ lookup('env', 'OVERCLOUD_HOST_IMAGE_TAG') }}" + tasks: + + - name: Check whether the image exists + pulp.squeezer.file_distribution: + pulp_url: "{{ remote_pulp_url }}" + username: "{{ remote_pulp_username }}" + password: "{{ remote_pulp_password }}" + name: "{{ repository_name }}_{{ promotion_tag }}" + base_path: "{{ base_path }}/{{ promotion_tag }}" + register: distribution_details + + - name: Fail if the image does not exist + fail: + msg: "Image {{ promotion_tag }} does not exist" + when: distribution_details.distribution is none + + - name: Ensure production content guard is set + pulp.squeezer.file_distribution: + pulp_url: "{{ remote_pulp_url }}" + username: "{{ remote_pulp_username }}" + password: "{{ remote_pulp_password }}" + name: "{{ repository_name }}_{{ promotion_tag }}" + base_path: "{{ base_path }}/{{ promotion_tag }}" + content_guard: release + state: present + + - name: Print version tag and os + debug: + msg: "Promoted tag: {{ promotion_tag }}" diff --git a/etc/kayobe/ansible/pulp-host-image-upload.yml b/etc/kayobe/ansible/pulp-host-image-upload.yml new file mode 100644 index 000000000..b2945c83f --- /dev/null +++ b/etc/kayobe/ansible/pulp-host-image-upload.yml @@ -0,0 +1,109 @@ +--- +- name: Upload and create a distribution for an image + hosts: localhost + vars: + remote_pulp_url: "{{ stackhpc_release_pulp_url }}" + remote_pulp_username: "{{ stackhpc_image_repository_username }}" + remote_pulp_password: "{{ stackhpc_image_repository_password }}" + repository_name: "kayobe-images-{{ openstack_release }}-{{ os_distribution }}-{{ os_release }}" + base_path: "kayobe-images/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}" + tasks: + - name: Get filename + find: + paths: "{{ image_path }}" + patterns: '*.qcow2' + register: found_files + + - name: Upload an artifact + pulp.squeezer.artifact: + pulp_url: "{{ remote_pulp_url }}" + username: "{{ remote_pulp_username }}" + password: "{{ remote_pulp_password }}" + file: "{{ found_files.files[0].path }}" + state: present + + - name: Get sha256 hash + ansible.builtin.stat: + path: "{{ found_files.files[0].path }}" + checksum_algorithm: sha256 + register: file_stats + + - name: Create file content from artifact + pulp.squeezer.file_content: + pulp_url: "{{ remote_pulp_url }}" + username: "{{ remote_pulp_username }}" + password: "{{ remote_pulp_password }}" + sha256: "{{ file_stats.stat.checksum }}" + relative_path: "{{ found_files.files[0].path | basename }}" + state: present + + - name: Ensure file repo exists + pulp.squeezer.file_repository: + pulp_url: "{{ remote_pulp_url }}" + username: "{{ remote_pulp_username }}" + password: "{{ remote_pulp_password }}" + name: "{{ repository_name }}" + state: present + + - name: Add content to file repo + pulp.squeezer.file_repository_content: + pulp_url: "{{ remote_pulp_url }}" + username: "{{ remote_pulp_username }}" + password: "{{ remote_pulp_password }}" + repository: "{{ repository_name }}" + present_content: + - relative_path: "{{ found_files.files[0].path | basename }}" + sha256: "{{ file_stats.stat.checksum }}" + + - name: Create a new publication to point to this version + pulp.squeezer.file_publication: + pulp_url: "{{ remote_pulp_url }}" + username: "{{ remote_pulp_username }}" + password: "{{ remote_pulp_password }}" + repository: "{{ repository_name }}" + state: present + register: publication_details + + - name: Update distribution for latest version + pulp.squeezer.file_distribution: + pulp_url: "{{ remote_pulp_url }}" + username: "{{ remote_pulp_username }}" + password: "{{ remote_pulp_password }}" + name: "{{ repository_name }}_latest" + base_path: "{{ base_path }}/latest" + publication: "{{ publication_details.publication.pulp_href }}" + content_guard: development + state: present + register: latest_distribution_details + + - name: Create distribution for given version + pulp.squeezer.file_distribution: + pulp_url: "{{ remote_pulp_url }}" + username: "{{ remote_pulp_username }}" + password: "{{ remote_pulp_password }}" + name: "{{ repository_name }}_{{ ansible_facts.date_time.iso8601_basic_short }}" + base_path: "{{ base_path }}/{{ ansible_facts.date_time.iso8601_basic_short }}" + publication: "{{ publication_details.publication.pulp_href }}" + content_guard: development + state: present + when: latest_distribution_details.changed + + - name: Update new images file with versioned path + lineinfile: + path: /tmp/updated_images.txt + line: "{{ remote_pulp_url }}/pulp/content/{{ base_path }}/\ + {{ ansible_facts.date_time.iso8601_basic_short }}/{{ found_files.files[0].path | basename }}" + create: true + when: latest_distribution_details.changed + + - name: Update new images file with latest path + lineinfile: + path: /tmp/updated_images.txt + line: "{{ remote_pulp_url }}/pulp/content/{{ base_path }}/\ + latest/{{ found_files.files[0].path | basename }}" + when: latest_distribution_details.changed + + - name: Print version tag + debug: + msg: "New tag: {{ ansible_facts.date_time.iso8601_basic_short }}" + when: latest_distribution_details.changed diff --git a/etc/kayobe/environments/ci-builder/overcloud-dib.yml b/etc/kayobe/environments/ci-builder/overcloud-dib.yml new file mode 100644 index 000000000..6a8f3b0f1 --- /dev/null +++ b/etc/kayobe/environments/ci-builder/overcloud-dib.yml @@ -0,0 +1,20 @@ +--- +# Overcloud host disk image configuration. + +############################################################################### +# Diskimage-builder configuration for overcloud host disk images. + +# Whether to build host disk images with DIB directly instead of through +# Bifrost. Setting it to true disables Bifrost image build and allows images to +# be built with the `kayobe overcloud host image build` command. Default value +# is False. This will change in a future release. +overcloud_dib_build_host_images: true + +# List of overcloud host disk images to build. Each element is a dict defining +# an image in a format accepted by the stackhpc.os-images role. Default is to +# build an image named "deployment_image" configured with the overcloud_dib_* +# variables defined below: {"name": "deployment_image", "elements": "{{ +# overcloud_dib_elements }}", "env": "{{ overcloud_dib_env_vars }}", +# "packages": "{{ overcloud_dib_packages }}"}. +overcloud_dib_host_images: + - "{{ stackhpc_overcloud_dib_host_image }}" diff --git a/etc/kayobe/overcloud-dib.yml b/etc/kayobe/overcloud-dib.yml index 9c34650ff..46a02830b 100644 --- a/etc/kayobe/overcloud-dib.yml +++ b/etc/kayobe/overcloud-dib.yml @@ -71,7 +71,7 @@ overcloud_dib_host_packages_extra: overcloud_dib_git_elements_extra: - repo: "https://github.com/stackhpc/stackhpc-image-elements" local: "{{ source_checkout_path }}/stackhpc-image-elements" - version: "v1.4.0" + version: "v1.5.0" elements_path: "elements" # List of git repositories containing Diskimage Builder (DIB) elements. See diff --git a/etc/kayobe/stackhpc-overcloud-dib.yml b/etc/kayobe/stackhpc-overcloud-dib.yml index a39e59608..5ba915c25 100644 --- a/etc/kayobe/stackhpc-overcloud-dib.yml +++ b/etc/kayobe/stackhpc-overcloud-dib.yml @@ -25,6 +25,7 @@ stackhpc_overcloud_dib_elements: - "cloud-init-datasources" - "{% if os_distribution in ['centos', 'rocky'] %}disable-selinux{% endif %}" - "enable-serial-console" + - "{% if kayobe_environment == 'ci-builder' %}etc-hosts{% endif %}" - "vm" - "block-device-efi" - "cloud-init" @@ -45,6 +46,9 @@ stackhpc_overcloud_dib_env_vars: # sometimes). # DIB_DISTRIBUTION_MIRROR: "{{ stackhpc_repo_ubuntu_focal_url if os_distribution == 'ubuntu' else '' }}" DIB_DRACUT_ENABLED_MODULES_DEFAULT_CONFIG: "{{ stackhpc_overcloud_dib_dracut_enabled_modules_default_config }}" + # FIXME: Workaround for lack of internal DNS on SMS lab. + DIB_ETC_HOSTS_EXTRA: | + 10.205.3.187 pulp-server pulp-server.internal.sms-cloud DIB_RELEASE: "{{ overcloud_dib_os_release }}" DIB_SUDOERS_FILENAME: "no-fqdn" # Avoid DNS queries during sudo commands, since we might not always have working DNS. From dd040f8d3299b712b3e2d075e1cb96ca3ebc2423 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 15 May 2023 14:55:06 +0100 Subject: [PATCH 280/292] Revert "Add workflows to build and promote overcloud host images (#509)" This reverts commit f2f389fd399375bd4d0bbf57314b1e7b6de2d48d. This change was originally developed on xena however by the time it was in a state to merge into main there was no use for it outside yoga. The changes have been merged up to stackhpc/yoga where development will continue. --- .../workflows/overcloud-host-image-build.yml | 177 ------------------ .../overcloud-host-image-promote.yml | 121 ------------ .../ansible/pulp-host-image-promote.yml | 40 ---- etc/kayobe/ansible/pulp-host-image-upload.yml | 109 ----------- .../environments/ci-builder/overcloud-dib.yml | 20 -- 5 files changed, 467 deletions(-) delete mode 100644 .github/workflows/overcloud-host-image-build.yml delete mode 100644 .github/workflows/overcloud-host-image-promote.yml delete mode 100644 etc/kayobe/ansible/pulp-host-image-promote.yml delete mode 100644 etc/kayobe/ansible/pulp-host-image-upload.yml delete mode 100644 etc/kayobe/environments/ci-builder/overcloud-dib.yml diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml deleted file mode 100644 index cf9b62806..000000000 --- a/.github/workflows/overcloud-host-image-build.yml +++ /dev/null @@ -1,177 +0,0 @@ ---- -name: Build overcloud host images -on: - workflow_dispatch: - inputs: - centos: - description: Build CentOS Stream - type: boolean - default: true - rocky: - description: Build Rocky Linux - type: boolean - default: true - ubuntu: - description: Build Ubuntu - type: boolean - default: true - -env: - ANSIBLE_FORCE_COLOR: True -jobs: - overcloud-host-image-build: - name: Build overcloud host images - if: github.repository == 'stackhpc/stackhpc-kayobe-config' - runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder] - steps: - - uses: actions/checkout@v3 - with: - path: src/kayobe-config - - - name: Clone StackHPC Kayobe repository - uses: actions/checkout@v3 - with: - repository: stackhpc/kayobe - ref: refs/heads/stackhpc/xena - path: src/kayobe - - # FIXME: Failed in kolla-ansible : Ensure the latest version of pip is installed - - name: Install dependencies - run: | - sudo dnf -y install python3-virtualenv - - - name: Setup networking - run: | - if ! ip l show breth1 >/dev/null 2>&1; then - sudo ip l add breth1 type bridge - fi - sudo ip l set breth1 up - if ! ip a show breth1 | grep 192.168.33.3/24; then - sudo ip a add 192.168.33.3/24 dev breth1 - fi - if ! ip l show dummy1 >/dev/null 2>&1; then - sudo ip l add dummy1 type dummy - fi - sudo ip l set dummy1 up - sudo ip l set dummy1 master breth1 - - - name: Install Kayobe - run: | - mkdir -p venvs && - pushd venvs && - python3 -m venv kayobe && - source kayobe/bin/activate && - pip install -U pip && - pip install ../src/kayobe - - - name: Bootstrap the control host - run: | - source venvs/kayobe/bin/activate && - source src/kayobe-config/kayobe-env --environment ci-builder && - kayobe control host bootstrap - - - name: Configure the seed host - run: | - source venvs/kayobe/bin/activate && - source src/kayobe-config/kayobe-env --environment ci-builder && - kayobe seed host configure - env: - KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - - - name: Create bifrost_httpboot Docker volume - run: | - if [[ $(sudo docker volume ls -f Name=bifrost_httpboot -q | wc -l) = 0 ]]; then - sudo docker volume create bifrost_httpboot - fi - - - name: Clean any previous build artifact - run: | - rm -f /tmp/updated_images.txt - - - name: Build a CentOS Stream 8 overcloud host image - run: | - source venvs/kayobe/bin/activate && - source src/kayobe-config/kayobe-env --environment ci-builder && - kayobe overcloud host image build --force-rebuild \ - -e os_distribution=centos \ - -e stackhpc_overcloud_dib_name=overcloud-centos-stream-8 - env: - KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - if: inputs.centos - - - name: Upload CentOS Stream 8 overcloud host image artifact - run: | - source venvs/kayobe/bin/activate && - source src/kayobe-config/kayobe-env --environment ci-builder && - kayobe playbook run \ - src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \ - -e image_path='/opt/kayobe/images/overcloud-centos-stream-8' \ - -e os_distribution='centos' - env: - KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - if: inputs.centos - - - name: Build a Rocky Linux 8 overcloud host image - run: | - source venvs/kayobe/bin/activate && - source src/kayobe-config/kayobe-env --environment ci-builder && - kayobe overcloud host image build --force-rebuild \ - -e os_distribution=rocky \ - -e stackhpc_overcloud_dib_name=overcloud-rocky-linux-8 - env: - KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - if: inputs.rocky - - - name: Upload Rocky Linux 8 overcloud host image artifact - run: | - source venvs/kayobe/bin/activate && - source src/kayobe-config/kayobe-env --environment ci-builder && - kayobe playbook run \ - src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \ - -e image_path='/opt/kayobe/images/overcloud-rocky-linux-8' \ - -e os_distribution='rocky' - env: - KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - if: inputs.rocky - - # FIXME: Need EPEL on CentOS for debootstrap. It is disabled by default. - # Do this via config? - - name: Enable EPEL repository - run: | - sudo dnf config-manager --set-enabled epel - - - name: Build a Ubuntu Focal 20.04 overcloud host image - run: | - source venvs/kayobe/bin/activate && - source src/kayobe-config/kayobe-env --environment ci-builder && - kayobe overcloud host image build --force-rebuild \ - -e os_distribution=ubuntu \ - -e stackhpc_overcloud_dib_name=overcloud-ubuntu-focal - env: - KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - if: inputs.ubuntu - - - name: Upload Ubuntu Focal 20.04 overcloud host image artifact - run: | - source venvs/kayobe/bin/activate && - source src/kayobe-config/kayobe-env --environment ci-builder && - kayobe playbook run \ - src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \ - -e image_path='/opt/kayobe/images/overcloud-ubuntu-focal' \ - -e os_distribution='ubuntu' - env: - KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - if: inputs.ubuntu - - - name: Upload updated images artifact - uses: actions/upload-artifact@v3 - with: - name: Updated images list - path: /tmp/updated_images.txt - retention-days: 7 - if: always() - - - name: Clean up old images - run: | - sudo rm -rf /opt/kayobe/images/ - if: always() diff --git a/.github/workflows/overcloud-host-image-promote.yml b/.github/workflows/overcloud-host-image-promote.yml deleted file mode 100644 index cacbd575e..000000000 --- a/.github/workflows/overcloud-host-image-promote.yml +++ /dev/null @@ -1,121 +0,0 @@ ---- -name: Promote overcloud host image -on: - workflow_dispatch: - inputs: - os_image: - description: Image to promote - type: choice - required: true - default: 'CentOS Stream 8' - options: - - 'CentOS Stream 8' - - 'Rocky Linux 8' - - 'Ubuntu Focal 20.04' - image_tag: - description: Tag to promote - type: string - required: true -env: - ANSIBLE_FORCE_COLOR: True -jobs: - overcloud-host-image-promote: - name: Promote overcloud host image - if: github.repository == 'stackhpc/stackhpc-kayobe-config' - runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder] - steps: - - uses: actions/checkout@v3 - with: - path: src/kayobe-config - - - name: Clone StackHPC Kayobe repository - uses: actions/checkout@v3 - with: - repository: stackhpc/kayobe - ref: refs/heads/stackhpc/xena - path: src/kayobe - - # FIXME: Failed in kolla-ansible : Ensure the latest version of pip is installed - - name: Install dependencies - run: | - sudo dnf -y install python3-virtualenv - - - name: Setup networking - run: | - if ! ip l show breth1 >/dev/null 2>&1; then - sudo ip l add breth1 type bridge - fi - sudo ip l set breth1 up - if ! ip a show breth1 | grep 192.168.33.3/24; then - sudo ip a add 192.168.33.3/24 dev breth1 - fi - if ! ip l show dummy1 >/dev/null 2>&1; then - sudo ip l add dummy1 type dummy - fi - sudo ip l set dummy1 up - sudo ip l set dummy1 master breth1 - - - name: Install Kayobe - run: | - mkdir -p venvs && - pushd venvs && - python3 -m venv kayobe && - source kayobe/bin/activate && - pip install -U pip && - pip install ../src/kayobe - - - name: Bootstrap the control host - run: | - source venvs/kayobe/bin/activate && - source src/kayobe-config/kayobe-env --environment ci-builder && - kayobe control host bootstrap - - - name: Configure the seed host - run: | - source venvs/kayobe/bin/activate && - source src/kayobe-config/kayobe-env --environment ci-builder && - kayobe seed host configure - env: - KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - - - name: Promote CentOS Stream 8 overcloud host image artifact - run: | - source venvs/kayobe/bin/activate && - source src/kayobe-config/kayobe-env --environment ci-builder && - kayobe playbook run \ - src/kayobe-config/etc/kayobe/ansible/pulp-host-image-promote.yml \ - -e image_path='/opt/kayobe/images/overcloud-centos-stream-8' \ - -e os_distribution='centos' \ - -e os_release='8-stream' - env: - OVERCLOUD_HOST_IMAGE_TAG: ${{ inputs.image_tag }} - KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - if: os_image == 'CentOS Stream 8' - - - name: Promote Rocky Linux 8 overcloud host image artifact - run: | - source venvs/kayobe/bin/activate && - source src/kayobe-config/kayobe-env --environment ci-builder && - kayobe playbook run \ - src/kayobe-config/etc/kayobe/ansible/pulp-host-image-promote.yml \ - -e image_path='/opt/kayobe/images/overcloud-rocky-linux-8' \ - -e os_distribution='rocky' \ - -e os_release='8' - env: - OVERCLOUD_HOST_IMAGE_TAG: ${{ inputs.image_tag }} - KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - if: os_image == 'Rocky Linux 8' - - - name: Promote Ubuntu Focal 20.04 overcloud host image artifact - run: | - source venvs/kayobe/bin/activate && - source src/kayobe-config/kayobe-env --environment ci-builder && - kayobe playbook run \ - src/kayobe-config/etc/kayobe/ansible/pulp-host-image-promote.yml \ - -e image_path='/opt/kayobe/images/overcloud-ubuntu-focal' \ - -e os_distribution='ubuntu' \ - -e os_release='focal' - env: - OVERCLOUD_HOST_IMAGE_TAG: ${{ inputs.image_tag }} - KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - if: os_image == 'Ubuntu Focal 20.04' diff --git a/etc/kayobe/ansible/pulp-host-image-promote.yml b/etc/kayobe/ansible/pulp-host-image-promote.yml deleted file mode 100644 index d93d71d51..000000000 --- a/etc/kayobe/ansible/pulp-host-image-promote.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -# Tag is the version of the image to promote, stored in an env variable named OVERCLOUD_HOST_IMAGE_TAG -- name: Promote an image to production - hosts: localhost - vars: - remote_pulp_url: "{{ stackhpc_release_pulp_url }}" - remote_pulp_username: "{{ stackhpc_image_repository_username }}" - remote_pulp_password: "{{ stackhpc_image_repository_password }}" - repository_name: "kayobe-images-{{ openstack_release }}-{{ os_distribution }}-{{ os_release }}" - base_path: "kayobe-images/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}" - promotion_tag: "{{ lookup('env', 'OVERCLOUD_HOST_IMAGE_TAG') }}" - tasks: - - - name: Check whether the image exists - pulp.squeezer.file_distribution: - pulp_url: "{{ remote_pulp_url }}" - username: "{{ remote_pulp_username }}" - password: "{{ remote_pulp_password }}" - name: "{{ repository_name }}_{{ promotion_tag }}" - base_path: "{{ base_path }}/{{ promotion_tag }}" - register: distribution_details - - - name: Fail if the image does not exist - fail: - msg: "Image {{ promotion_tag }} does not exist" - when: distribution_details.distribution is none - - - name: Ensure production content guard is set - pulp.squeezer.file_distribution: - pulp_url: "{{ remote_pulp_url }}" - username: "{{ remote_pulp_username }}" - password: "{{ remote_pulp_password }}" - name: "{{ repository_name }}_{{ promotion_tag }}" - base_path: "{{ base_path }}/{{ promotion_tag }}" - content_guard: release - state: present - - - name: Print version tag and os - debug: - msg: "Promoted tag: {{ promotion_tag }}" diff --git a/etc/kayobe/ansible/pulp-host-image-upload.yml b/etc/kayobe/ansible/pulp-host-image-upload.yml deleted file mode 100644 index b2945c83f..000000000 --- a/etc/kayobe/ansible/pulp-host-image-upload.yml +++ /dev/null @@ -1,109 +0,0 @@ ---- -- name: Upload and create a distribution for an image - hosts: localhost - vars: - remote_pulp_url: "{{ stackhpc_release_pulp_url }}" - remote_pulp_username: "{{ stackhpc_image_repository_username }}" - remote_pulp_password: "{{ stackhpc_image_repository_password }}" - repository_name: "kayobe-images-{{ openstack_release }}-{{ os_distribution }}-{{ os_release }}" - base_path: "kayobe-images/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}" - tasks: - - name: Get filename - find: - paths: "{{ image_path }}" - patterns: '*.qcow2' - register: found_files - - - name: Upload an artifact - pulp.squeezer.artifact: - pulp_url: "{{ remote_pulp_url }}" - username: "{{ remote_pulp_username }}" - password: "{{ remote_pulp_password }}" - file: "{{ found_files.files[0].path }}" - state: present - - - name: Get sha256 hash - ansible.builtin.stat: - path: "{{ found_files.files[0].path }}" - checksum_algorithm: sha256 - register: file_stats - - - name: Create file content from artifact - pulp.squeezer.file_content: - pulp_url: "{{ remote_pulp_url }}" - username: "{{ remote_pulp_username }}" - password: "{{ remote_pulp_password }}" - sha256: "{{ file_stats.stat.checksum }}" - relative_path: "{{ found_files.files[0].path | basename }}" - state: present - - - name: Ensure file repo exists - pulp.squeezer.file_repository: - pulp_url: "{{ remote_pulp_url }}" - username: "{{ remote_pulp_username }}" - password: "{{ remote_pulp_password }}" - name: "{{ repository_name }}" - state: present - - - name: Add content to file repo - pulp.squeezer.file_repository_content: - pulp_url: "{{ remote_pulp_url }}" - username: "{{ remote_pulp_username }}" - password: "{{ remote_pulp_password }}" - repository: "{{ repository_name }}" - present_content: - - relative_path: "{{ found_files.files[0].path | basename }}" - sha256: "{{ file_stats.stat.checksum }}" - - - name: Create a new publication to point to this version - pulp.squeezer.file_publication: - pulp_url: "{{ remote_pulp_url }}" - username: "{{ remote_pulp_username }}" - password: "{{ remote_pulp_password }}" - repository: "{{ repository_name }}" - state: present - register: publication_details - - - name: Update distribution for latest version - pulp.squeezer.file_distribution: - pulp_url: "{{ remote_pulp_url }}" - username: "{{ remote_pulp_username }}" - password: "{{ remote_pulp_password }}" - name: "{{ repository_name }}_latest" - base_path: "{{ base_path }}/latest" - publication: "{{ publication_details.publication.pulp_href }}" - content_guard: development - state: present - register: latest_distribution_details - - - name: Create distribution for given version - pulp.squeezer.file_distribution: - pulp_url: "{{ remote_pulp_url }}" - username: "{{ remote_pulp_username }}" - password: "{{ remote_pulp_password }}" - name: "{{ repository_name }}_{{ ansible_facts.date_time.iso8601_basic_short }}" - base_path: "{{ base_path }}/{{ ansible_facts.date_time.iso8601_basic_short }}" - publication: "{{ publication_details.publication.pulp_href }}" - content_guard: development - state: present - when: latest_distribution_details.changed - - - name: Update new images file with versioned path - lineinfile: - path: /tmp/updated_images.txt - line: "{{ remote_pulp_url }}/pulp/content/{{ base_path }}/\ - {{ ansible_facts.date_time.iso8601_basic_short }}/{{ found_files.files[0].path | basename }}" - create: true - when: latest_distribution_details.changed - - - name: Update new images file with latest path - lineinfile: - path: /tmp/updated_images.txt - line: "{{ remote_pulp_url }}/pulp/content/{{ base_path }}/\ - latest/{{ found_files.files[0].path | basename }}" - when: latest_distribution_details.changed - - - name: Print version tag - debug: - msg: "New tag: {{ ansible_facts.date_time.iso8601_basic_short }}" - when: latest_distribution_details.changed diff --git a/etc/kayobe/environments/ci-builder/overcloud-dib.yml b/etc/kayobe/environments/ci-builder/overcloud-dib.yml deleted file mode 100644 index 6a8f3b0f1..000000000 --- a/etc/kayobe/environments/ci-builder/overcloud-dib.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -# Overcloud host disk image configuration. - -############################################################################### -# Diskimage-builder configuration for overcloud host disk images. - -# Whether to build host disk images with DIB directly instead of through -# Bifrost. Setting it to true disables Bifrost image build and allows images to -# be built with the `kayobe overcloud host image build` command. Default value -# is False. This will change in a future release. -overcloud_dib_build_host_images: true - -# List of overcloud host disk images to build. Each element is a dict defining -# an image in a format accepted by the stackhpc.os-images role. Default is to -# build an image named "deployment_image" configured with the overcloud_dib_* -# variables defined below: {"name": "deployment_image", "elements": "{{ -# overcloud_dib_elements }}", "env": "{{ overcloud_dib_env_vars }}", -# "packages": "{{ overcloud_dib_packages }}"}. -overcloud_dib_host_images: - - "{{ stackhpc_overcloud_dib_host_image }}" From 37b3e0bd7d70427548649c76be0d1464db252f76 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Tue, 16 May 2023 11:04:42 +0200 Subject: [PATCH 281/292] Fix Pulp container deployment with HTTPS enabled The Pulp project has stopped maintaining separate https images and there is no pulp/pulp image tagged as 3.23-https. The PULP_HTTPS variable should be set to ``true`` instead [1]. [1] https://github.com/pulp/pulp-oci-images/pull/426 --- etc/kayobe/seed.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index e900a587b..a5f477dc8 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -106,7 +106,7 @@ seed_pulp_container: image: pulp/pulp pre: "{{ kayobe_config_path }}/containers/pulp/pre.yml" post: "{{ kayobe_config_path }}/containers/pulp/post.yml" - tag: "{{ '3.23-https' if pulp_enable_tls | bool else '3.23' }}" + tag: "3.23" network_mode: host # Override deploy_containers_defaults.init == true to ensure # s6-overlay-suexec starts as pid 1 @@ -114,6 +114,7 @@ seed_pulp_container: env: PULP_CONTENT_WORKERS: "{{ ansible_facts.processor_vcpus * 2 + 1 }}" PULP_API_WORKERS: "{{ ansible_facts.processor_vcpus * 2 + 1 }}" + PULP_HTTPS: "{{ 'true' if pulp_enable_tls | bool else 'false' }}" volumes: - /opt/kayobe/containers/pulp:/etc/pulp - pulp_storage:/var/lib/pulp From cbed0329b26f9d0faa096d1ef99b650ffd502c48 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Tue, 16 May 2023 12:43:31 +0200 Subject: [PATCH 282/292] Add missing proxy_url to Pulp repositories This was causing pulp-container-sync to hang on synchronising ceph/ceph in an environment behind an HTTP proxy. --- etc/kayobe/pulp.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index b20000e30..c6be5ac8a 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -257,6 +257,7 @@ stackhpc_pulp_repository_rpm_repos: client_cert: "" client_key: "" policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_complete state: present required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" @@ -267,6 +268,7 @@ stackhpc_pulp_repository_rpm_repos: client_cert: "" client_key: "" policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_complete state: present required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" @@ -277,6 +279,7 @@ stackhpc_pulp_repository_rpm_repos: client_cert: "" client_key: "" policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_complete state: present required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" @@ -287,6 +290,7 @@ stackhpc_pulp_repository_rpm_repos: client_cert: "" client_key: "" policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_complete state: present required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" @@ -297,6 +301,7 @@ stackhpc_pulp_repository_rpm_repos: client_cert: "" client_key: "" policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_complete state: present required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" @@ -346,6 +351,7 @@ stackhpc_pulp_repository_rpm_repos: client_cert: "" client_key: "" policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_complete state: present required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}" @@ -357,6 +363,7 @@ stackhpc_pulp_repository_rpm_repos: client_cert: "" client_key: "" policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_complete state: present required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}" @@ -368,6 +375,7 @@ stackhpc_pulp_repository_rpm_repos: client_cert: "" client_key: "" policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_complete state: present required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}" @@ -379,6 +387,7 @@ stackhpc_pulp_repository_rpm_repos: client_cert: "" client_key: "" policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_complete state: present required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}" @@ -390,6 +399,7 @@ stackhpc_pulp_repository_rpm_repos: client_cert: "" client_key: "" policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_complete state: present required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}" @@ -401,6 +411,7 @@ stackhpc_pulp_repository_rpm_repos: client_cert: "" client_key: "" policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_complete state: present required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}" @@ -412,6 +423,7 @@ stackhpc_pulp_repository_rpm_repos: client_cert: "" client_key: "" policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_complete state: present required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}" @@ -423,6 +435,7 @@ stackhpc_pulp_repository_rpm_repos: client_cert: "" client_key: "" policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_complete state: present required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}" @@ -434,6 +447,7 @@ stackhpc_pulp_repository_rpm_repos: client_cert: "" client_key: "" policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_complete state: present required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}" @@ -445,6 +459,7 @@ stackhpc_pulp_repository_rpm_repos: client_cert: "" client_key: "" policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_complete state: present required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}" @@ -456,6 +471,7 @@ stackhpc_pulp_repository_rpm_repos: client_cert: "" client_key: "" policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_complete state: present required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}" @@ -870,6 +886,7 @@ stackhpc_pulp_repository_container_repos_ceph: - name: "ceph/ceph" url: "https://quay.io" policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" state: present include_tags: "{{ cephadm_image_tag }}" required: "{{ stackhpc_sync_ceph_images | bool }}" From 094ef75b377c551b87f51963e14389309d831120 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Tue, 16 May 2023 16:43:31 +0200 Subject: [PATCH 283/292] Update Pulp to 3.24 --- etc/kayobe/seed.yml | 2 +- releasenotes/notes/update-pulp-3.24-d770db48deaf6076.yaml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/update-pulp-3.24-d770db48deaf6076.yaml diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index a5f477dc8..a13e4b0ac 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -106,7 +106,7 @@ seed_pulp_container: image: pulp/pulp pre: "{{ kayobe_config_path }}/containers/pulp/pre.yml" post: "{{ kayobe_config_path }}/containers/pulp/post.yml" - tag: "3.23" + tag: "3.24" network_mode: host # Override deploy_containers_defaults.init == true to ensure # s6-overlay-suexec starts as pid 1 diff --git a/releasenotes/notes/update-pulp-3.24-d770db48deaf6076.yaml b/releasenotes/notes/update-pulp-3.24-d770db48deaf6076.yaml new file mode 100644 index 000000000..32138adb6 --- /dev/null +++ b/releasenotes/notes/update-pulp-3.24-d770db48deaf6076.yaml @@ -0,0 +1,3 @@ +--- +features: + - Upgrades Pulp from ``3.23`` to ``3.24``. From eec982da674681126aa3b03950346af967202801 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Tue, 6 Jun 2023 10:07:35 +0100 Subject: [PATCH 284/292] Correct Manila backend variable in multinode docs --- doc/source/contributor/environments/ci-multinode.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/contributor/environments/ci-multinode.rst b/doc/source/contributor/environments/ci-multinode.rst index 4bef5fd39..feaeca400 100644 --- a/doc/source/contributor/environments/ci-multinode.rst +++ b/doc/source/contributor/environments/ci-multinode.rst @@ -64,7 +64,7 @@ is not enabled by default. To enable it, set the following in .. code-block:: yaml kolla_enable_manila: true - kolla_enable_manila_backend_cephfs: true + kolla_enable_manila_backend_cephfs_native: true And re-run ``kayobe overcloud service deploy`` if you are working on an existing deployment. From 4d2635d34540c076b86dc1a91bbb9d14e836ccb6 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Mon, 12 Jun 2023 18:11:36 +0200 Subject: [PATCH 285/292] Cap number of Pulp workers Running Pulp with a number of workers based on CPUs can cause errors such as PostgreSQL connection limits on hosts with many cores: psycopg.OperationalError: connection failed: FATAL: remaining connection slots are reserved for non-replication superuser connections Cap workers to 32 until large scale testing can identify a more suitable limit. --- etc/kayobe/seed.yml | 4 ++-- releasenotes/notes/cap-pulp-workers-e0f12c0b67d3d0bf.yaml | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/cap-pulp-workers-e0f12c0b67d3d0bf.yaml diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index a13e4b0ac..50d3d0942 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -112,8 +112,8 @@ seed_pulp_container: # s6-overlay-suexec starts as pid 1 init: false env: - PULP_CONTENT_WORKERS: "{{ ansible_facts.processor_vcpus * 2 + 1 }}" - PULP_API_WORKERS: "{{ ansible_facts.processor_vcpus * 2 + 1 }}" + PULP_CONTENT_WORKERS: "{{ [ansible_facts.processor_vcpus * 2 + 1, 32] | min }}" + PULP_API_WORKERS: "{{ [ansible_facts.processor_vcpus * 2 + 1, 32] | min }}" PULP_HTTPS: "{{ 'true' if pulp_enable_tls | bool else 'false' }}" volumes: - /opt/kayobe/containers/pulp:/etc/pulp diff --git a/releasenotes/notes/cap-pulp-workers-e0f12c0b67d3d0bf.yaml b/releasenotes/notes/cap-pulp-workers-e0f12c0b67d3d0bf.yaml new file mode 100644 index 000000000..b1ca452fa --- /dev/null +++ b/releasenotes/notes/cap-pulp-workers-e0f12c0b67d3d0bf.yaml @@ -0,0 +1,4 @@ +fixes: + - | + Caps the number of Pulp API and content workers to 32 each to avoid errors + on hosts with many CPUs. From 10d5d7cdc19c3ce064b4af3bd089120cc85c3d8e Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Wed, 16 Aug 2023 12:28:07 +0200 Subject: [PATCH 286/292] Fix documentation builds on Read the Docs --- .readthedocs.yaml | 23 +++++++++++++++++++ ...hedocs-configuration-17cf3e6a63f571fa.yaml | 4 ++++ 2 files changed, 27 insertions(+) create mode 100644 .readthedocs.yaml create mode 100644 releasenotes/notes/readthedocs-configuration-17cf3e6a63f571fa.yaml diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..4a1b3efc0 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,23 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + apt_packages: + - tox + os: ubuntu-22.04 + tools: + python: "3.11" + +# Build documentation in the doc/source/ directory with Sphinx +sphinx: + configuration: doc/source/conf.py + +# We recommend specifying your dependencies to enable reproducible builds: +# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: doc/requirements.txt diff --git a/releasenotes/notes/readthedocs-configuration-17cf3e6a63f571fa.yaml b/releasenotes/notes/readthedocs-configuration-17cf3e6a63f571fa.yaml new file mode 100644 index 000000000..2445b5cbc --- /dev/null +++ b/releasenotes/notes/readthedocs-configuration-17cf3e6a63f571fa.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Fixes documentation builds on Read the Docs. From 1b91a4de82d861473bdd3367de83ceeeee1e4bc0 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Wed, 16 Aug 2023 15:34:08 +0200 Subject: [PATCH 287/292] Attempt to fix RTD failures using Python 3.7 The default Python version has changed from 3.7 to 3.11 when updating Read the Docs configuration. See the announcement of configuration file version 2 for more details [1]. [1] https://blog.readthedocs.com/migrate-configuration-v2/ --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 4a1b3efc0..0448e4a40 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -10,7 +10,7 @@ build: - tox os: ubuntu-22.04 tools: - python: "3.11" + python: "3.7" # Build documentation in the doc/source/ directory with Sphinx sphinx: From a2b4eb0e522b53db6f13fee5f5fa9b13bb65b066 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Thu, 17 Aug 2023 07:57:52 +0200 Subject: [PATCH 288/292] Unshallow Read the Docs git clone Read the Docs does not perform a full clone on checkout job to reduce network data and speed up the build process. Because of this, extensions that depend on the full Git history will fail [1]. This breaks build of release notes, since reno needs Git history access. [1] https://docs.readthedocs.io/en/stable/build-customization.html#unshallow-git-clone --- .readthedocs.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 0448e4a40..3fcf3b70e 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -11,6 +11,9 @@ build: os: ubuntu-22.04 tools: python: "3.7" + jobs: + post_checkout: + - git fetch --unshallow || true # Build documentation in the doc/source/ directory with Sphinx sphinx: From 274f0b0a38dad00c1c843b6d35adef510afcd318 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Thu, 17 Aug 2023 08:17:55 +0200 Subject: [PATCH 289/292] Restore default Git fetch configuration on RTD The Git shallow clone used on Read the Docs is configured to only fetch a specific Git branch: [remote "origin"] url = https://github.com/stackhpc/stackhpc-kayobe-config.git fetch = +refs/heads/stackhpc/yoga:refs/remotes/origin/stackhpc/yoga This prevents subsequent `git fetch` commands from fetching all remote branches. Use `git remote set-branches origin '*'` to revert to the default Git fetch configuration. --- .readthedocs.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 3fcf3b70e..6038107f5 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -13,6 +13,7 @@ build: python: "3.7" jobs: post_checkout: + - git remote set-branches origin '*' || true - git fetch --unshallow || true # Build documentation in the doc/source/ directory with Sphinx From 2e2886f3c45f4fd5fb0f327693e830ba585ae389 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Thu, 17 Aug 2023 08:30:37 +0200 Subject: [PATCH 290/292] Escape wildcard character in custom build command The wildcard glob matched files in the current directory which resulted in `git fetch` failing with: fatal: couldn't find remote ref refs/heads/LICENSE --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 6038107f5..3c6286c52 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -13,7 +13,7 @@ build: python: "3.7" jobs: post_checkout: - - git remote set-branches origin '*' || true + - git remote set-branches origin '\*' || true - git fetch --unshallow || true # Build documentation in the doc/source/ directory with Sphinx From 607817f13036535ff5ddda2eb01f9fbc0b808e14 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Thu, 17 Aug 2023 09:15:01 +0200 Subject: [PATCH 291/292] Work around Read the Docs wildcard escaping issues Use an explicit list of remote branches with release notes instead of a wildcard which is behaving differently on Read the Docs than locally. Also remove true fallback to fail early. --- .readthedocs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 3c6286c52..e77d7d9cb 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -13,8 +13,8 @@ build: python: "3.7" jobs: post_checkout: - - git remote set-branches origin '\*' || true - - git fetch --unshallow || true + - git remote set-branches origin master stackhpc/yoga stackhpc/xena stackhpc/wallaby + - git fetch --unshallow # Build documentation in the doc/source/ directory with Sphinx sphinx: From 0b545d6cf821c99d3a0689aea723073454e47c32 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 10 Jul 2023 17:19:48 +0100 Subject: [PATCH 292/292] Pin pulp version to 3.24.0 The 3.24 tag was recently updated to 3.24.1 which includes a version of pulp_container that causes client pulp container syncs to fail. --- etc/kayobe/seed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index 50d3d0942..96fa86ac0 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -106,7 +106,7 @@ seed_pulp_container: image: pulp/pulp pre: "{{ kayobe_config_path }}/containers/pulp/pre.yml" post: "{{ kayobe_config_path }}/containers/pulp/post.yml" - tag: "3.24" + tag: "3.24.0" network_mode: host # Override deploy_containers_defaults.init == true to ensure # s6-overlay-suexec starts as pid 1