From afd842ab182f0018c4be72cbd0fe2f6058290431 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Mon, 13 Feb 2023 10:44:24 +0000 Subject: [PATCH 01/16] Update releasenotes/notes/release-train-02-2023-package-refresh-9de8049f8cb08a5e.yaml Co-authored-by: Mark Goddard --- ...elease-train-02-2023-package-refresh-9de8049f8cb08a5e.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/releasenotes/notes/release-train-02-2023-package-refresh-9de8049f8cb08a5e.yaml b/releasenotes/notes/release-train-02-2023-package-refresh-9de8049f8cb08a5e.yaml index 058cb3945..72171afcb 100644 --- a/releasenotes/notes/release-train-02-2023-package-refresh-9de8049f8cb08a5e.yaml +++ b/releasenotes/notes/release-train-02-2023-package-refresh-9de8049f8cb08a5e.yaml @@ -1,7 +1,9 @@ --- features: - | - The package repositories have been refreshed. Key packages to note are: + Package repositories and container images for CentOS Stream based deployments have been updated. + Key packages to note are: + * Kernel * version: 4.18.0 * release: 408.el8 From 6344d42c214d0068aaa94a5cbba1c43cc5f835dc Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 14 Feb 2023 12:50:25 +0000 Subject: [PATCH 02/16] fix: task would fail to delegate and run on host --- etc/kayobe/ansible/nova-compute-drain.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etc/kayobe/ansible/nova-compute-drain.yml b/etc/kayobe/ansible/nova-compute-drain.yml index 2296a0762..a29ac3169 100644 --- a/etc/kayobe/ansible/nova-compute-drain.yml +++ b/etc/kayobe/ansible/nova-compute-drain.yml @@ -17,6 +17,8 @@ extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" run_once: true delegate_to: "{{ groups['controllers'][0] }}" + vars: + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" - block: - name: Query instances From c3676e00205bdd4f35766c59394d11c695c52a94 Mon Sep 17 00:00:00 2001 From: Dawud M <7688823+technowhizz@users.noreply.github.com> Date: Fri, 23 Dec 2022 10:14:38 +0000 Subject: [PATCH 03/16] Fix missing group in smart alerting rules (cherry picked from commit 02301f4f8420a33c0cf7e25c2a5ad3ee4c488988) --- etc/kayobe/kolla/config/prometheus/smart.rules | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/kolla/config/prometheus/smart.rules b/etc/kayobe/kolla/config/prometheus/smart.rules index 0b6552598..aea36bdf8 100644 --- a/etc/kayobe/kolla/config/prometheus/smart.rules +++ b/etc/kayobe/kolla/config/prometheus/smart.rules @@ -1,6 +1,10 @@ {% raw %} -- alert: DiskSmartStatusUnhealthy +groups: +- name: Smart + rules: + + - alert: DiskSmartStatusUnhealthy expr: smartmon_device_smart_healthy < 1 for: 10m labels: @@ -9,4 +13,4 @@ summary: "SMART monitor reports bad disk on (instance {{ $labels.instance }})" description: "{{ $labels.instance }} is reporting unhealthy for the disk at {{ $labels.disk }}. Disk serial number is: {{ $labels.serial_number }}" -{% endraw %} +{% endraw %} \ No newline at end of file From 9ccd872732f81f22cfd789a872a6143ff4bdb2b7 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 23 Jan 2023 16:57:10 +0000 Subject: [PATCH 04/16] alertmanager example and monitoring docs update (cherry picked from commit 452f6af91c47acd679c1fcc67d40089e28c3b031) --- doc/source/configuration/monitoring.rst | 55 ++++++++++++++++++- .../prometheus-alertmanager.yml.example | 47 ++++++++++++++++ 2 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.yml.example diff --git a/doc/source/configuration/monitoring.rst b/doc/source/configuration/monitoring.rst index 6045f3c29..c2384ba93 100644 --- a/doc/source/configuration/monitoring.rst +++ b/doc/source/configuration/monitoring.rst @@ -7,9 +7,20 @@ Monitoring Configuration StackHPC kayobe config includes a reference monitoring and alerting stack based on Prometheus, Alertmanager, Grafana, Fluentd, Elasticsearch & Kibana. These -services by default come enabled and configured. Central Elasticsearch cluster -collects OpenStack logs, with an option to receive operating system logs too. -In order to enable this, execute custom playbook after deployment: +services by default come enabled and configured. + +Monitoring hosts, usually the controllers, should be added to the monitoring +group. The group definition can be applied in various different places. For +example, this configuration could be added to etc/kayobe/inventory/groups: + +.. code-block:: yaml + + [monitoring:children] + controllers + +Central Elasticsearch cluster collects OpenStack logs, with an option to receive +operating system logs too. In order to enable this, execute custom playbook +after deployment: .. code-block:: console @@ -78,3 +89,41 @@ on the overcloud hosts: SMART reporting should now be enabled along with a Prometheus alert for unhealthy disks and a Grafana dashboard called ``Hardware Overview``. + +Alertmanager and Slack +====================== + +StackHPC Kayobe configuration comes bundled with an array of alerts but does not +enable any receivers for notifications by default. Various receivers can be +configured for Alertmanager. Slack is currently the most common. + +To set up a receiver, create a ``prometheus-alertmanager.yml`` file under +``etc/kayobe/kolla/config/prometheus/``. An example config is stored in this +directory. The example configuration uses two Slack channels. One channel +receives all alerts while the other only receives alerts tagged as critical. It +also adds a silence button to temporarily mute alerts. To use the example in a +deployment, you will need to generate two webhook URLs, one for each channel. + +To generate a slack webhook, `create a new app +`__ in the workspace you want to add alerts to. +From the Features page, toggle Activate incoming webhooks on. Click Add new +webhook to workspace. Pick a channel that the app will post to, then click +Authorise. You only need one app to generate both webhooks. + +Both URLs should be encrypted using ansible vault, as they give anyone access to +your slack channels. The standard practice is to store them in +``kayobe/secrets.yml`` as: + +.. code-block:: yaml + + secrets_slack_notification_channel_url: + secrets_slack_critical_notification_channel_url: + +These should then be set as the ``slack_api_url`` and ``api_url`` for the +regular and critical alerts channels respectively. Both slack channel names will +need to be set, and the proxy URL sould be set or removed. + +If you want to add an alerting rule, there are many good examples of alerts are +available `here `__. They simply +need to be added to one of the ``*.rules`` files in the prometheus configuration +directory. \ No newline at end of file diff --git a/etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.yml.example b/etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.yml.example new file mode 100644 index 000000000..435d63c09 --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.yml.example @@ -0,0 +1,47 @@ +--- +global: + resolve_timeout: 5m + smtp_require_tls: true + slack_api_url: 'https://hooks.slack.com/services/example/alerts/webhook' + +route: + receiver: 'slack-notifications' + group_by: [alertname] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h + + routes: + - matchers: + - severity=~"critical|alert" + receiver: 'slack-critical-notifications' + + +receivers: + - name: 'slack-notifications' + slack_configs: + - channel: '#notifications' + actions: + - type: button + text: 'Silence 🔕' + url: {{ '{% raw %}' }}{% raw %} '{{ template "__alert_silence_link" . }}' +{% endraw %}{{ '{% endraw %}' }} + send_resolved: true + http_config: + proxy_url: http://1.2.3.4:3128 + - name: 'slack-critical-notifications' + slack_configs: + - channel: '#notifications-critical' + actions: + - type: button + text: 'Silence 🔕' + url: {{ '{% raw %}' }}{% raw %} '{{ template "__alert_silence_link" . }}' +{% endraw %}{{ '{% endraw %}' }} + send_resolved: true + http_config: + proxy_url: http://1.2.3.4:3128 + api_url: 'https://hooks.slack.com/services/example/alerts/webhook-critical' + + +templates: + - '/etc/prometheus/*.tmpl' \ No newline at end of file From 2bf04bf81a5953239e51558c8c6a3f2bf1bea4c7 Mon Sep 17 00:00:00 2001 From: Dawud M <7688823+technowhizz@users.noreply.github.com> Date: Fri, 30 Dec 2022 23:11:55 +0000 Subject: [PATCH 05/16] Add drive temps to hardware dashboard This adds a column for the drive temperatures to the table on the hardware overview dashboard and changes the column ordering for better readability. (cherry picked from commit 7f48ed25c1fdd35582e7822f40861a51301ee399) --- .../openstack/hardware_overview.json | 156 ++++++++++++++---- 1 file changed, 122 insertions(+), 34 deletions(-) diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json index e4f78aee3..0c236dce4 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json @@ -25,6 +25,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, + "id": 1935501, "links": [], "liveNow": false, "panels": [ @@ -251,23 +252,7 @@ "filterable": false, "inspect": false }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "Failed" - }, - "1": { - "color": "dark-green", - "index": 0, - "text": "Ok" - } - }, - "type": "value" - } - ], + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ @@ -306,7 +291,7 @@ { "matcher": { "id": "byName", - "options": "Value" + "options": "Value #Health" }, "properties": [ { @@ -315,13 +300,37 @@ }, { "id": "custom.width" + }, + { + "id": "displayName", + "value": "Health" + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Bad" + }, + "1": { + "color": "green", + "index": 0, + "text": "Ok" + } + }, + "type": "value" + } + ] } ] }, { "matcher": { "id": "byName", - "options": "type" + "options": "type 1" }, "properties": [ { @@ -353,7 +362,7 @@ { "matcher": { "id": "byName", - "options": "Time" + "options": "Time 1" }, "properties": [ { @@ -365,7 +374,7 @@ { "matcher": { "id": "byName", - "options": "disk" + "options": "disk 1" }, "properties": [ { @@ -381,7 +390,7 @@ { "matcher": { "id": "byName", - "options": "instance" + "options": "instance 1" }, "properties": [ { @@ -397,7 +406,7 @@ { "matcher": { "id": "byName", - "options": "Type" + "options": "Hostname" }, "properties": [ { @@ -408,33 +417,60 @@ { "matcher": { "id": "byName", - "options": "Serial Number" + "options": "group 1" }, "properties": [ { - "id": "custom.width" + "id": "displayName", + "value": "Group" } ] }, { "matcher": { "id": "byName", - "options": "Hostname" + "options": "smart_id" }, "properties": [ { - "id": "custom.width" + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".* 2" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true } ] }, { "matcher": { "id": "byName", - "options": "Disk" + "options": "Value #Temp" }, "properties": [ { - "id": "custom.width" + "id": "displayName", + "value": "Temperature" + }, + { + "id": "unit", + "value": "celsius" + }, + { + "id": "noValue", + "value": "-" + }, + { + "id": "custom.displayMode", + "value": "color-text" } ] } @@ -442,7 +478,7 @@ }, "gridPos": { "h": 13, - "w": 18, + "w": 20, "x": 0, "y": 7 }, @@ -455,6 +491,7 @@ ], "show": false }, + "frameIndex": 1, "showHeader": true, "sortBy": [] }, @@ -481,13 +518,64 @@ "interval": "", "legendFormat": "", "range": false, - "refId": "A", + "refId": "Health", "units": "none", "valueHandler": "Number Threshold", "warn": 0 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "smartmon_temperature_case_raw_value", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "range": false, + "refId": "Temp" + } + ], + "title": "SMART Info", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "serial_number" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Time 1": 3, + "Time 2": 10, + "Value #Health": 8, + "Value #Temp": 9, + "__name__ 1": 4, + "__name__ 2": 11, + "disk 1": 5, + "disk 2": 12, + "group 1": 0, + "group 2": 13, + "instance 1": 1, + "instance 2": 14, + "job 1": 6, + "job 2": 15, + "serial_number": 7, + "smart_id": 16, + "type 1": 2, + "type 2": 17 + }, + "renameByName": {} + } } ], - "title": "Panel Title", "transparent": true, "type": "table" } @@ -511,7 +599,7 @@ }, { "current": { - "selected": true, + "selected": false, "text": "Prometheus", "value": "Prometheus" }, @@ -537,7 +625,7 @@ "timezone": "", "title": "Hardware Overview", "uid": "TCN51Y25P", - "version": 1, + "version": 3, "weekStart": "" } {% endraw %} \ No newline at end of file From b66e05cec0e2ee12f8557c9eb48a72b15a9a3108 Mon Sep 17 00:00:00 2001 From: Dawud M <7688823+technowhizz@users.noreply.github.com> Date: Wed, 4 Jan 2023 11:21:46 +0000 Subject: [PATCH 06/16] Update the hardware overview dashboard Adds the temperature of drives as a column to the table. Adds a timeseries to track the history of the temperature. Introduces a variable dropdown picker for nodes so that only information about disks on a specific node is shown. Change-Id: Ibff3037865e1891d8e07a945dbeedf08e1f8c0d2 (cherry picked from commit 4999f9c910204090d05b3338c5c60d73cf07b2dc) --- .../openstack/hardware_overview.json | 142 ++++++++++++++++-- 1 file changed, 133 insertions(+), 9 deletions(-) diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json index 0c236dce4..63bbf4af3 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json @@ -84,7 +84,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "count(smartmon_device_smart_healthy > 0)", + "expr": "count(smartmon_device_smart_healthy{instance=~\"$node\"} > 0 )", "format": "time_series", "instant": false, "interval": "", @@ -157,7 +157,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "count(smartmon_device_smart_healthy < 1) ", + "expr": "count(smartmon_device_smart_healthy{instance=~\"$node\"} < 1) ", "format": "time_series", "instant": false, "interval": "", @@ -225,7 +225,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "count(smartmon_device_smart_healthy)", + "expr": "count(smartmon_device_smart_healthy{instance=~\"$node\"})", "format": "time_series", "instant": false, "interval": "", @@ -477,7 +477,7 @@ ] }, "gridPos": { - "h": 13, + "h": 10, "w": 20, "x": 0, "y": 7 @@ -512,7 +512,7 @@ "displayValueWithAlias": "Never", "editorMode": "code", "exemplar": false, - "expr": "smartmon_device_smart_healthy", + "expr": "smartmon_device_smart_healthy{instance=~\"$node\"}", "format": "table", "instant": true, "interval": "", @@ -530,7 +530,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "smartmon_temperature_case_raw_value", + "expr": "smartmon_temperature_case_raw_value{instance=~\"$node\"}", "format": "table", "hide": false, "instant": true, @@ -578,6 +578,102 @@ ], "transparent": true, "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Temperature (°C)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 20, + "x": 0, + "y": 17 + }, + "hideTimeOverride": false, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg_over_time(smartmon_temperature_case_raw_value{instance=~\"$node\"}[1h])", + "instant": false, + "interval": "", + "legendFormat": "{{instance}} - {{disk}} - {{serial_number}}", + "range": true, + "refId": "A" + } + ], + "title": "Disk Temperatures", + "type": "timeseries" } ], "refresh": false, @@ -599,7 +695,7 @@ }, { "current": { - "selected": false, + "selected": true, "text": "Prometheus", "value": "Prometheus" }, @@ -614,18 +710,46 @@ "regex": "", "skipUrlSync": false, "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(node_cpu_seconds_total{job=\"node\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "Host:", + "multi": false, + "name": "node", + "options": [], + "query": { + "query": "label_values(node_cpu_seconds_total{job=\"node\"}, instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" } ] }, "time": { - "from": "now-1h", + "from": "now-24h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Hardware Overview", "uid": "TCN51Y25P", - "version": 3, + "version": 5, "weekStart": "" } {% endraw %} \ No newline at end of file From efc90d241935836835b30dc3e89bf72035b4b803 Mon Sep 17 00:00:00 2001 From: Dawud M <7688823+technowhizz@users.noreply.github.com> Date: Fri, 23 Dec 2022 12:05:35 +0000 Subject: [PATCH 07/16] Add release note for smart rules bugfix (cherry picked from commit 5e237eb2c525558db67f6e22914bdd2c302b9a84) --- releasenotes/notes/bugfix-smart-rules-53ff5400e7f681c8.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 releasenotes/notes/bugfix-smart-rules-53ff5400e7f681c8.yaml diff --git a/releasenotes/notes/bugfix-smart-rules-53ff5400e7f681c8.yaml b/releasenotes/notes/bugfix-smart-rules-53ff5400e7f681c8.yaml new file mode 100644 index 000000000..8da7f80fc --- /dev/null +++ b/releasenotes/notes/bugfix-smart-rules-53ff5400e7f681c8.yaml @@ -0,0 +1,6 @@ +--- + +fixes: + - | + Fixed a syntax error in Prometheus SMART monitoring rules + From be5e64d2ea6b350dd4c1742d7491cde2b1e976fc Mon Sep 17 00:00:00 2001 From: Dawud M <7688823+technowhizz@users.noreply.github.com> Date: Wed, 4 Jan 2023 17:26:21 +0000 Subject: [PATCH 08/16] Add awesome alerts alert for NTP Change-Id: I80610eeaa5cd7f83b0753af97d53cedbe12c2c86 (cherry picked from commit c82762c5691e3cbc27a5d0f4f664cf30c2c51548) --- .../kolla/config/prometheus/system.rules | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/etc/kayobe/kolla/config/prometheus/system.rules b/etc/kayobe/kolla/config/prometheus/system.rules index 532710fd3..be1343b82 100644 --- a/etc/kayobe/kolla/config/prometheus/system.rules +++ b/etc/kayobe/kolla/config/prometheus/system.rules @@ -78,6 +78,24 @@ groups: summary: "Host EDAC Uncorrectable Errors detected (instance {{ $labels.instance }})" description: "{{ $labels.instance }} has had {{ printf \"%.0f\" $value }} uncorrectable memory errors reported by EDAC in the last 5 minutes." + - alert: HostClockSkew + expr: (node_timex_offset_seconds > 0.05 and deriv(node_timex_offset_seconds[5m]) >= 0) or (node_timex_offset_seconds < -0.05 and deriv(node_timex_offset_seconds[5m]) <= 0) + for: 2m + labels: + severity: warning + annotations: + summary: Host clock skew (instance {{ $labels.instance }}) + description: "Clock skew detected. Clock is out of sync. Ensure NTP is configured correctly on this host." + + - alert: HostClockNotSynchronising + expr: min_over_time(node_timex_sync_status[1m]) == 0 and node_timex_maxerror_seconds >= 16 + for: 2m + labels: + severity: warning + annotations: + summary: Host clock not synchronising (instance {{ $labels.instance }}) + description: "Clock not synchronising. Ensure NTP is configured on this host." + - alert: HostConntrackLimit expr: node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8 for: 5m From f6888bb8462c04719f36359747f81484d1dd3458 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Tue, 14 Feb 2023 15:31:16 +0000 Subject: [PATCH 09/16] Fix versions & releases in package refresh note --- ...ain-02-2023-package-refresh-9de8049f8cb08a5e.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/releasenotes/notes/release-train-02-2023-package-refresh-9de8049f8cb08a5e.yaml b/releasenotes/notes/release-train-02-2023-package-refresh-9de8049f8cb08a5e.yaml index 72171afcb..72a3318e0 100644 --- a/releasenotes/notes/release-train-02-2023-package-refresh-9de8049f8cb08a5e.yaml +++ b/releasenotes/notes/release-train-02-2023-package-refresh-9de8049f8cb08a5e.yaml @@ -6,13 +6,13 @@ features: * Kernel * version: 4.18.0 - * release: 408.el8 + * release: 448.el8 * Libvirt - * version: 7.6.0 - * release: 6.el8s + * version: 8.0.0 + * release: 6.module_el8.7.0+1140+ff0772f9 * OVS * version: 2.17.0 - * release: 31.el8s + * release: 71.el8s * OVN - * version: 22.06.0 - * release: 24.el8s + * version: 22.09.0 + * release: 11.el8s From 0aff381440b06a839d6418cff2766e0093c7998d Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 15 Feb 2023 16:29:21 +0000 Subject: [PATCH 10/16] Refresh ubuntu containers Periodic refresh to get updated packages. Build logs: https://github.com/stackhpc/stackhpc-kayobe-config/actions/runs/4184397411 --- etc/kayobe/kolla.yml | 2 +- etc/kayobe/kolla/globals.yml | 14 +------------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index aa701a84c..d049150d4 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -95,7 +95,7 @@ kolla_docker_registry_password: "{{ stackhpc_docker_registry_password }}" # Kolla OpenStack release version. This should be a Docker image tag. # Default is {{ openstack_release }}. -kolla_openstack_release: "{% if kolla_base_distro == 'centos' %}wallaby-20230207T161158{% else %}wallaby-20220819T112725{% endif %}" +kolla_openstack_release: "{% if kolla_base_distro == 'centos' %}wallaby-20230207T161158{% else %}wallaby-20230215T134157{% endif %}" # Docker tag applied to built container images. Default is # {{ kolla_openstack_release }}. diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 0b0e21e2c..02f46e14e 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -6,19 +6,7 @@ docker_yum_gpgkey: "https://download.docker.com/linux/centos/gpg" {% if kolla_base_distro == 'centos' %} bifrost_tag: wallaby-20230207T194135 {% else %} -bifrost_tag: wallaby-20220825T112231 -cinder_tag: wallaby-20230125T132411 -cloudkitty_tag: wallaby-20221215T220154 -etcd_tag: wallaby-20230131T123519 -glance_tag: wallaby-20230125T132411 -kolla_toolbox_tag: wallaby-20221222T161624 -magnum_tag: wallaby-20230111T103759 -neutron_tag: wallaby-20221222T161624 -nova_tag: wallaby-20230125T132411 -octavia_tag: wallaby-20221222T161624 -openvswitch_tag: wallaby-20221222T161624 -ovn_tag: wallaby-20221222T161624 -rabbitmq_tag: wallaby-20230105T144604 +bifrost_tag: wallaby-20230215T160405 {% endif %} glance_tls_proxy_tag: "{% raw %}{{ haproxy_tag | default(openstack_tag) }}{% endraw %}" From 9a03f40c2fc1137fa45be464ae936641a6709988 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 17 Feb 2023 14:32:35 +0100 Subject: [PATCH 11/16] Document how to override swap_device --- etc/kayobe/ansible/swap.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/swap.yml b/etc/kayobe/ansible/swap.yml index e47397212..97d7a5a0b 100644 --- a/etc/kayobe/ansible/swap.yml +++ b/etc/kayobe/ansible/swap.yml @@ -6,7 +6,9 @@ # * swap_group: Host pattern against which to target the playbook. Default is # 'overcloud'. # * swap_device: Name of the swap device to configure. Default is -# '/dev/rootvg/lv_swap'. +# '/dev/rootvg/lv_swap'. To use a different value, set an extra +# var (e.g. in $KAYOBE_CONFIG_PATH/swap.yml) because play vars +# take precedence over any inventory variable. - name: Configure swap hosts: "{{ swap_group | default('overcloud') }}" From eaa75644f0a7bfa902ba70fddb28c8b811b3a9f6 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Sun, 19 Feb 2023 15:09:13 +0100 Subject: [PATCH 12/16] Fix Ansible deprecation warning --- etc/kayobe/ansible.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/ansible.cfg b/etc/kayobe/ansible.cfg index 310c79499..515af8d32 100644 --- a/etc/kayobe/ansible.cfg +++ b/etc/kayobe/ansible.cfg @@ -7,7 +7,7 @@ bin_ansible_callbacks = True # Disable fact variable injection to improve performance. inject_facts_as_vars = False # Add timing information to output -callback_whitelist = ansible.posix.profile_tasks +callbacks_enabled = ansible.posix.profile_tasks [ssh_connection] pipelining = True From 02cf4783e99ba52f27c164a9cfd88e3c6575ac2f Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Mon, 20 Feb 2023 14:17:38 +0000 Subject: [PATCH 13/16] Adds release note for ubuntu container refresh Package versions from output below. old: ```javascript ()[root@01b9e3ddbfbb /]# dpkg -l | grep libvirt ii libvirt-clients 6.0.0-0ubuntu8.16 amd64 Programs for the libvirt library ii libvirt-daemon 6.0.0-0ubuntu8.16 amd64 Virtualization daemon ii libvirt-daemon-driver-qemu 6.0.0-0ubuntu8.16 amd64 Virtualization daemon QEMU connection driver ii libvirt-daemon-system 6.0.0-0ubuntu8.16 amd64 Libvirt daemon configuration files ii libvirt-daemon-system-systemd 6.0.0-0ubuntu8.16 amd64 Libvirt daemon configuration files (systemd) ii libvirt0:amd64 6.0.0-0ubuntu8.16 amd64 library for interfacing with different virtualization systems ``` new: ```javascript ()[root@3493c5fdc96a /]# dpkg -l | grep libvirt ii libvirt-clients 6.0.0-0ubuntu8.16 amd64 Programs for the libvirt library ii libvirt-daemon 6.0.0-0ubuntu8.16 amd64 Virtualization daemon ii libvirt-daemon-driver-qemu 6.0.0-0ubuntu8.16 amd64 Virtualization daemon QEMU connection driver ii libvirt-daemon-system 6.0.0-0ubuntu8.16 amd64 Libvirt daemon configuration files ii libvirt-daemon-system-systemd 6.0.0-0ubuntu8.16 amd64 Libvirt daemon configuration files (systemd) ii libvirt0:amd64 6.0.0-0ubuntu8.16 amd64 library for interfacing with different virtualization systems ``` old: ``` (venv-pulp) [cloud-user@will-control-host-0 ~]$ docker run -it pulp-server.internal.sms-cloud:8080/stackhpc-dev/ubuntu-source-openvswitch-vswitchd:wallaby-20221222T161624 bash ()[root@9f15e78c72fc /]# ovs-vswitchd --version ovs-vswitchd (Open vSwitch) 2.17.2 ()[root@9f15e78c72fc /]# dpkg -l | grep openvswitch-switch ii openvswitch-switch 2.17.2-0ubuntu0.22.04.2~cloud0 amd64 Open vSwitch switch implementations ``` new: ``` ()[root@ca7be869ba89 /]# dpkg -l | grep vswitch ii openvswitch-common 2.17.3-0ubuntu0.22.04.1~cloud0 amd64 Open vSwitch common components ii openvswitch-switch 2.17.3-0ubuntu0.22.04.1~cloud0 amd64 Open vSwitch switch implementations ii python3-openvswitch 2.17.3-0ubuntu0.22.04.1~cloud0 all Python 3 bindings for Open vSwitch ``` old: ``` ()[root@c2f9a61b303e /]# dpkg -l | grep ovn ii ovn-central 22.03.0-0ubuntu1~cloud0 amd64 OVN central components ii ovn-common 22.03.0-0ubuntu1~cloud0 amd64 OVN common components ``` new: ``` Status: Downloaded newer image for pulp-server.internal.sms-cloud:8080/stackhpc-dev/ubuntu-source-ovn-northd:wallaby-20230215T134157 ()[root@fd78907feea3 /]# dpkg -l | grep ovn ii ovn-central 22.03.0-0ubuntu1~cloud0 amd64 OVN central components ii ovn-common 22.03.0-0ubuntu1~cloud0 amd64 OVN common components ``` --- ...3-package-refresh-ubuntu-a9fe8a1c3c2f2796.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 releasenotes/notes/release-train-02-2023-package-refresh-ubuntu-a9fe8a1c3c2f2796.yaml diff --git a/releasenotes/notes/release-train-02-2023-package-refresh-ubuntu-a9fe8a1c3c2f2796.yaml b/releasenotes/notes/release-train-02-2023-package-refresh-ubuntu-a9fe8a1c3c2f2796.yaml new file mode 100644 index 000000000..722b68034 --- /dev/null +++ b/releasenotes/notes/release-train-02-2023-package-refresh-ubuntu-a9fe8a1c3c2f2796.yaml @@ -0,0 +1,15 @@ +--- +features: + - | + Container images for Ubuntu based deployments have been updated. + Key packages to note are: + + * Libvirt (unchanged since last container build) + * version: 6.0.0 + * release: 0ubuntu8.16 + * OVS + * version: 2.17.3 + * release: 0ubuntu0.22.04.1~cloud0 + * OVN (unchanged since last container build) + * version: 22.03.0 + * release: 0ubuntu1~cloud0 From c6c5a01fff32237fa81fd3c03594a9eeb71e6735 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Mon, 20 Feb 2023 15:03:05 +0000 Subject: [PATCH 14/16] Refresh Ubuntu images Build logs: https://github.com/stackhpc/stackhpc-kayobe-config/actions/runs/4186064857 --- etc/kayobe/kolla.yml | 2 +- etc/kayobe/kolla/globals.yml | 13 +------------ 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index f4c70bc34..b328f0d94 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -96,7 +96,7 @@ kolla_docker_registry_password: "{{ stackhpc_docker_registry_password }}" # Kolla OpenStack release version. This should be a Docker image tag. # Default is {{ openstack_release }}. -kolla_openstack_release: "{% if kolla_base_distro == 'centos' %}xena-20230214T150257{% else %}xena-20221213T204703{% endif %}" +kolla_openstack_release: "{% if kolla_base_distro == 'centos' %}xena-20230214T150257{% else %}xena-20230215T164524{% endif %}" # Docker tag applied to built container images. Default is # {{ kolla_openstack_release }}. diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index f0d66d241..0b1c0d62c 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -10,18 +10,7 @@ enable_docker_repo: {% raw %}"{{ 'overcloud' not in group_names or ansible_facts {% if kolla_base_distro == 'centos' %} bifrost_tag: xena-20230214T165534 {% else %} -bifrost_tag: xena-20221213T224057 -cinder_tag: xena-20230206T165220 -etcd_tag: xena-20230201T131415 -glance_tag: xena-20230206T165220 -kolla_toolbox_tag: xena-20230104T145414 -magnum_tag: xena-20230131T151601 -neutron_tag: xena-20230104T145414 -nova_tag: xena-20230206T165220 -octavia_tag: xena-20230104T145414 -openvswitch_tag: xena-20230104T145414 -ovn_tag: xena-20230104T145414 -rabbitmq_tag: xena-20230105T145825 +bifrost_tag: xena-20230215T195824 {% endif %} glance_tls_proxy_tag: "{% raw %}{{ haproxy_tag | default(openstack_tag) }}{% endraw %}" From 315c3fd35486d31c6f41624fb07090e14b81f4ce Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Tue, 21 Feb 2023 09:34:07 +0000 Subject: [PATCH 15/16] Refresh ubuntu container images Build logs: https://github.com/stackhpc/stackhpc-kayobe-config/actions/runs/4224755555 --- etc/kayobe/kolla.yml | 2 +- etc/kayobe/kolla/globals.yml | 9 +-------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 5641dbce1..9df522533 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -96,7 +96,7 @@ kolla_docker_registry_password: "{{ stackhpc_docker_registry_password }}" # Kolla OpenStack release version. This should be a Docker image tag. # Default is {{ openstack_release }}. -kolla_openstack_release: "{% if kolla_base_distro == 'centos' %}yoga-20230217T135826{% else %}yoga-20221215T122529{% endif %}" +kolla_openstack_release: "{% if kolla_base_distro == 'centos' %}yoga-20230217T135826{% else %}yoga-20230220T181235{% endif %}" # Docker tag applied to built container images. Default is # {{ kolla_openstack_release }}. diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index f5628f283..3a7ec0ef7 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -10,14 +10,7 @@ enable_docker_repo: {% raw %}"{{ 'overcloud' not in group_names or ansible_facts {% if kolla_base_distro == 'centos' %} bifrost_tag: yoga-20230217T160618 {% else %} -bifrost_tag: yoga-20221215T142947 -blazar_tag: yoga-20230215T113406 -cinder_tag: yoga-20230125T114437 -etcd_tag: yoga-20230207T153642 -glance_tag: yoga-20230125T114437 -magnum_tag: yoga-20230131T174509 -neutron_tag: yoga-20230104T163118 -nova_tag: yoga-20230125T114437 +bifrost_tag: yoga-20230220T184947 {% endif %} glance_tls_proxy_tag: "{% raw %}{{ haproxy_tag | default(openstack_tag) }}{% endraw %}" From af6e34e95b2ff9e4fc362be3b6e9fbf90379d7d3 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Tue, 21 Feb 2023 09:51:29 +0000 Subject: [PATCH 16/16] Correct release note for yoga release old: ``` ()[root@e2fca7bbade9 /]# dpkg -l | grep libvirt ii libvirt-clients 8.0.0-1ubuntu7.1~cloud0 amd64 Programs for the libvirt library ii libvirt-daemon 8.0.0-1ubuntu7.1~cloud0 amd64 Virtualization daemon ii libvirt-daemon-config-network 8.0.0-1ubuntu7.1~cloud0 all Libvirt daemon configuration files (default network) ii libvirt-daemon-config-nwfilter 8.0.0-1ubuntu7.1~cloud0 all Libvirt daemon configuration files (default network filters) ii libvirt-daemon-driver-qemu 8.0.0-1ubuntu7.1~cloud0 amd64 Virtualization daemon QEMU connection driver ii libvirt-daemon-system 8.0.0-1ubuntu7.1~cloud0 amd64 Libvirt daemon configuration files ii libvirt-daemon-system-systemd 8.0.0-1ubuntu7.1~cloud0 all Libvirt daemon configuration files (systemd) ii libvirt0:amd64 8.0.0-1ubuntu7.1~cloud0 amd64 library for interfacing with different virtualization systems ``` new: ``` Status: Downloaded newer image for pulp-server.internal.sms-cloud:8080/stackhpc-dev/ubuntu-source-nova-libvirt:yoga-20230220T181235 ()[root@2ab3130b59ef /]# dpkg -l | grep libvirt ii libvirt-clients 8.0.0-1ubuntu7.4~cloud0 amd64 Programs for the libvirt library ii libvirt-daemon 8.0.0-1ubuntu7.4~cloud0 amd64 Virtualization daemon ii libvirt-daemon-config-network 8.0.0-1ubuntu7.4~cloud0 all Libvirt daemon configuration files (default network) ii libvirt-daemon-config-nwfilter 8.0.0-1ubuntu7.4~cloud0 all Libvirt daemon configuration files (default network filters) ii libvirt-daemon-driver-qemu 8.0.0-1ubuntu7.4~cloud0 amd64 Virtualization daemon QEMU connection driver ii libvirt-daemon-system 8.0.0-1ubuntu7.4~cloud0 amd64 Libvirt daemon configuration files ii libvirt-daemon-system-systemd 8.0.0-1ubuntu7.4~cloud0 all Libvirt daemon configuration files (systemd) ii libvirt0:amd64 8.0.0-1ubuntu7.4~cloud0 amd64 library for interfacing with different virtualization systems ``` --- ...ain-02-2023-package-refresh-ubuntu-a9fe8a1c3c2f2796.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/releasenotes/notes/release-train-02-2023-package-refresh-ubuntu-a9fe8a1c3c2f2796.yaml b/releasenotes/notes/release-train-02-2023-package-refresh-ubuntu-a9fe8a1c3c2f2796.yaml index 722b68034..1b9d4f1d8 100644 --- a/releasenotes/notes/release-train-02-2023-package-refresh-ubuntu-a9fe8a1c3c2f2796.yaml +++ b/releasenotes/notes/release-train-02-2023-package-refresh-ubuntu-a9fe8a1c3c2f2796.yaml @@ -4,9 +4,9 @@ features: Container images for Ubuntu based deployments have been updated. Key packages to note are: - * Libvirt (unchanged since last container build) - * version: 6.0.0 - * release: 0ubuntu8.16 + * Libvirt + * version: 8.0.0 + * release: 1ubuntu7.4~cloud0 * OVS * version: 2.17.3 * release: 0ubuntu0.22.04.1~cloud0