From bc66f63c48b6911c15b2290ad0089b83fe34c6af Mon Sep 17 00:00:00 2001
From: Pierre Riteau <pierre@stackhpc.com>
Date: Tue, 19 Oct 2021 10:58:19 +0200
Subject: [PATCH 1/3] Add support for configuring proxy settings

Change-Id: Ic5130a7512d4a26354bd292b0ab51ab4a9279f0a
(cherry picked from commit e48960ecf20c696b2a623bdfbfa7fa4637ff4588)
---
 ansible/group_vars/all/kolla                  | 15 +++++++
 ansible/group_vars/all/proxy                  | 19 +++++++++
 ansible/proxy.yml                             | 41 +++++++++++++++++++
 ansible/roles/kolla-ansible/defaults/main.yml | 14 ++++++-
 .../kolla-ansible/templates/kolla/globals.yml | 21 ++++++++++
 etc/kayobe/kolla.yml                          | 15 +++++++
 etc/kayobe/proxy.yml                          | 16 ++++++++
 kayobe/cli/commands.py                        | 12 ++++--
 kayobe/tests/unit/cli/test_commands.py        |  4 ++
 .../proxy-settings-32911948a517b35b.yaml      |  8 ++++
 10 files changed, 160 insertions(+), 5 deletions(-)
 create mode 100644 ansible/group_vars/all/proxy
 create mode 100644 ansible/proxy.yml
 create mode 100644 etc/kayobe/proxy.yml
 create mode 100644 releasenotes/notes/proxy-settings-32911948a517b35b.yaml

diff --git a/ansible/group_vars/all/kolla b/ansible/group_vars/all/kolla
index 0f7e155f4..5eeb81a48 100644
--- a/ansible/group_vars/all/kolla
+++ b/ansible/group_vars/all/kolla
@@ -675,3 +675,18 @@ kolla_internal_tls_cert:
 # in admin-openrc.sh file when TLS is enabled, instead of Kolla-Ansible's
 # default.
 kolla_internal_fqdn_cacert:
+
+###############################################################################
+# Proxy configuration
+
+# HTTP proxy URL (format: http(s)://[user:password@]proxy_name:port) used by
+# Kolla. Default value is "{{ http_proxy }}".
+kolla_http_proxy: "{{ http_proxy }}"
+
+# HTTPS proxy URL (format: http(s)://[user:password@]proxy_name:port) used by
+# Kolla. Default value is "{{ https_proxy }}".
+kolla_https_proxy: "{{ https_proxy }}"
+
+# List of domains, hostnames, IP addresses and networks for which no proxy is
+# used. Default value is "{{ no_proxy }}".
+kolla_no_proxy: "{{ no_proxy }}"
diff --git a/ansible/group_vars/all/proxy b/ansible/group_vars/all/proxy
new file mode 100644
index 000000000..ef1f89635
--- /dev/null
+++ b/ansible/group_vars/all/proxy
@@ -0,0 +1,19 @@
+---
+###############################################################################
+# Configuration of HTTP(S) proxies.
+
+# HTTP proxy URL (format: http(s)://[user:password@]proxy_name:port). By
+# default no proxy is used.
+http_proxy: ""
+
+# HTTPS proxy URL (format: http(s)://[user:password@]proxy_name:port). By
+# default no proxy is used.
+https_proxy: ""
+
+# List of domains, hostnames, IP addresses and networks for which no proxy is
+# used. Defaults to ["127.0.0.1", "localhost", "{{ docker_registry }}"]. This
+# is configured only if either http_proxy or https_proxy is set.
+no_proxy:
+  - "127.0.0.1"
+  - "localhost"
+  - "{{ docker_registry }}"
diff --git a/ansible/proxy.yml b/ansible/proxy.yml
new file mode 100644
index 000000000..14c7b72c2
--- /dev/null
+++ b/ansible/proxy.yml
@@ -0,0 +1,41 @@
+- name: Configure HTTP(S) proxy settings
+  hosts: seed-hypervisor:seed:overcloud
+  vars:
+    ansible_python_interpreter: /usr/bin/python3
+  tags:
+    - proxy
+  tasks:
+    - name: Add HTTP proxy configuration to /etc/environment
+      lineinfile:
+        path: "/etc/environment"
+        create: yes
+        mode: 0644
+        state: present
+        regexp: "^http_proxy=.*"
+        line: "http_proxy={{ http_proxy }}"
+      become: True
+      when: http_proxy is defined and http_proxy | length > 0
+
+    - name: Add HTTPS proxy configuration to /etc/environment
+      lineinfile:
+        path: "/etc/environment"
+        create: yes
+        mode: 0644
+        state: present
+        regexp: "^https_proxy=.*"
+        line: "https_proxy={{ https_proxy }}"
+      become: True
+      when: https_proxy is defined and https_proxy | length > 0
+
+    - name: Add no_proxy configuration to /etc/environment
+      lineinfile:
+        path: "/etc/environment"
+        create: yes
+        mode: 0644
+        state: present
+        regexp: "^no_proxy=.*"
+        line: "no_proxy={{ no_proxy | select | join(',') }}"
+      become: True
+      when:
+        - no_proxy | length > 0
+        - http_proxy is defined and http_proxy | length > 0 or https_proxy is defined and https_proxy | length > 0
diff --git a/ansible/roles/kolla-ansible/defaults/main.yml b/ansible/roles/kolla-ansible/defaults/main.yml
index d366bac07..9a867069b 100644
--- a/ansible/roles/kolla-ansible/defaults/main.yml
+++ b/ansible/roles/kolla-ansible/defaults/main.yml
@@ -137,7 +137,6 @@ kolla_external_vip_address:
 # kolla_external_vip_address.
 kolla_external_fqdn:
 
-
 ####################
 # Networking options
 ####################
@@ -312,3 +311,16 @@ docker_daemon_mtu: 1500
 
 # Enable live-restore on docker daemon
 docker_daemon_live_restore: false
+
+###############################################################################
+# Proxy configuration
+
+# HTTP proxy URL (format: http(s)://[user:password@]proxy_name:port).
+kolla_http_proxy:
+
+# HTTPS proxy URL (format: http(s)://[user:password@]proxy_name:port).
+kolla_https_proxy:
+
+# List of domains, hostnames, IP addresses and networks for which no proxy is
+# used.
+kolla_no_proxy:
diff --git a/ansible/roles/kolla-ansible/templates/kolla/globals.yml b/ansible/roles/kolla-ansible/templates/kolla/globals.yml
index cc7df6b2b..22e2fbaf5 100644
--- a/ansible/roles/kolla-ansible/templates/kolla/globals.yml
+++ b/ansible/roles/kolla-ansible/templates/kolla/globals.yml
@@ -50,6 +50,17 @@ kolla_external_vip_address: "{{ kolla_external_vip_address }}"
 kolla_external_fqdn: "{{ kolla_external_fqdn }}"
 {% endif %}
 
+# Proxy settings for containers such as magnum that need Internet access
+{% if kolla_http_proxy is not none and kolla_http_proxy | length > 0 %}
+container_http_proxy: "{{ kolla_http_proxy }}"
+{% endif %}
+{% if kolla_https_proxy is not none and kolla_https_proxy | length > 0 %}
+container_https_proxy: "{{ kolla_https_proxy }}"
+{% endif %}
+{% if kolla_no_proxy is not none and kolla_no_proxy | length > 0 %}
+container_no_proxy: "{{ kolla_no_proxy | select | join(',') }}"
+{% endif %}
+
 ################
 # Docker options
 ################
@@ -66,6 +77,16 @@ docker_registry_username: "{{ kolla_docker_registry_username }}"
 docker_storage_driver: "{{ docker_storage_driver }}"
 docker_custom_config: {{ kolla_docker_custom_config | to_nice_json | indent(2) }}
 
+{% if kolla_http_proxy is not none and kolla_http_proxy | length > 0 %}
+docker_http_proxy: "{{ kolla_http_proxy }}"
+{% endif %}
+{% if kolla_https_proxy is not none and kolla_https_proxy | length > 0 %}
+docker_https_proxy: "{{ kolla_https_proxy }}"
+{% endif %}
+{% if kolla_no_proxy is not none and kolla_no_proxy | length > 0 %}
+docker_no_proxy: "{{ kolla_no_proxy | select | join(',') }}"
+{% endif %}
+
 #docker_configure_for_zun: "no"
 
 ###################
diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml
index 5ef2c460b..8a228e7fc 100644
--- a/etc/kayobe/kolla.yml
+++ b/etc/kayobe/kolla.yml
@@ -480,6 +480,21 @@
 # default.
 #kolla_internal_fqdn_cacert:
 
+###############################################################################
+# Proxy configuration
+
+# HTTP proxy URL (format: http(s)://[user:password@]proxy_name:port) used by
+# Kolla. Default value is "{{ http_proxy }}".
+#kolla_http_proxy:
+
+# HTTPS proxy URL (format: http(s)://[user:password@]proxy_name:port) used by
+# Kolla. Default value is "{{ https_proxy }}".
+#kolla_https_proxy:
+
+# List of domains, hostnames, IP addresses and networks for which no proxy is
+# used. Default value is "{{ no_proxy }}".
+#kolla_no_proxy:
+
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.
 workaround_ansible_issue_8743: yes
diff --git a/etc/kayobe/proxy.yml b/etc/kayobe/proxy.yml
new file mode 100644
index 000000000..aaf938983
--- /dev/null
+++ b/etc/kayobe/proxy.yml
@@ -0,0 +1,16 @@
+---
+###############################################################################
+# Configuration of HTTP(S) proxies.
+
+# HTTP proxy URL (format: http(s)://[user:password@]proxy_name:port). By
+# default no proxy is used.
+#http_proxy:
+
+# HTTPS proxy URL (format: http(s)://[user:password@]proxy_name:port). By
+# default no proxy is used.
+#https_proxy:
+
+# List of domains, hostnames, IP addresses and networks for which no proxy is
+# used. Defaults to ["127.0.0.1", "localhost", "{{ docker_registry }}"]. This
+# is configured only if either http_proxy or https_proxy is set.
+#no_proxy:
diff --git a/kayobe/cli/commands.py b/kayobe/cli/commands.py
index f68b4e9e6..8bb1d41b0 100644
--- a/kayobe/cli/commands.py
+++ b/kayobe/cli/commands.py
@@ -409,6 +409,7 @@ class SeedHypervisorHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin,
     * Allocate IP addresses for all configured networks.
     * Add the host to SSH known hosts.
     * Configure a user account for use by kayobe for SSH access.
+    * Configure proxy settings.
     * Configure package repos.
     * Configure a PyPI mirror.
     * Optionally, create a virtualenv for remote target hosts.
@@ -441,7 +442,7 @@ def take_action(self, parsed_args):
                                   limit="seed-hypervisor")
 
         playbooks = _build_playbook_list(
-            "ssh-known-host", "kayobe-ansible-user",
+            "ssh-known-host", "kayobe-ansible-user", "proxy",
             "apt", "dnf", "pip", "kayobe-target-venv")
         if parsed_args.wipe_disks:
             playbooks += _build_playbook_list("wipe-disks")
@@ -557,6 +558,7 @@ class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
     * Allocate IP addresses for all configured networks.
     * Add the host to SSH known hosts.
     * Configure a user account for use by kayobe for SSH access.
+    * Configure proxy settings.
     * Configure package repos.
     * Configure a PyPI mirror.
     * Optionally, create a virtualenv for remote target hosts.
@@ -596,7 +598,7 @@ def take_action(self, parsed_args):
 
         # Run kayobe playbooks.
         playbooks = _build_playbook_list(
-            "ssh-known-host", "kayobe-ansible-user",
+            "ssh-known-host", "kayobe-ansible-user", "proxy",
             "apt", "dnf", "pip", "kayobe-target-venv")
         if parsed_args.wipe_disks:
             playbooks += _build_playbook_list("wipe-disks")
@@ -867,6 +869,7 @@ class InfraVMHostConfigure(KayobeAnsibleMixin, VaultMixin,
     * Allocate IP addresses for all configured networks.
     * Add the host to SSH known hosts.
     * Configure a user account for use by kayobe for SSH access.
+    * Configure proxy settings.
     * Configure package repos.
     * Configure a PyPI mirror.
     * Optionally, create a virtualenv for remote target hosts.
@@ -901,7 +904,7 @@ def take_action(self, parsed_args):
 
         # Kayobe playbooks.
         playbooks = _build_playbook_list(
-            "ssh-known-host", "kayobe-ansible-user",
+            "ssh-known-host", "kayobe-ansible-user", "proxy",
             "dnf", "pip", "kayobe-target-venv")
         if parsed_args.wipe_disks:
             playbooks += _build_playbook_list("wipe-disks")
@@ -1115,6 +1118,7 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
     * Allocate IP addresses for all configured networks.
     * Add the host to SSH known hosts.
     * Configure a user account for use by kayobe for SSH access.
+    * Configure proxy settings.
     * Configure package repos.
     * Configure a PyPI mirror.
     * Optionally, create a virtualenv for remote target hosts.
@@ -1153,7 +1157,7 @@ def take_action(self, parsed_args):
 
         # Kayobe playbooks.
         playbooks = _build_playbook_list(
-            "ssh-known-host", "kayobe-ansible-user",
+            "ssh-known-host", "kayobe-ansible-user", "proxy",
             "apt", "dnf", "pip", "kayobe-target-venv")
         if parsed_args.wipe_disks:
             playbooks += _build_playbook_list("wipe-disks")
diff --git a/kayobe/tests/unit/cli/test_commands.py b/kayobe/tests/unit/cli/test_commands.py
index d0cbbd8d3..5bcbab85d 100644
--- a/kayobe/tests/unit/cli/test_commands.py
+++ b/kayobe/tests/unit/cli/test_commands.py
@@ -319,6 +319,7 @@ def test_seed_hypervisor_host_configure(self, mock_run):
                     utils.get_data_files_path("ansible", "ssh-known-host.yml"),
                     utils.get_data_files_path(
                         "ansible", "kayobe-ansible-user.yml"),
+                    utils.get_data_files_path("ansible", "proxy.yml"),
                     utils.get_data_files_path("ansible", "apt.yml"),
                     utils.get_data_files_path("ansible", "dnf.yml"),
                     utils.get_data_files_path("ansible", "pip.yml"),
@@ -491,6 +492,7 @@ def test_seed_host_configure(self, mock_kolla_run, mock_run):
                     utils.get_data_files_path("ansible", "ssh-known-host.yml"),
                     utils.get_data_files_path(
                         "ansible", "kayobe-ansible-user.yml"),
+                    utils.get_data_files_path("ansible", "proxy.yml"),
                     utils.get_data_files_path("ansible", "apt.yml"),
                     utils.get_data_files_path("ansible", "dnf.yml"),
                     utils.get_data_files_path("ansible", "pip.yml"),
@@ -982,6 +984,7 @@ def test_infra_vm_host_configure(self, mock_run):
                     utils.get_data_files_path("ansible", "ssh-known-host.yml"),
                     utils.get_data_files_path(
                         "ansible", "kayobe-ansible-user.yml"),
+                    utils.get_data_files_path("ansible", "proxy.yml"),
                     utils.get_data_files_path("ansible", "dnf.yml"),
                     utils.get_data_files_path("ansible", "pip.yml"),
                     utils.get_data_files_path(
@@ -1259,6 +1262,7 @@ def test_overcloud_host_configure(self, mock_kolla_run, mock_run):
                     utils.get_data_files_path("ansible", "ssh-known-host.yml"),
                     utils.get_data_files_path(
                         "ansible", "kayobe-ansible-user.yml"),
+                    utils.get_data_files_path("ansible", "proxy.yml"),
                     utils.get_data_files_path("ansible", "apt.yml"),
                     utils.get_data_files_path("ansible", "dnf.yml"),
                     utils.get_data_files_path("ansible", "pip.yml"),
diff --git a/releasenotes/notes/proxy-settings-32911948a517b35b.yaml b/releasenotes/notes/proxy-settings-32911948a517b35b.yaml
new file mode 100644
index 000000000..8084c8217
--- /dev/null
+++ b/releasenotes/notes/proxy-settings-32911948a517b35b.yaml
@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Adds support for configuring HTTP(S) proxy settings using the
+    ``http_proxy``, ``https_proxy`` and ``no_proxy`` variables in
+    ``proxy.yml``. These variables are passed down to Kolla Ansible which uses
+    them to configure Docker, allowing container image pull operations and
+    container networking to use HTTP(S) proxies.

From 29b16119855a7c05d48ca06af309dee6195fb06c Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Mon, 11 Apr 2022 14:38:15 +0100
Subject: [PATCH 2/3] proxy: add ansible issue 8743 workaround

Change-Id: I6515d028914e8956eac23662c3714a908fda6ac4
(cherry picked from commit 5ede87656c05a95e34369417a660fd1b6d06bbfc)
---
 etc/kayobe/proxy.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/etc/kayobe/proxy.yml b/etc/kayobe/proxy.yml
index aaf938983..7d63017d6 100644
--- a/etc/kayobe/proxy.yml
+++ b/etc/kayobe/proxy.yml
@@ -14,3 +14,7 @@
 # used. Defaults to ["127.0.0.1", "localhost", "{{ docker_registry }}"]. This
 # is configured only if either http_proxy or https_proxy is set.
 #no_proxy:
+
+###############################################################################
+# Dummy variable to allow Ansible to accept this file.
+workaround_ansible_issue_8743: yes

From 829f87bed621596d6798f6539ee93e2eb928ce06 Mon Sep 17 00:00:00 2001
From: Pierre Riteau <pierre@stackhpc.com>
Date: Tue, 19 Apr 2022 08:58:22 +0200
Subject: [PATCH 3/3] Fix no_proxy configuration

The no_proxy list should only contain domains, hostnames, IP addresses
and networks, but docker_registry is often in the form ip_address:port.

Use urlsplit to extract the hostname from the docker_registry variable
after prepending http:// to turn it into a valid URL.

Also add missing infra-vms to hosts in proxy.yml.

Change-Id: I6424fc405894514a63fb2b641637bbb9d5c070c0
(cherry picked from commit 71eb21a3ddab249fe733b56c5f9b70d2947308a5)
---
 ansible/group_vars/all/proxy | 8 +++++---
 ansible/proxy.yml            | 8 ++++----
 etc/kayobe/proxy.yml         | 6 ++++--
 3 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/ansible/group_vars/all/proxy b/ansible/group_vars/all/proxy
index ef1f89635..eb791bbef 100644
--- a/ansible/group_vars/all/proxy
+++ b/ansible/group_vars/all/proxy
@@ -11,9 +11,11 @@ http_proxy: ""
 https_proxy: ""
 
 # List of domains, hostnames, IP addresses and networks for which no proxy is
-# used. Defaults to ["127.0.0.1", "localhost", "{{ docker_registry }}"]. This
-# is configured only if either http_proxy or https_proxy is set.
+# used. Defaults to ["127.0.0.1", "localhost", "{{ ('http://' ~
+# docker_registry) | urlsplit('hostname') }}"] if docker_registry is set, or
+# ["127.0.0.1", "localhost"] otherwise. This is configured only if either
+# http_proxy or https_proxy is set.
 no_proxy:
   - "127.0.0.1"
   - "localhost"
-  - "{{ docker_registry }}"
+  - "{{ ('http://' ~ docker_registry) | urlsplit('hostname') if docker_registry else '' }}"
diff --git a/ansible/proxy.yml b/ansible/proxy.yml
index 14c7b72c2..e618b9c0d 100644
--- a/ansible/proxy.yml
+++ b/ansible/proxy.yml
@@ -1,5 +1,5 @@
 - name: Configure HTTP(S) proxy settings
-  hosts: seed-hypervisor:seed:overcloud
+  hosts: seed-hypervisor:seed:overcloud:infra-vms
   vars:
     ansible_python_interpreter: /usr/bin/python3
   tags:
@@ -14,7 +14,7 @@
         regexp: "^http_proxy=.*"
         line: "http_proxy={{ http_proxy }}"
       become: True
-      when: http_proxy is defined and http_proxy | length > 0
+      when: http_proxy | length > 0
 
     - name: Add HTTPS proxy configuration to /etc/environment
       lineinfile:
@@ -25,7 +25,7 @@
         regexp: "^https_proxy=.*"
         line: "https_proxy={{ https_proxy }}"
       become: True
-      when: https_proxy is defined and https_proxy | length > 0
+      when: https_proxy | length > 0
 
     - name: Add no_proxy configuration to /etc/environment
       lineinfile:
@@ -38,4 +38,4 @@
       become: True
       when:
         - no_proxy | length > 0
-        - http_proxy is defined and http_proxy | length > 0 or https_proxy is defined and https_proxy | length > 0
+        - http_proxy | length > 0 or https_proxy | length > 0
diff --git a/etc/kayobe/proxy.yml b/etc/kayobe/proxy.yml
index 7d63017d6..714b9dae5 100644
--- a/etc/kayobe/proxy.yml
+++ b/etc/kayobe/proxy.yml
@@ -11,8 +11,10 @@
 #https_proxy:
 
 # List of domains, hostnames, IP addresses and networks for which no proxy is
-# used. Defaults to ["127.0.0.1", "localhost", "{{ docker_registry }}"]. This
-# is configured only if either http_proxy or https_proxy is set.
+# used. Defaults to ["127.0.0.1", "localhost", "{{ ('http://' ~
+# docker_registry) | urlsplit('hostname') }}"] if docker_registry is set, or
+# ["127.0.0.1", "localhost"] otherwise. This is configured only if either
+# http_proxy or https_proxy is set.
 #no_proxy:
 
 ###############################################################################