Merge branch 'ypid-fix/debops-dev-patchset-13'

debops · Jan 9, 2023 · eea3870 · eea3870
2 parents 3be2958 + b6d39de
commit eea3870
Show file tree

Hide file tree

Showing 22 changed files with 136 additions and 62 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -182,7 +182,7 @@ General
 
 - The debops-contrib :file:`dropbear_initramfs` playbook has been moved to
   the :ref:`debops.dropbear_initramfs` playbook. The role variable
-  ``dropbear_initramfs__host_authorized_keys`` now uses the same keys as 
+  ``dropbear_initramfs__host_authorized_keys`` now uses the same keys as
   the ``ansible.posix.authorized_key`` module.
 
 - Various tasks that interact with the MariaDB/MySQL databases will now use the
@@ -227,9 +227,9 @@ General
 
 - The role has been rewritten from scratch and now can be used to deploy and
   manage a `GitLab Omnibus`__ instance (managed internally by Chef) on Debian
-  or Ubuntu hosts. Role integrates various DebOps services (firewall, PKI
-  infrastructure, LDAP environment) with GitLab Omnibus. Bot Community Edition
-  (default) and Enterprise Edition are supported.
+  or Ubuntu hosts. The role integrates with various DebOps services (firewall,
+  PKI infrastructure, LDAP environment) with GitLab Omnibus. Both Community
+  Edition (default) and Enterprise Edition are supported.
 
   .. __: https://docs.gitlab.com/omnibus/
 

diff --git a/ansible/debops-contrib-playbooks/service/all.yml b/ansible/debops-contrib-playbooks/service/all.yml
@@ -8,7 +8,6 @@
 - import_playbook: bitcoind.yml
 - import_playbook: btrfs.yml
 - import_playbook: dnsmasq.yml
-- import_playbook: dropbear_initramfs.yml
 - import_playbook: firejail.yml
 - import_playbook: foodsoft.yml
 - import_playbook: fuse.yml

diff --git a/ansible/debops-contrib-playbooks/service/homeassistant-nginx.yml b/ansible/debops-contrib-playbooks/service/homeassistant-nginx.yml
@@ -13,6 +13,15 @@
                    | combine(inventory__group_environment | d({}))
                    | combine(inventory__host_environment  | d({})) }}'
 
+  pre_tasks:
+
+    - name: Prepare homeassistant environment
+      ansible.builtin.import_role:
+        name: 'homeassistant'
+        tasks_from: 'main_env'
+      tags: [ 'role::homeassistant', 'role::nginx' ]
+
+
   roles:
 
     - role: keyring

diff --git a/ansible/roles/authorized_keys/defaults/main.yml b/ansible/roles/authorized_keys/defaults/main.yml
@@ -42,7 +42,7 @@ authorized_keys__system: True
 # -----------------------------
 
 # These lists define SSH public key identities managed by the role with
-# coorresponding UNIX accounts. See :ref:`authorized_keys__ref_identities` for
+# corresponding UNIX accounts. See :ref:`authorized_keys__ref_identities` for
 # more details.
 
 # .. envvar:: authorized_keys__identities [[[

diff --git a/ansible/roles/btrfs/defaults/main.yml b/ansible/roles/btrfs/defaults/main.yml
@@ -23,7 +23,7 @@
 #
 # List of base packages to install.
 btrfs__base_packages:
-  - 'btrfs-tools'
+  - 'btrfs-progs'
                                                                    # ]]]
                                                                    # ]]]
 # Subvolumes [[[

diff --git a/ansible/roles/etesync/defaults/main.yml b/ansible/roles/etesync/defaults/main.yml
@@ -446,22 +446,22 @@ etesync__nginx__dependent_servers:
           proxy_set_header X-Forwarded-Host $server_name;
           proxy_set_header X-Real-IP $remote_addr;
           proxy_set_header X-Forwarded-Proto $scheme;
-          {% if etesync__http_psk_subpath | d() %}
+          {% if etesync__http_psk_subpath %}
           proxy_set_header SCRIPT_NAME /{{ etesync__http_psk_subpath }};
           {% endif %}
           proxy_connect_timeout {{ etesync__app_timeout }};
           proxy_send_timeout {{ etesync__app_timeout }};
           proxy_read_timeout {{ etesync__app_timeout }};
 
       - pattern: '{{ (("/" + etesync__http_psk_subpath)
-                      if (etesync__http_psk_subpath | d())
+                      if (etesync__http_psk_subpath_enabled | bool)
                       else "") + "/admin" }}'
         options: |-
           proxy_pass http://etesync;
           proxy_set_header X-Forwarded-Host $server_name;
           proxy_set_header X-Real-IP $remote_addr;
           proxy_set_header X-Forwarded-Proto $scheme;
-          {% if etesync__http_psk_subpath | d() %}
+          {% if etesync__http_psk_subpath_enabled | bool %}
           proxy_set_header SCRIPT_NAME /{{ etesync__http_psk_subpath }};
           {% endif %}
           proxy_connect_timeout {{ etesync__app_timeout }};
@@ -470,7 +470,7 @@ etesync__nginx__dependent_servers:
 
           auth_basic "{{ etesync__admin_auth_basic_realm }}";
           auth_basic_user_file {{ etesync__admin_auth_basic_filename }};
-        enabled: '{{ True if etesync__admin_auth_basic_filename | d() else False }}'
+        enabled: '{{ True if (etesync__admin_auth_basic_filename != "") else False }}'
 # ]]]
 # ]]]
 # ]]]
diff --git a/ansible/roles/homeassistant/defaults/main.yml b/ansible/roles/homeassistant/defaults/main.yml
@@ -23,10 +23,6 @@
 #
 # List of base packages to install.
 homeassistant__base_packages:
-  - 'python3'
-  - 'python3-dev'
-  - 'python3-venv'
-  - 'python3-pip'
   - 'libffi-dev'
   - 'libssl-dev'
   - 'libjpeg-dev'
@@ -51,6 +47,10 @@ homeassistant__packages: []
 # List of Home Assistant Core dependency packages.
 # This refers to Debian system packages and not Python packages.
 homeassistant__dependency_python_packages:
+  - 'python3-dev'
+  - 'python3-venv'
+  - 'python3-pip'
+  - 'python3-virtualenv'
   - 'python3-requests'
   - 'python3-yaml'
   - 'python3-tz'
@@ -346,6 +346,34 @@ homeassistant__nginx__dependent_servers:
               autoindex off;
               alias {{ homeassistant__home_path }}/www/;
       }
+      location /local_brands/ {
+              rewrite /local_brands/(.*) /$1  break;
+
+              proxy_pass https://brands.home-assistant.io/;
+
+              proxy_set_header Host brands.home-assistant.io;
+              proxy_ssl_verify on;
+              proxy_ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt;
+              proxy_ssl_server_name on;
+
+              # Aggressively cache files on disk.
+              proxy_buffering on;
+              proxy_cache brands;
+              # Exclude $scheme$proxy_host from key to ease testing with different proxy_pass.
+              proxy_cache_key $request_uri;
+              proxy_cache_lock on;
+
+              # This might end up serving place holder files for 6 months.
+              # Ref: https://github.com/home-assistant/brands#caching
+              proxy_cache_valid 6M;
+
+              proxy_cache_valid 404 60m;
+              proxy_method GET;
+              proxy_pass_request_body off;
+              proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504;
+              proxy_cache_bypass $http_pragma;
+              add_header X-Cache-Status $upstream_cache_status;
+      }
     proxy_pass: 'http://homeassistant'
     proxy_options: |
       proxy_redirect http:// https://;

diff --git a/ansible/roles/homeassistant/tasks/main.yml b/ansible/roles/homeassistant/tasks/main.yml
@@ -69,6 +69,20 @@
   when: homeassistant__virtualenv | bool and homeassistant__register_git is changed
   notify: [ 'Restart Home Assistant' ]
 
+  # https://community.home-assistant.io/t/wth-why-are-brand-icons-a-cloud-dependency/226359/13
+  # Does not work. Home assistant still requests the files from https://brands.home-assistant.io/ …
+# - name: Change hardcoded brands.home-assistant.io domain and point to Nginx
+#   ansible.builtin.shell: |
+#     set -o pipefail -o errexit;
+#     for i in $(find lib/python*/site-packages/hass_frontend/ lib/python*/site-packages/homeassistant/components/ -iname '*.js.gz' -type f); do zcat "$i" | sed --regexp-extended 's#https://brands\.home-assistant\.io/#https://{{ homeassistant__fqdn }}/local_brands/brands/#g;s#brands\.home-assistant\.io#{{ homeassistant__fqdn }}#g;' > "$i.copy"; gzip "$i.copy"; mv "$i.copy.gz" "$i" ; done
+#     find lib/python*/site-packages/hass_frontend/ lib/python*/site-packages/homeassistant/components/ -not -iname '*.gz' -type f -print0 | xargs --null sed --in-place --regexp-extended 's#https://brands\.home-assistant\.io/#https://{{ homeassistant__fqdn }}/local_brands/brands/#g;s#brands\.home-assistant\.io#{{ homeassistant__fqdn }}#g;'
+#   args:
+#     chdir: '{{ homeassistant__virtualenv_path }}'
+#     executable: 'bash'
+#   become: True
+#   become_user: '{{ homeassistant__user }}'
+#   when: homeassistant__register_git is changed
+
 - name: Ensure Home Assistant config dir exists
   ansible.builtin.file:
     path: '{{ homeassistant__home_path }}/.homeassistant'

diff --git a/ansible/roles/homeassistant/tasks/main_env.yml b/ansible/roles/homeassistant/tasks/main_env.yml
@@ -0,0 +1,24 @@
+---
+# vim: foldmarker=[[[,]]]:foldmethod=marker
+
+# Copyright (C) 2022 Robin Schneider <ypid@riseup.net>
+# Copyright (C) 2022 DebOps <https://debops.org/>
+# SPDX-License-Identifier: GPL-3.0-only
+
+# - name: Ensure Nginx cache dir exists
+#   ansible.builtin.file:
+#     path: '/var/cache/nginx/proxy'
+#     mode: '0700'
+#     state: 'directory'
+#     owner: 'www-data'
+#     group: 'root'
+#   when: (homeassistant__deploy_state == "present")
+
+# - name: Configure Nginx proxy_cache_path
+#   ansible.builtin.template:
+#     src: 'etc/nginx/conf.d/debops.homeassistant.conf.j2'
+#     dest: '/etc/nginx/conf.d/debops.homeassistant.conf'
+#     owner: 'root'
+#     group: 'root'
+#     mode: '0644'
+#   when: (homeassistant__deploy_state == "present")
diff --git a/ansible/roles/homeassistant/templates/etc/nginx/conf.d/debops.homeassistant.conf.j2 b/ansible/roles/homeassistant/templates/etc/nginx/conf.d/debops.homeassistant.conf.j2
@@ -0,0 +1,7 @@
+{# Copyright (C) 2022 Robin Schneider <ypid@riseup.net>
+ # Copyright (C) 2022 DebOps <https://debops.org/>
+ # SPDX-License-Identifier: GPL-3.0-only
+ #}
+# {{ ansible_managed }}
+
+proxy_cache_path /var/cache/nginx/proxy levels=1:2 keys_zone=brands:10m use_temp_path=off inactive=5y max_size=5g;
diff --git a/ansible/roles/nginx/templates/etc/nginx/conf.d/upstream_default.conf.j2 b/ansible/roles/nginx/templates/etc/nginx/conf.d/upstream_default.conf.j2
@@ -87,7 +87,7 @@ upstream {{ item.name }} {
         server 127.0.0.1{% if item.port is defined and item.port %}:{{ item.port }}{% endif %}{% if item.enabled is defined and not item.enabled %} down{% endif %};
 {%             else                                                               %}
 {%                 if host in play_hosts and hostvars[host].ansible_all_ipv4_addresses is defined %}
-{%                     for address in hostvars[host].ansible_all_ipv4_addresses | unique %}
+{%                     for address in hostvars[host].ansible_all_ipv4_addresses | unique | sort %}
         server {{ address }}{% if item.port is defined and item.port %}:{{ item.port }}{% endif %}{% if item.enabled is defined and not item.enabled %} down{% endif %};
 {%                     endfor                                                     %}
 {%                 else                                                           %}

diff --git a/ansible/roles/nginx/templates/etc/nginx/conf.d/upstream_rails.conf.j2 b/ansible/roles/nginx/templates/etc/nginx/conf.d/upstream_rails.conf.j2
@@ -39,7 +39,7 @@ upstream {{ item.name }} {
         server 127.0.0.1{% if item.port is defined and item.port %}:{{ item.port }}{% endif %}{% if item.enabled is defined and not item.enabled %} down{% endif %};
 {%             else                                                               %}
 {%                 if host in play_hosts and hostvars[host].ansible_all_ipv4_addresses is defined %}
-{%                     for address in hostvars[host].ansible_all_ipv4_addresses | unique %}
+{%                     for address in hostvars[host].ansible_all_ipv4_addresses | unique | sort %}
         server {{ address }}{% if item.port is defined and item.port %}:{{ item.port }}{% endif %}{% if item.enabled is defined and not item.enabled %} down{% endif %};
 {%                     endfor                                                     %}
 {%                 else                                                           %}

diff --git a/ansible/roles/nginx/templates/etc/nginx/sites-available/default.conf.j2 b/ansible/roles/nginx/templates/etc/nginx/sites-available/default.conf.j2
@@ -128,7 +128,7 @@
         location {{ path }} {
 {%                 if location_referers is defined                             %}
 {%                     if location_referers[path] is defined and location_referers[path] %}
-                valid_referers none blocked {{ location_referers[path] | unique | join(' ') }};
+                valid_referers none blocked {{ location_referers[path] | unique | sort | join(' ') }};
                 if ($invalid_referer) {
                         return 403;
                 }
@@ -137,12 +137,12 @@
                 {{ item.location[path] | indent(16) | regex_replace("(?m)^\s*$", "") }}
 {%                 if location_allow is defined                                %}
 {%                     if location_allow[path] is defined and location_allow[path] %}
-{%                         for address in location_allow[path] | unique        %}
+{%                         for address in location_allow[path] | unique | sort %}
                 allow {{ address }};
 {%                         endfor                                              %}
 {%                         if location_deny is defined                         %}
 {%                             if location_deny[path] is defined               %}
-{%                                 for address in location_deny[path] | unique %}
+{%                                 for address in location_deny[path] | unique | sort %}
                 deny {{ address }};
 {%                                 endfor                                      %}
 {%                             endif                                           %}
@@ -191,13 +191,13 @@
 {%                         if entry.allow is string                %}
                 allow {{ entry.allow }};
 {%                         else                                   %}
-{%                             for address in entry.allow | unique %}
+{%                             for address in entry.allow | unique | sort %}
                 allow {{ address }};
 {%                             endfor                             %}
 {%                         endif                                  %}
 {%                     endif                                      %}
 {%                     if entry.access_policy | d() and (entry.access_policy in nginx_access_policy_allow_map) %}
-{%                         for address in nginx_access_policy_allow_map[entry.access_policy] | unique       %}
+{%                         for address in nginx_access_policy_allow_map[entry.access_policy] | unique | sort %}
                 allow {{ address }};
 {%                         endfor                                                                          %}
 {%                     endif                                                                               %}
@@ -357,13 +357,13 @@
 {%             if item.allow is string                %}
         allow {{ item.allow }};
 {%             else                                   %}
-{%                 for address in item.allow | unique %}
+{%                 for address in item.allow | unique | sort %}
         allow {{ address }};
 {%                 endfor                             %}
 {%             endif                                  %}
 {%         endif                                      %}
 {%         if item.access_policy | d() and (item.access_policy in nginx_access_policy_allow_map) %}
-{%             for address in nginx_access_policy_allow_map[item.access_policy] | unique       %}
+{%             for address in nginx_access_policy_allow_map[item.access_policy] | unique | sort  %}
         allow {{ address }};
 {%             endfor                                                                          %}
 {%         endif                                                                               %}
@@ -413,18 +413,18 @@
         location = {{ item.status_name | default(nginx_status_name) }} {
                 stub_status on;
                 access_log off;
-{%         if nginx_status_localhost                          %}
-{%             for address in nginx_status_localhost | unique %}
+{%         if nginx_status_localhost                                 %}
+{%             for address in nginx_status_localhost | unique | sort %}
                 allow {{ address }};
 {%             endfor                                         %}
 {%         endif                                              %}
 {%         if nginx_status                                    %}
-{%             for address in nginx_status | unique           %}
+{%             for address in nginx_status | unique | sort    %}
                 allow {{ address }};
 {%             endfor                                         %}
 {%         endif                                              %}
-{%         if item.status | d()                                 %}
-{%             for address in item.status | unique            %}
+{%         if item.status | d()                               %}
+{%             for address in item.status | unique | sort     %}
                 allow {{ address }};
 {%             endfor                                         %}
 {%         endif                                              %}
@@ -527,7 +527,7 @@ server {
         listen {{ port }};
 {%       endfor                                        %}
 
-{%       for address in nginx__tpl_hostnames | unique %}
+{%       for address in nginx__tpl_hostnames | unique | sort %}
         server_name {{ address }};
 {%       endfor                                        %}
 
@@ -543,7 +543,7 @@ server {
         listen {{ port }};
 {%       endfor                                        %}
 
-{%       for address in nginx__tpl_hostnames | unique %}
+{%       for address in nginx__tpl_hostnames | unique | sort %}
         server_name {{ address + '.lxc' }};
 {%       endfor                                        %}
 
@@ -559,7 +559,7 @@ server {
         listen {{ port }};
 {%       endfor                                        %}
 
-{%       for address in nginx__tpl_hostnames | unique %}
+{%       for address in nginx__tpl_hostnames | unique | sort %}
         server_name {{ address }}.local;
 {%       endfor                                        %}
 

diff --git a/ansible/roles/nginx/templates/etc/nginx/sites-available/php.conf.j2 b/ansible/roles/nginx/templates/etc/nginx/sites-available/php.conf.j2
@@ -24,12 +24,12 @@
                 access_log off;
 {%     set nginx__tpl_status_allow = [] %}
 {%     if nginx_status_localhost %}
-{%       for address in nginx_status_localhost | unique %}
+{%       for address in nginx_status_localhost | unique | sort %}
 {%         set _ = nginx__tpl_status_allow.append(address) %}
 {%       endfor %}
 {%     endif %}
 {%     if nginx_status %}
-{%       for address in nginx_status | unique %}
+{%       for address in nginx_status | unique | sort %}
 {%         set _ = nginx__tpl_status_allow.append(address) %}
 {%       endfor %}
 {%     endif %}
@@ -38,7 +38,7 @@
 {%          set _ = nginx__tpl_status_allow.append(address) %}
 {%       endfor %}
 {%     endif %}
-{%     for address in nginx__tpl_status_allow | sort | unique %}
+{%     for address in nginx__tpl_status_allow | unique | sort %}
                 allow {{ address }};
 {%     endfor %}
                 deny all;

diff --git a/ansible/roles/nginx/templates/etc/nginx/sites-available/php5.conf.j2 b/ansible/roles/nginx/templates/etc/nginx/sites-available/php5.conf.j2
@@ -75,17 +75,17 @@
         location ~ ^/({{ item.php5_status_name | default(nginx_php5_status_name) }}|{{ item.php5_ping_name | default(nginx_php5_ping_name) }})$ {
                 access_log off;
 {% if nginx_status_localhost %}
-{% for address in nginx_status_localhost | unique %}
+{% for address in nginx_status_localhost | unique | sort %}
                 allow {{ address }};
 {% endfor %}
 {% endif %}
 {% if nginx_status %}
-{% for address in nginx_status | unique %}
+{% for address in nginx_status | unique | sort %}
                 allow {{ address }};
 {% endfor %}
 {% endif %}
 {% if item.php5_status is defined and item.php5_status %}
-{% for address in item.php5_status | unique %}
+{% for address in item.php5_status | unique | sort %}
                 allow {{ address }};
 {% endfor %}
 {% endif %}

diff --git a/ansible/roles/opensearch/defaults/main.yml b/ansible/roles/opensearch/defaults/main.yml
@@ -126,7 +126,7 @@ opensearch__default_configuration:
     value: True
 
   - name: 'bootstrap.memory_lock'
-    comment: '# Lock the memory on startup'
+    comment: 'Lock the memory on startup'
     value: '{{ True if opensearch__memory_lock | bool else False }}'
 
   - name: 'cluster.initial_master_nodes'

diff --git a/ansible/roles/owncloud/defaults/main.yml b/ansible/roles/owncloud/defaults/main.yml
@@ -1761,7 +1761,7 @@ owncloud__theme_conf_map:
 # .. warning:: This breaks the ``/.well-known`` URIs intentionally.
 #
 # Hint for migration: Nextcloud is able to auto detect the webroot that it is
-# being access. This means that it can be accessed by subpath and without, with
+# being accessed. This means that it can be accessed by subpath and without, with
 # the same Nginx config. This allows soft migration of all users to the subpath
 # and then disabling ``/`` when all are migrated.
 # The role does not support generating such a Webserver config but it is easy