Upgrade lts to 20.04 (#249)

* make local dev compatible with arm64 for docker...

This dramatically improves provisioning time and general performance for local development.

As an added benefit, it also brings us up to Ubuntu 20.04 LTS and the latest Salt release (3004).

* wait for pillar refresh to complete

* make room for salt-2004.nyc1.psf.io

* fixup consul join template when not all nodes are online

* boto3 from distro

* fix for DNS entry creation

* update call signature for match.compound

* add new consul servers

* Update the state file for docs to accomodate ubuntu 20.04 (#248)

- stopped installing pythons from the distro, relying on deadsnakes ppa
- bootstrap pip via venv module instead of get-pip

Co-authored-by: Chloe Gerhardson <66563430+cegerhardson@users.noreply.github.com>

Dockerfile

@@ -3,15 +3,15 @@
 # Adds SSH daemon, Systemd
 # Adapted from https://github.com/BashtonLtd/docker-vagrant-images/blob/master/ubuntu1404/Dockerfile
 
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 ENV container docker
 
 RUN ln -snf /usr/share/zoneinfo/UTC /etc/localtime && echo $TZ > /etc/timezone
 
 RUN apt-get update -y && apt-get dist-upgrade -y
 
 # Install system dependencies, you may not need all of these
-RUN apt-get install -y --no-install-recommends ssh sudo libffi-dev systemd openssh-client wget gnupg-utils gnupg apt-utils ca-certificates dbus locales cron dialog rsyslog
+RUN apt-get install -y --no-install-recommends ssh sudo libffi-dev systemd openssh-client wget gnupg-utils gnupg apt-utils ca-certificates dbus locales cron dialog rsyslog iproute2
 
 RUN locale-gen en_US.UTF-8
 COPY ./docker/etc/locale.conf /etc/locale.conf
@@ -45,8 +45,8 @@ RUN /usr/sbin/sshd
 
 # Setup Salt Common
 
-RUN wget --quiet -O - https://archive.repo.saltstack.com/py3/ubuntu/18.04/amd64/2018.3/SALTSTACK-GPG-KEY.pub | apt-key add -
-RUN echo 'deb http://archive.repo.saltstack.com/py3/ubuntu/18.04/amd64/2018.3 bionic main' > /etc/apt/sources.list.d/saltstack.list
+RUN wget --quiet -O /usr/share/keyrings/salt-archive-keyring.gpg https://repo.saltproject.io/py3/ubuntu/20.04/arm64/3004/salt-archive-keyring.gpg
+RUN echo "deb [signed-by=/usr/share/keyrings/salt-archive-keyring.gpg arch=arm64] https://repo.saltproject.io/py3/ubuntu/20.04/arm64/3004 focal main" > /etc/apt/sources.list.d/salt.list
 RUN apt-get update -y && apt-get install -y --no-install-recommends salt-minion
 
 # Start Systemd (systemctl)

Vagrantfile

@@ -35,7 +35,7 @@ Vagrant.configure("2") do |config|
     override.ssh.insert_key = true
 
     docker.build_dir = '.'
-    docker.build_args = ['--platform', 'linux/amd64']
+    #docker.build_args = ['--platform', 'linux/amd64']
     docker.has_ssh = true
     docker.remains_running = true
     docker.privileged = true
@@ -65,7 +65,10 @@ Vagrant.configure("2") do |config|
       echo 'master: #{MASTER1}\n' > /etc/salt/minion.d/local.conf
       service salt-minion restart
       salt-call state.highstate
-      sudo salt '*' saltutil.refresh_pillar
+      while ! salt-call consul.cluster_ready | grep True; do echo 'waiting for consul'; sleep 1; done
+      salt '*' saltutil.sync_all
+      salt '*' saltutil.refresh_grains
+      salt '*' saltutil.refresh_pillar wait=True timeout=30
     HEREDOC
 
     # Run this always, because we need to sync our states.

pillar/dev/consul.sls

@@ -2,6 +2,8 @@ consul:
   bootstrap:
     vagrant:
       - salt-master.vagrant.psf.io
+      - consul.vagrant.psf.io
+      - none.vagrant.psf.io
   acl:
     default: deny
     dc: vagrant

pillar/prod/roles.sls

@@ -63,7 +63,7 @@ roles:
     purpose:  "Test resources for CPython's test suite."
     contact:  "Infrastructure Staff"
   salt-master: 
-    pattern:  "salt.nyc1.psf.io"
+    pattern:  "salt*.nyc1.psf.io"
     purpose:  "Salt server"
     contact:  "Infrastructure Staff"
   moin: 

salt/_extensions/pillar/ca.py

@@ -4,24 +4,17 @@
 import datetime
 import os.path
 
-import salt.minion
-
-from salt._compat import string_types
+import salt.loader
 
 import OpenSSL
 
 
 def compound(tgt, minion_id=None):
-    opts = {"grains": __grains__}
-    if minion_id is not None:
-        if not isinstance(minion_id, string_types):
-            minion_id = str(minion_id)
-        else:
-            minion_id = __grains__["id"]
-    opts["id"] = minion_id
-    matcher = salt.minion.Matcher(opts, __salt__)
+    opts = {'grains': __grains__}
+    opts['id'] = minion_id
+    matcher = salt.loader.matchers(dict(__opts__, **opts))['compound_match.match']
     try:
-        return matcher.compound_match(tgt)
+        return matcher(tgt)
     except Exception:
         pass
     return False

salt/_extensions/pillar/consul.py

@@ -109,7 +109,7 @@ def ext_pillar(minion_id, pillar, key_path, acl_path):
     master_acl_token = _gen_master_acl("__master__", acl_path)
 
     # If this is a server in the ACL data center, give it the acl master token
-    is_server = __salt__["match.compound"](pillar["roles"]["consul"]["pattern"])
+    is_server = __salt__["match.compound"](pillar["roles"]["consul"]["pattern"], minion_id=minion_id)
     in_acl_dc = bool(pillar["dc"] == pillar["consul"]["acl"]["dc"])
     if is_server and in_acl_dc:
         data["consul"]["acl"]["tokens"]["__master__"] = master_acl_token

salt/_extensions/pillar/dc.py

@@ -1,19 +1,11 @@
-import salt.minion
-
-from salt._compat import string_types
-
+import salt.loader
 
 def compound(tgt, minion_id=None):
     opts = {'grains': __grains__}
-    if minion_id is not None:
-        if not isinstance(minion_id, string_types):
-            minion_id = str(minion_id)
-        else:
-            minion_id = __grains__['id']
     opts['id'] = minion_id
-    matcher = salt.minion.Matcher(opts, __salt__)
+    matcher = salt.loader.matchers(dict(__opts__, **opts))['compound_match.match']
     try:
-        return matcher.compound_match(tgt)
+        return matcher(tgt)
     except Exception:
         pass
     return False

salt/base/config/sources.list.jinja

@@ -1,9 +1,15 @@
+{% if grains["osarch"] == "arm64" %}
+{% set base_repo = "ports.ubuntu.com/ubuntu-ports/" %}
+{% else %}
+{% set base_repo = "us.archive.ubuntu.com/ubuntu/" %}
+{% endif %}
+
 ###### Ubuntu Main Repos
-deb http://us.archive.ubuntu.com/ubuntu/ {{ grains["oscodename"] }} main restricted universe
-deb-src http://us.archive.ubuntu.com/ubuntu/ {{ grains["oscodename"] }} main restricted universe
+deb http://{{ base_repo }} {{ grains["oscodename"] }} main restricted universe
+deb-src http://{{ base_repo }} {{ grains["oscodename"] }} main restricted universe
 
 ###### Ubuntu Update Repos
-deb http://us.archive.ubuntu.com/ubuntu/ {{ grains["oscodename"] }}-security main restricted universe
-deb http://us.archive.ubuntu.com/ubuntu/ {{ grains["oscodename"] }}-updates main restricted universe
-deb-src http://us.archive.ubuntu.com/ubuntu/ {{ grains["oscodename"] }}-security main restricted universe
-deb-src http://us.archive.ubuntu.com/ubuntu/ {{ grains["oscodename"] }}-updates main restricted universe
+deb http://{{ base_repo }} {{ grains["oscodename"] }}-security main restricted universe
+deb http://{{ base_repo }} {{ grains["oscodename"] }}-updates main restricted universe
+deb-src http://{{ base_repo }} {{ grains["oscodename"] }}-security main restricted universe
+deb-src http://{{ base_repo }} {{ grains["oscodename"] }}-updates main restricted universe

salt/base/salt.sls

@@ -22,8 +22,8 @@ salt-2018.3:
   pkgrepo.managed:
     - humanname: repo.saltstack.org
     {% if grains["oscodename"] == "focal" %}
-    - name: deb [arch=amd64] https://archive.repo.saltproject.io/py3/ubuntu/20.04/amd64/archive/3003.3 focal main
-    - key_url: https://archive.repo.saltproject.io/py3/ubuntu/20.04/amd64/latest/salt-archive-keyring.gpg
+    - name: deb https://archive.repo.saltproject.io/py3/ubuntu/20.04/{{ grains["osarch"] }}/archive/3004 focal main
+    - key_url: https://archive.repo.saltproject.io/py3/ubuntu/20.04/{{ grains["osarch"] }}/archive/3004/salt-archive-keyring.gpg
     {% else %}
     - name: deb http://archive.repo.saltstack.com/py3/ubuntu/{{ grains["osrelease"] }}/{{ grains["osarch"] }}/2018.3 {{ grains["oscodename"] }} main
     - key_url: https://archive.repo.saltstack.com/py3/ubuntu/18.04/amd64/2018.3/SALTSTACK-GPG-KEY.pub

salt/dns/init.sls

@@ -1,16 +1,19 @@
-boto3:
-  pip.installed
+boto-pkgs:
+  pkg.installed:
+    - python3-boto
+    - python3-boto3
 
 {% set public_ipv4 = salt["mine.get"]("*", "public_ipv4") %}
 
 # We assume that a server will always have an IPv4 address.
 
+# TODO: Update back to boto3_route53 when https://github.com/saltstack/salt/pull/60951 makes it into a release
 {% for server in public_ipv4 %}
 {{ server }}-route53:
-  boto3_route53.rr_present:
-    - DomainName: psf.io.
-    - Name: {{ server }}.
-    - TTL: 3600
-    - Type: A
-    - ResourceRecords: {{ public_ipv4.get(server, []) }}
+  boto_route53.rr_present:
+    - zone: psf.io.
+    - name: {{ server }}.
+    - ttl: 3600
+    - record_type: A
+    - value: {{ public_ipv4.get(server, []) }}
 {% endfor %}

salt/docs/init.sls

@@ -13,9 +13,6 @@ doc-pkgs:
       - fonts-freefont-otf
       - git
       - mercurial
-      - python-dev
-      - python-virtualenv
-      - python3-venv
       - python3.10-dev
       - python3.10-venv
       - latexmk
@@ -50,34 +47,21 @@ docsbuild-scripts:
 virtualenv:
   cmd.run:
     - runas: docsbuild
-    - name: 'python3.10 -m venv --without-pip /srv/docsbuild/venv'
-    - creates: /srv/docsbuild/venv/bin/python
+    - name: 'python3.10 -m venv /srv/docsbuild/venv'
+    - creates:
+      - /srv/docsbuild/venv/bin/python
+      - /srv/docsbuild/venv/bin/pip
     - require:
       - pkg: doc-pkgs
 
-/srv/docsbuild/venv/get-pip.py:
-  file.managed:
-    - user: docsbuild
-    - source: https://bootstrap.pypa.io/get-pip.py
-    - source_hash: sha256=d1563edc7e23c98ac4f82d354606d5205d09ce4cc0f971edc80daa7978762d90
-    - require:
-      - cmd: virtualenv
-
-virtualenv-pip:
-  cmd.run:
-    - runas: docsbuild
-    - name: /srv/docsbuild/venv/bin/python /srv/docsbuild/venv/get-pip.py
-    - creates: /srv/docsbuild/venv/bin/pip
-    - require:
-      - file: /srv/docsbuild/venv/get-pip.py
-
 virtualenv-dependencies:
   cmd.run:
     - runas: docsbuild
     - cwd: /srv/docsbuild/scripts
     - name: /srv/docsbuild/venv/bin/pip install -r /srv/docsbuild/scripts/requirements.txt
     - require:
-      - cmd: virtualenv-pip
+      - git: docsbuild-scripts
+      - cmd: virtualenv
     - onchanges:
       - git: docsbuild-scripts