Convert local dev to Docker (#245)

* docker local dev: initial stab at dockerifying our salt local dev setup

* docker local dev: adjust so red starts to go away

* docker local dev: silence some warnings

* docker local dev: set the timezone

* docker local dev: fix locale issues

* docker local dev: fixup some docker flags and run from amd so we can get salt debs

* docker local dev: add cron

* docker local dev: add rsyslog for consul

* docker local dev: add system openssl python package for salt-master ca

* docker local dev: use pregenerated dhparams to avoid slow provisioning

* docker local dev: make the salt-master a consul server

this *mostly* removes a chicken-n-egg issue with salt needing consul and consul needing salt

there may be state ordering issues and requisites to get a clean vagrant up for salt master still

* docker local dev: install salt-minion in docker container

significantly speeds up vagrant provision for subsequent hosts

* docker local dev: update datadoghq apt signing key

* docker local dev: disable iptables/netfilter/etc in docker

Dockerfile

@@ -0,0 +1,53 @@
+# Docker image to use with Vagrant
+# Aims to be as similar to normal Vagrant usage as possible
+# Adds SSH daemon, Systemd
+# Adapted from https://github.com/BashtonLtd/docker-vagrant-images/blob/master/ubuntu1404/Dockerfile
+
+FROM ubuntu:18.04
+ENV container docker
+
+RUN ln -snf /usr/share/zoneinfo/UTC /etc/localtime && echo $TZ > /etc/timezone
+
+RUN apt-get update -y && apt-get dist-upgrade -y
+
+# Install system dependencies, you may not need all of these
+RUN apt-get install -y --no-install-recommends ssh sudo libffi-dev systemd openssh-client wget gnupg-utils gnupg apt-utils ca-certificates dbus locales cron dialog rsyslog
+
+RUN locale-gen en_US.UTF-8
+COPY ./docker/etc/locale.conf /etc/locale.conf
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
+COPY ./docker/etc/ssl/private/dhparams.pem /etc/ssl/private/dhparams.pem
+
+# Needed to run systemd
+# VOLUME [ "/sys/fs/cgroup" ]
+# Doesn't appear to be necessary? See comments
+
+# Add vagrant user and key for SSH
+RUN useradd --create-home -s /bin/bash vagrant
+RUN echo -n 'vagrant:vagrant' | chpasswd
+RUN echo 'vagrant ALL = NOPASSWD: ALL' > /etc/sudoers.d/vagrant
+RUN chmod 440 /etc/sudoers.d/vagrant
+RUN mkdir -p /home/vagrant/.ssh
+RUN chmod 700 /home/vagrant/.ssh
+RUN echo "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ==" > /home/vagrant/.ssh/authorized_keys
+RUN chmod 600 /home/vagrant/.ssh/authorized_keys
+RUN chown -R vagrant:vagrant /home/vagrant/.ssh
+RUN sed -i -e 's/Defaults.*requiretty/#&/' /etc/sudoers
+RUN sed -i -e 's/\(UsePAM \)yes/\1 no/' /etc/ssh/sshd_config
+
+# Start SSH
+RUN mkdir /var/run/sshd
+EXPOSE 22
+RUN /usr/sbin/sshd
+
+# Setup Salt Common
+
+RUN wget --quiet -O - https://archive.repo.saltstack.com/py3/ubuntu/18.04/amd64/2018.3/SALTSTACK-GPG-KEY.pub | apt-key add -
+RUN echo 'deb http://archive.repo.saltstack.com/py3/ubuntu/18.04/amd64/2018.3 bionic main' > /etc/apt/sources.list.d/saltstack.list
+RUN apt-get update -y && apt-get install -y --no-install-recommends salt-minion
+
+# Start Systemd (systemctl)
+CMD ["/lib/systemd/systemd"]

Vagrantfile

@@ -26,30 +26,33 @@ MASTER2 = "#{SUBNET2}.2"
 
 
 Vagrant.configure("2") do |config|
-  config.vm.box = "ubuntu/bionic64"
+  config.vm.provider "vmware" do |config|
+    config.vm.box = "hashicorp/bionic64"
+  end
+
+  config.vm.provider "docker" do |docker, override|
+    override.vm.box = nil
+    override.ssh.insert_key = true
+
+    docker.build_dir = '.'
+    docker.build_args = ['--platform', 'linux/amd64']
+    docker.has_ssh = true
+    docker.remains_running = true
+    docker.privileged = true
+  end
 
   config.vm.define "salt-master" do |s_config|
     s_config.vm.hostname = "salt-master.vagrant.psf.io"
-    s_config.vm.network "private_network", ip: MASTER1, virtualbox__intnet: "psf1"
-    s_config.vm.network "private_network", ip: MASTER2, virtualbox__intnet: "psf2"
-
-    s_config.vm.provider "virtualbox" do |v|
-      v.memory = 1024
-      v.cpus = 2
-    end
+    s_config.vm.network "private_network", ip: MASTER1
+    s_config.vm.network "private_network", ip: MASTER2
 
     s_config.vm.synced_folder "salt/", "/srv/salt/"
     s_config.vm.synced_folder "pillar/", "/srv/pillar/"
 
     # Provision the salt-master.
-    s_config.vm.provision :shell, :inline => <<-HEREDOC
-      wget -O - https://repo.saltstack.com/py3/ubuntu/18.04/amd64/2018.3/SALTSTACK-GPG-KEY.pub | apt-key add -
-      echo 'deb http://repo.saltstack.com/py3/ubuntu/18.04/amd64/2018.3 bionic main' > /etc/apt/sources.list.d/saltstack.list
-    HEREDOC
-
     s_config.vm.provision :shell, :inline => <<-HEREDOC
       apt-get update
-      apt-get install -y salt-master
+      apt-get install -y salt-master python3-openssl
       ln -sf /vagrant/conf/vagrant/master.conf /etc/salt/master.d/local.conf
     HEREDOC
 
@@ -62,6 +65,7 @@ Vagrant.configure("2") do |config|
       echo 'master: #{MASTER1}\n' > /etc/salt/minion.d/local.conf
       service salt-minion restart
       salt-call state.highstate
+      sudo salt '*' saltutil.refresh_pillar
     HEREDOC
 
     # Run this always, because we need to sync our states.
@@ -89,21 +93,14 @@ Vagrant.configure("2") do |config|
       end
 
       s_config.vm.hostname = "#{server}.vagrant.psf.io"
-      s_config.vm.network "private_network", ip: "#{SUBNET1}.#{num + 10}", virtualbox__intnet: "psf1"
-      s_config.vm.network "private_network", ip: "#{SUBNET2}.#{num + 10}", virtualbox__intnet: "psf2"
+      s_config.vm.network "private_network", ip: "#{SUBNET1}.#{num + 10}"
+      s_config.vm.network "private_network", ip: "#{SUBNET2}.#{num + 10}"
 
       ports.each do |port|
         s_config.vm.network "forwarded_port", guest: port, host: port
       end
 
       # Provision the salt-minion
-      if codename == "bionic"
-        s_config.vm.provision :shell, :inline => <<-HEREDOC
-          wget -O - https://repo.saltstack.com/py3/ubuntu/18.04/amd64/2018.3/SALTSTACK-GPG-KEY.pub | apt-key add -
-          echo 'deb http://repo.saltstack.com/py3/ubuntu/18.04/amd64/2018.3 bionic main' > /etc/apt/sources.list.d/saltstack.list
-        HEREDOC
-      end
-
       s_config.vm.provision :shell, :inline => <<-HEREDOC
         apt-get update
         apt-get install -y salt-minion

docker/etc/locale.conf

@@ -0,0 +1 @@
+LANG=en_US.UTF-8

docker/etc/ssl/private/dhparams.pem

@@ -0,0 +1,13 @@
+-----BEGIN DH PARAMETERS-----
+MIICCAKCAgEA2/pV4Uvvm/c4DgpcTDg/jO0ndhXkUxwwfeXMxMd/ilvJeRyvtGtS
+HnBmYzuGWtthaGV8t/pvUO3rpoUv3SpGvGe9WhxIFM1/Yqsik9sToP8og7o2t9Qz
+xB1RxqDMRlXgx1nywiDeMb4dj/Mlxw7iXNrb/K6keoFdoL/YYUtIxzxyyzwCAB7+
+hJVRuRN37U0Opab4kwTjv0+6b4ykvFU9HbPT37mML9Es1kOkkT7K4VUiVV5MFz+4
+VAi/2zc1demBAAbU5ZLG7bteFivW11gJm10jXN5gFnODC2vLbQfc84EFgtEuk9oZ
+KiwwFGwKk/9gm1bWICTUMv/0gYYgwf6YB53AZ05CMWbWxvBzMjQmVGoseQtSwkzQ
+PUcSsYZHCmwkbXf2XbVoQGmMgSJV7fyasftWU1Dmbj2AmsyeRf8VUlNA6Z9YghMj
+TrpP1+2uKPsarCLtau8RYIHlKFENbhY+RPZNf3ADoP0SP28Mcq44Byt05A9WajO5
+9UUUIEXTpppNqE0qF/ZD1xdaOzS/Rz5x/5Np0ZuOKwrR6qytTcOOG6LcTJJ2qp2G
+sudIOnKI7IieMmRmpyx3ED9Xx+/awQr4jHEIbGH8UirfrkOORo0R0IvQQQd2jGNK
+5AzrHjBnBWVmOvCOczwFsBWUDF8ul+/CQKt9xxzrYaemdUaFvADEFzsCAQI=
+-----END DH PARAMETERS-----

pillar/dev/consul.sls

@@ -1,14 +1,14 @@
 consul:
   bootstrap:
     vagrant:
-      - consul.vagrant.psf.io
+      - salt-master.vagrant.psf.io
   acl:
     default: deny
     dc: vagrant
     down: extend-cache
     ttl: 30s
   dcs:
-    vagrant: consul.vagrant.psf.io
+    vagrant: salt-master.vagrant.psf.io
   external:
     - datacenter: vagrant
       node: pythonanywhere

pillar/dev/roles.sls

@@ -27,7 +27,7 @@ roles:
     purpose: ""
     contact: ""
   consul:
-    pattern: "consul.vagrant.psf.io"
+    pattern: "E@(consul|salt-master).vagrant.psf.io"
     purpose: ""
     contact: ""
   docs: