Add: Basic ansible playbook for configuring new caching content servers

ansible/README.md

@@ -0,0 +1,14 @@
+# Extremely basic instructions for using ansible
+
+Requirements:
+* Public ssh keys to authorized_keys of the debian user on all content servers
+
+Then run:
+
+```
+ansible-galaxy install -r requirements.yml
+ansible-playbook --user debian --inventory inventory --diff content-servers.yml
+```
+
+Due to a quirk of configuration, the staging hosts happen to be the same hosts as the production hosts
+If this ever changes, remove the "staging" specific stuff from the playbook and add the staging hosts (and variables) to the inventory.

ansible/ansible.cfg

@@ -0,0 +1,2 @@
+[defaults]
+interpreter_python=/usr/bin/python3

ansible/content-servers.yml

@@ -0,0 +1,170 @@
+- hosts: bananas_fileservers
+  gather_facts: true
+  become: true
+
+  pre_tasks:
+    # Unlikely to be necessary after upgrading to bullseye (needed to pull in python3-firewall)
+    - name: Add backports repo
+      apt_repository:
+        repo: deb http://deb.debian.org/debian {{ ansible_distribution_release }}-backports main
+        state: present
+        filename: backports
+
+    - name: Update apt
+      apt:
+        update_cache: true
+        cache_valid_time: 86400  # a day is plenty
+      changed_when: false
+
+    - name: Upgrade & cleanup apt
+      apt:
+        upgrade: safe
+        autoremove: true
+        autoclean: true
+
+    - name: Install apt https support & cron
+      apt:
+        name:
+          - apt-transport-https
+          - cron  # required for certbot
+        state: present
+
+    - name: Install firewalld & python bindings
+      apt:
+        name:  # required for ansible.posix.firewalld
+          - firewalld
+          - python3-firewall
+          - iptables  # Newer version fixes a bug in buster iptables
+        default_release: "{{ ansible_distribution_release }}-backports"
+        state: present
+
+  roles:
+    - geerlingguy.certbot
+    - geerlingguy.nginx
+
+  tasks:
+    - name: Flush handlers in case any configs have changed
+      meta: flush_handlers
+
+    - name: Add users' ssh keys to the current account
+      authorized_key:
+        user: "{{ ansible_user }}"
+        key: "{{ item }}"
+      with_items: "{{ keys }}"
+
+    - name: Install some helpful utilities
+      apt:
+        name:
+          - bash-completion
+          - logrotate
+          - molly-guard
+          - rsync
+          - sshguard
+          - unattended-upgrades
+          - vim
+        state: present
+
+    - name: Enable persistent systemd journal
+      lineinfile:
+        path: /etc/systemd/journald.conf
+        regexp: "^Storage="
+        line: "Storage=persistent"
+      notify:
+        restart journal
+
+    - name: Copy sshd config
+      copy:
+        src: files/sshd_config
+        dest: /etc/ssh/sshd_config
+      notify:
+        restart ssh
+
+    ######
+    # Firewall stuff
+    ######
+
+    - name: Configure firewalld.
+      lineinfile:
+        path: /etc/firewalld/firewalld.conf
+        regexp: "^FirewallBackend="
+        line: "FirewallBackend=nftables"
+      notify:
+        - restart firewalld
+
+    - name: Configure firewall rules
+      ansible.posix.firewalld:
+        permanent: true
+        state: enabled
+        port: "{{ item }}"
+      with_items:
+        - "22/tcp"
+        - "80/tcp"
+        - "443/tcp"
+        - "67/udp"   # dhcp
+        - "68/udp"   # dhcp
+        - "123/udp"  # ntp
+      notify:
+        - restart firewalld
+
+    - name: Create sshguard blacklist db directory
+      file:
+        path: /var/db/sshguard
+        state: directory
+
+    # Use firewalld backend
+    # Workaround https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=928525
+    # Set persistent blacklist
+    - name: Configure sshguard
+      lineinfile:
+        path: /etc/sshguard/sshguard.conf
+        regexp: "{{ item.regexp }}"
+        line: "{{ item.line }}"
+      with_items:
+        - regexp: "^BACKEND="
+          line: 'BACKEND="/usr/lib/x86_64-linux-gnu/sshg-fw-firewalld"'
+        - regexp: "^LOGREADER="
+          line: 'LOGREADER="LANG=C /bin/journalctl -afb -p info -t sshd -n1 -o cat"'
+        - regexp: "^BLACKLIST_FILE="
+          line: 'BLACKLIST_FILE=100:/var/db/sshguard/blacklist.db'
+      notify:
+        - restart sshguard
+
+    - name: Flush handlers in case any configs have changed
+      meta: flush_handlers
+
+    # Put these last (and after a flush_handlers) so that pending firewall changes have been applied
+    - name: Test HTTPS to health check endpoint
+      uri:
+        url: https://{{ inventory_hostname }}/healthz
+        status_code: 200
+      delegate_to: localhost
+      become: false
+
+    # Note that, in effect, this tests against 'localhost', but also tests connecting to upstream
+    # Doesn't quite test actual external access, but it's much faster than waiting for my slow internet connection
+    - name: Test download of particular item (OGFX)
+      uri:
+        url: "{{ item }}://{{ inventory_hostname }}/base-graphics/4f474658/99ef7df70a3fe95f0f9da6dcb5e63444/FOR-TESTING-ONLY.tar.gz"
+        status_code: 200
+      with_items:
+        - http
+        - https
+      become: false
+
+  handlers:
+    - name: restart ssh
+      service:
+        name: ssh
+        state: restarted
+    - name: restart sshguard
+      service:
+        name: sshguard
+        state: restarted
+    - name: restart journal
+      service:
+        name: systemd-journald
+        state: restarted
+    - name: restart firewalld
+      service:
+        name: firewalld
+        state: restarted

ansible/files/sshd_config

@@ -0,0 +1,29 @@
+# Copied from https://infosec.mozilla.org/guidelines/openssh.html
+
+# Supported HostKey algorithms by order of preference.
+HostKey /etc/ssh/ssh_host_ed25519_key
+HostKey /etc/ssh/ssh_host_rsa_key
+HostKey /etc/ssh/ssh_host_ecdsa_key
+
+KexAlgorithms curve25519-sha256@libssh.org,ecdh-sha2-nistp521,ecdh-sha2-nistp384,ecdh-sha2-nistp256,diffie-hellman-group-exchange-sha256
+
+Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr
+
+MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-512,hmac-sha2-256,umac-128@openssh.com
+
+# Password based logins are disabled - only public key based logins are allowed.
+AuthenticationMethods publickey
+
+# LogLevel VERBOSE logs user's key fingerprint on login. Needed to have a clear audit track of which key was using to log in.
+LogLevel VERBOSE
+
+# Log sftp level file access (read/write/etc.) that would not be easily logged otherwise.
+Subsystem sftp internal-sftp -f AUTHPRIV -l INFO
+
+# Root login is not allowed for auditing reasons. This is because it's difficult to track which process belongs to which root user:
+#
+# On Linux, user sessions are tracking using a kernel-side session id, however, this session id is not recorded by OpenSSH.
+# Additionally, only tools such as systemd and auditd record the process session id.
+# On other OSes, the user session id is not necessarily recorded at all kernel-side.
+# Using regular users in combination with /bin/su or /usr/bin/sudo ensure a clear audit track.
+PermitRootLogin No

ansible/group_vars/all

@@ -0,0 +1,3 @@
+keys:
+  - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCntA+4D8gAXocrFrC1MYhAyT2QzS5Qpx2emsg/hhjrxYVizmDQFV0BpqSPXDGEEFAcZNIQay/0i3u6JXoLnoB5xHJYHAk7xlRVKsX3T74MXFxucNQK5ZBlMIDGd2+T1JnqLM4lJ3Qo76q05ljloURFCkkchTsczUiKxKei42RI7bBVAjvZTREqalZQ+Il4j8h7G7Uy3CVkyLxYYQka4vFPAUUGov4BTu/WOtbmmcZ9pkQFIkoGSqXlRznurpgOE5gf+AiPytHGmyvVOz/G2O0Bpha0pgZtxJVVZnEvi8PqLQCHX1qM/GoW50UYIVnKdyhlS6Mg9DByETlC86N6Uj5d lordaro@Apollo"
+  - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDAhcPjy6TLOEPhhCfW6ap3rGgJWxUHanrRb/7pgFTM1uDsu/6qWieP0I6ivVVs4GByJgqqPHfzscLuoGuZoJ/CoskXRaZCzeextCRFHWtiiwLR1HB0Z954oHi+u8e4avYEdrsIteJva+bPiW8pjaKq/3nUyr4sQZzRVdsmU9kM3ZzXFt7lQvm75Mphoeu3As2BbAAs9SJzC/P2Odu5jSINHyOpjp2mzz0NRcE25FQzjGOrkpuGNqdKM7NgkXSQxZ/ozbp2cNW2gF+fEDtI6+LbwkhkqYbuKJWw2zuuPTZ3UFv95RY7/M1nUH9SF8Ci/IY0h1g3QU7YOa/ZtoktQs/abyfjIYRyw28jC6V3DOGEifd3FlHpdVRN3H9aNNK207a+0eKKvM7C0ZWPOJQAJSrCndj6QlrKnpHNG5a8bDAwqdtDQ7pK1YJkhI1W9HNUJ6cf9E6JXEphdoaBxE2YaFPkSzjzIzTiTIehHN2VRfCl1DKNcKy2XpbwMYE+6/PZf2U= truebrain@truebrain"

ansible/group_vars/bananas_fileservers

@@ -0,0 +1,124 @@
+# Our vars, for easy reuse
+cache_timeout: "1y"  # Basically just as long as nginx runs for
+bananas_production_cdn: bananas.cdn.openttd.org
+bananas_staging_cdn: bananas.cdn.staging.openttd.org
+staging_hostname: "{{ inventory_hostname_short }}.cdn.staging.openttd.org"
+
+certbot_admin_email: info@openttd.org
+certbot_create_if_missing: true
+certbot_create_standalone_stop_services: []
+certbot_certs:
+  - domains:
+      - "{{ inventory_hostname }}" # Note that this is the "primary" and contains the certificate for the staging domain
+      - "{{ staging_hostname }}"
+
+
+nginx_remove_default_vhost: true
+nginx_extra_http_options: |
+  proxy_cache_path /tmp/nginx-production-cache/ levels=1:2 keys_zone=nginx-production-cache:16m max_size=1g inactive={{ cache_timeout }} use_temp_path=off;
+  proxy_cache_path /tmp/nginx-staging-cache/ levels=1:2 keys_zone=nginx-staging-cache:16m max_size=1g inactive={{ cache_timeout }} use_temp_path=off;
+
+nginx_healthz: |
+  location = /healthz {
+    access_log off;
+    return 200;
+  }
+
+# SSL mostly taken from https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.6
+# with OSCP & HSTS bits removed which we don't want due to OTTD client's HTTP-only requirement
+nginx_ssl_config: |
+  ssl_certificate     /etc/letsencrypt/live/{{ inventory_hostname }}/fullchain.pem;
+  ssl_certificate_key /etc/letsencrypt/live/{{ inventory_hostname }}/privkey.pem;
+  ssl_session_timeout 1d;
+  ssl_session_cache shared:MozSSL:10m;  # about 40000 sessions
+  ssl_session_tickets off;
+  ssl_protocols TLSv1.2 TLSv1.3;
+  ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
+  ssl_prefer_server_ciphers off;
+
+# Common bits of the proxy config
+nginx_proxy_config: |
+  proxy_ssl_server_name on;
+  proxy_hide_header Strict-Transport-Security;
+  proxy_hide_header X-Cache;
+  proxy_hide_header ETag;
+  proxy_hide_header Via;
+  proxy_hide_header X-Amz-Cf-Pop;
+  proxy_hide_header X-Amz-Cf-Id;
+  proxy_cache_valid 200 {{ cache_timeout }};
+  proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504;
+  proxy_cache_revalidate on;
+  proxy_cache_lock on;
+  proxy_ignore_headers Set-Cookie;
+  add_header X-Cache-Status $upstream_cache_status;
+
+rewrite_regex: |
+  # Rewrite the first to the second:
+  #   base-graphics/12345678/12345678901234567890123456789012/filename.tar.gz
+  #   base-graphics/12345678/12345678901234567890123456789012.tar.gz
+  # This allows the OpenTTD client to know the name to use for the file,
+  #   while the S3 only knows the md5sum based name.
+  rewrite "^/([a-z-]+)/([a-f0-9]{8})/([a-f0-9]{32})/[a-zA-Z0-9-_.]+\.tar\.gz$" /$1/$2/$3.tar.gz break;
+  return 404;
+
+# Note: Hacks to get around lack of IPv6 support. See https://github.com/geerlingguy/ansible-role-nginx/pull/125
+nginx_vhosts:
+  # production
+  - listen: "80; listen [::]:80"
+    server_name: "{{ inventory_hostname }}"
+    filename: "{{ inventory_hostname }}.80.conf"
+    extra_parameters: |
+      {{ nginx_healthz }}
+      location / {
+        {{ rewrite_regex }}
+
+        proxy_pass https://{{ bananas_production_cdn }}/;
+        proxy_cache nginx-production-cache;
+        proxy_set_header Host {{ bananas_production_cdn }};
+        {{ nginx_proxy_config }}
+      }
+
+  - listen: "443 ssl http2; listen [::]:443 ssl http2"
+    server_name: "{{ inventory_hostname }}"
+    filename: "{{ inventory_hostname }}.443.conf"
+    extra_parameters: |
+      {{ nginx_ssl_config }}
+      {{ nginx_healthz }}
+      location / {
+        {{ rewrite_regex }}
+
+        proxy_pass https://{{ bananas_production_cdn }}/;
+        proxy_cache nginx-production-cache;
+        proxy_set_header Host {{ bananas_production_cdn }};
+        {{ nginx_proxy_config }}
+      }
+
+  # staging
+  - listen: "80; listen [::]:80"
+    server_name: "{{ staging_hostname }}"
+    filename: "{{ staging_hostname }}.80.conf"
+    extra_parameters: |
+      {{ nginx_healthz }}
+      location / {
+        {{ rewrite_regex }}
+
+        proxy_pass https://{{ bananas_staging_cdn }}/;
+        proxy_cache nginx-staging-cache;
+        proxy_set_header Host {{ bananas_staging_cdn }};
+        {{ nginx_proxy_config }}
+      }
+
+  - listen: "443 ssl http2; listen [::]:443 ssl http2"
+    server_name: "{{ staging_hostname }}"
+    filename: "{{ staging_hostname }}.443.conf"
+    extra_parameters: |
+      {{ nginx_ssl_config }}
+      {{ nginx_healthz }}
+      location / {
+        {{ rewrite_regex }}
+
+        proxy_pass https://{{ bananas_staging_cdn }}/;
+        proxy_set_header Host {{ bananas_staging_cdn }};
+        proxy_cache nginx-staging-cache;
+        {{ nginx_proxy_config }}
+      }

ansible/inventory

@@ -0,0 +1,3 @@
+[bananas_fileservers]
+bananas-1.cdn.openttd.org
+bananas-2.cdn.openttd.org

ansible/requirements.yml

@@ -0,0 +1,8 @@
+collections:
+  - name: ansible.posix
+    version: "1.1.2-dev8"
+roles:
+  - name: geerlingguy.certbot
+    version: "3.1.0"
+  - name: geerlingguy.nginx
+    version: "2.8.0"