Merge pull request #5486 from freedomofpress/5468-libvirt-staging-focal

Adds libvirt staging environment for Focal

Makefile

@@ -195,11 +195,16 @@ docs:  ## Build project documentation with live reload for editing.
 	@echo
 
 .PHONY: staging
-staging:  ## Create a local staging environment in virtual machines.
-	@echo "███ Creating staging environment..."
+staging:  ## Create a local staging environment in virtual machines (Xenial)
+	@echo "███ Creating staging environment on Ubuntu Xenial..."
 	@$(SDROOT)/devops/scripts/create-staging-env
 	@echo
 
+.PHONY: staging-focal
+staging-focal:  ## Create a local staging environment in virtual machines (Focal)
+	@echo "███ Creating staging environment on Ubuntu Focal..."
+	@$(SDROOT)/devops/scripts/create-staging-env focal
+	@echo
 
 .PHONY: testinfra
 testinfra:  ## Run infra tests against a local staging environment.

devops/scripts/select-staging-env

@@ -11,7 +11,9 @@
 set -e
 set -o pipefail
 
-securedrop_platform_suffix="-xenial"
+
+# Support overrides for LTS version
+securedrop_platform_suffix="-${1:-xenial}"
 
 # Respect explicit choice of Vagrant provider if set.
 if [[ -n "${VAGRANT_DEFAULT_PROVIDER:-}" ]] ; then

install_files/ansible-base/roles/common/defaults/main.yml

@@ -3,6 +3,15 @@
 # and aid in clearing memory. Only the hour is configurable.
 daily_reboot_time: 4 # An integer between 0 and 23
 
+securedrop_common_packages:
+  - apt-transport-https
+  - aptitude
+  - cron-apt
+  - ntp
+  - ntpdate
+  - resolvconf
+  - tmux
+
 disabled_kernel_modules:
   - btusb
   - bluetooth

install_files/ansible-base/roles/common/tasks/apt_upgrade.yml

@@ -20,6 +20,8 @@
   command: aptitude search '~U' --display-format '%p' --disable-columns
   register: tor_upgradable_result
   changed_when: false
+  # aptitude >= 0.7.6 will exit non-zero if no hits
+  failed_when: false
 
 - name: Hold tor package to prevent upgrade breaking SSH connection.
   command: aptitude hold tor

install_files/ansible-base/roles/common/tasks/install_packages.yml

@@ -0,0 +1,7 @@
+---
+- name: Install base apt depedencies
+  apt:
+    name: "{{ securedrop_common_packages }}"
+    state: present
+    update_cache: yes
+    cache_valid_time: 3600

install_files/ansible-base/roles/common/tasks/main.yml

@@ -1,5 +1,7 @@
 ---
-- include: install_ntp.yml
+- include_vars: "{{ ansible_distribution }}_{{ ansible_distribution_release }}.yml"
+
+- include: install_packages.yml
 
 - include: post_ubuntu_install_checks.yml
 
@@ -9,8 +11,6 @@
 
 - include: harden_dns.yml
 
-- include: install_tmux.yml
-
 - include: cron_apt.yml
   tags:
     - reboot

install_files/ansible-base/roles/common/tasks/remove_unused_packages.yml

@@ -13,14 +13,7 @@
 # We must used command due to the use of wildcards
 - name: Remove generic kernel packages.
   command: apt-get remove -y {{ item }}
-  with_items:
-    - linux-signed-generic
-    - linux-signed-generic-lts-utopic
-    - linux-signed-image-generic
-    - linux-signed-image-generic-lts-utopic
-    - linux-image-generic-lts-xenial
-    - 'linux-image-.*generic'
-    - 'linux-headers-.*'
+  with_items: "{{ securedrop_kernel_packages_to_remove }}"
   register: apt_removed_kernels
   changed_when: "'The following packages will be REMOVED' in apt_removed_kernels.stdout"
   tags:
@@ -41,6 +34,7 @@
   with_items: "{{ apt_installed_kernels.stdout_lines }}"
   tags:
     - apt
+    - grsecurity
 
 - name: Remove dependencies that are no longer required
   apt:

install_files/ansible-base/roles/common/vars/Ubuntu_focal.yml

@@ -0,0 +1,4 @@
+---
+securedrop_kernel_packages_to_remove:
+  - linux-virtual
+  - linux-generic

install_files/ansible-base/roles/common/vars/Ubuntu_xenial.yml

@@ -0,0 +1,9 @@
+---
+securedrop_kernel_packages_to_remove:
+  - linux-signed-generic
+  - linux-signed-generic-lts-utopic
+  - linux-signed-image-generic
+  - linux-signed-image-generic-lts-utopic
+  - linux-image-generic-lts-xenial
+  - 'linux-image-.*generic'
+  - 'linux-headers-.*'

molecule/libvirt-staging-focal/ansible-override-vars.yml

@@ -0,0 +1,10 @@
+---
+# Permit direct access via SSH
+ssh_net_in_override: 0.0.0.0/0
+
+# In libvirt, we want to connect over eth0, not eth1 which is used for
+# inter-VM communication for OSSEC.
+ssh_ip: "{{ ansible_default_ipv4.address }}"
+
+# Make sure Focal packages are used during installation
+securedrop_staging_install_target_distro: "focal"

molecule/libvirt-staging-focal/create.yml

@@ -0,0 +1,56 @@
+---
+- name: Create
+  hosts: localhost
+  connection: local
+  vars:
+    molecule_file: "{{ lookup('env', 'MOLECULE_FILE') }}"
+    molecule_instance_config: "{{ lookup('env', 'MOLECULE_INSTANCE_CONFIG') }}"
+    molecule_yml: "{{ lookup('file', molecule_file) | molecule_from_yaml }}"
+  tasks:
+
+    - name: Create molecule instance(s)
+      molecule_vagrant:
+        instance_name: "{{ item.name }}"
+        instance_interfaces: "{{ item.interfaces | default(omit) }}"
+        instance_raw_config_args: "{{ item.instance_raw_config_args | default(omit) }}"
+
+        platform_box: "{{ item.box }}"
+        platform_box_version: "{{ item.box_version | default(omit) }}"
+        platform_box_url: "{{ item.box_url | default(omit) }}"
+
+        provider_name: "{{ molecule_yml.driver.provider.name }}"
+        provider_memory: "{{ item.memory | default(omit) }}"
+        provider_cpus: "{{ item.cpus | default(omit) }}"
+        provider_raw_config_args: "{{ item.raw_config_args | default(omit) }}"
+        force_stop: yes
+
+        state: up
+      register: server
+      with_items: "{{ molecule_yml.platforms }}"
+
+    # Mandatory configuration for Molecule to function.
+
+    - name: Populate instance config dict
+      set_fact:
+        instance_conf_dict: {
+          'instance': "{{ item.Host }}",
+          'address': "{{ item.HostName }}",
+          'user': "{{ item.User }}",
+          'port': "{{ item.Port }}",
+          'identity_file': "{{ item.IdentityFile }}", }
+      with_items: "{{ server.results }}"
+      register: instance_config_dict
+      when: server.changed | bool
+
+    - name: Convert instance config dict to a list
+      set_fact:
+        instance_conf: "{{ instance_config_dict.results | map(attribute='ansible_facts.instance_conf_dict') | list }}"
+      when: server.changed | bool
+
+    - name: Dump instance config
+      copy:
+        # NOTE(retr0h): Workaround for Ansible 2.2.
+        #               https://github.com/ansible/ansible/issues/20885
+        content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}"
+        dest: "{{ molecule_instance_config }}"
+      when: server.changed | bool

molecule/libvirt-staging-focal/destroy.yml

@@ -0,0 +1,35 @@
+---
+
+- name: Destroy
+  hosts: localhost
+  connection: local
+  vars:
+    molecule_file: "{{ lookup('env', 'MOLECULE_FILE') }}"
+    molecule_instance_config: "{{ lookup('env',' MOLECULE_INSTANCE_CONFIG') }}"
+    molecule_yml: "{{ lookup('file', molecule_file) | molecule_from_yaml }}"
+    molecule_ephemeral_directory: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}"
+  tasks:
+    - name: Destroy molecule instance(s)
+      molecule_vagrant:
+        instance_name: "{{ item.name }}"
+        platform_box: "{{ item.box }}"
+        provider_name: "{{ molecule_yml.driver.provider.name }}"
+        force_stop: "{{ item.force_stop | default(True) }}"
+
+        state: destroy
+      register: server
+      with_items: "{{ molecule_yml.platforms }}"
+
+    # Mandatory configuration for Molecule to function.
+
+    - name: Populate instance config
+      set_fact:
+        instance_conf: {}
+
+    - name: Dump instance config
+      copy:
+        # NOTE(retr0h): Workaround for Ansible 2.2.
+        #               https://github.com/ansible/ansible/issues/20885
+        content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}"
+        dest: "{{ molecule_instance_config }}"
+      when: server.changed | bool

molecule/libvirt-staging-focal/molecule.yml

@@ -0,0 +1,74 @@
+---
+driver:
+  name: vagrant
+  provider:
+    name: libvirt
+lint:
+  name: yamllint
+
+platforms:
+  - name: app-staging
+    box: bento/ubuntu-20.04
+    raw_config_args:
+      - "cpu_mode = 'host-passthrough'"
+      - "video_type = 'virtio'"
+    instance_raw_config_args:
+      - "vm.synced_folder './', '/vagrant', disabled: true"
+      - "vm.network 'private_network', ip: '10.0.1.2'"
+      - "ssh.insert_key = false"
+    memory: 1024
+    private_ip: 10.0.1.2
+    groups:
+      - securedrop_application_server
+      - securedrop
+      - staging
+
+  - name: mon-staging
+    box: bento/ubuntu-20.04
+    raw_config_args:
+      - "cpu_mode = 'host-passthrough'"
+      - "video_type = 'virtio'"
+    instance_raw_config_args:
+      - "vm.synced_folder './', '/vagrant', disabled: true"
+      - "vm.network 'private_network', ip: '10.0.1.3'"
+      - "ssh.insert_key = false"
+    memory: 1024
+    private_ip: 10.0.1.3
+    groups:
+      - securedrop_monitor_server
+      - securedrop
+      - staging
+
+provisioner:
+  name: ansible
+  lint:
+    name: ansible-lint
+  config_options:
+    defaults:
+      interpreter_python: auto
+  options:
+    e: "@ansible-override-vars.yml"
+  playbooks:
+    converge: ../../install_files/ansible-base/securedrop-staging.yml
+    create: create.yml
+    destroy: destroy.yml
+    prepare: prepare.yml
+  env:
+    ANSIBLE_CONFIG: ../../install_files/ansible-base/ansible.cfg
+
+scenario:
+  name: libvirt-staging-focal
+  test_sequence:
+    - destroy
+    - create
+    - converge
+    - verify
+verifier:
+  name: testinfra
+  lint:
+    name: flake8
+  directory: ../testinfra/staging/
+  options:
+    n: auto
+    v: 2
+    junit-xml: ../../junit/testinfra-results.xml

molecule/libvirt-staging-focal/prepare.yml

@@ -0,0 +1,9 @@
+---
+- name: Prepare
+  hosts: all
+  gather_facts: False
+  tasks:
+    - name: Install python for Ansible
+      raw: test -e /usr/bin/python3 || (apt -y update && apt install -y python3-minimal)
+      become: True
+      changed_when: False