Dynamic vps creation

.github/workflows/run_automated_builder.yml

@@ -1,43 +1,47 @@
-name: Build VMs
-
+---
+name: Run automated builder
 on: push
-
 concurrency:
   group: ${{ github.workflow }}
   cancel-in-progress: true
 
 jobs:
   build:
     runs-on: ubuntu-latest
-    env: 
+    env:
       ANSIBLE_VAULT_PASSWORD: ${{ secrets.ANSIBLE_VAULT_PASSWORD }}
       REPO_URL: ${{ github.repository }}
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
-
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.8
-
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install ansible
-
-    - name: Run automated builder
-      run: |
-        ./automated_builder/scripts/run_automated_builder.sh $ANSIBLE_VAULT_PASSWORD
-
-    - name: Handle artifacts
-      if: always()
-      run: |
-        ./automated_builder/scripts/handle_artifacts.sh $ANSIBLE_VAULT_PASSWORD
-
-    - name: Upload artifacts
-      uses: actions/upload-artifact@v3
-      if: always()
-      with:
-        name: logs
-        path: ./automated_builder/logs/
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install ansible
+
+      - name: Run automated builder
+        run: |
+          ./automated_builder/scripts/run_automated_builder.sh $ANSIBLE_VAULT_PASSWORD
+
+      - name: Handle artifacts
+        if: always()
+        run: |
+          ./automated_builder/scripts/handle_artifacts.sh $ANSIBLE_VAULT_PASSWORD
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: logs
+          path: ./automated_builder/logs/
+
+      - name: Teardown build
+        if: always()
+        run: |
+          ./automated_builder/scripts/teardown_build.sh $ANSIBLE_VAULT_PASSWORD

automated_builder/gather_build_logs.yml

@@ -0,0 +1,43 @@
+---
+- name: Gather build logs
+  hosts: vps_runner
+  gather_facts: false
+  vars_files:
+    - ./roles/common/vars/main.yml
+    - ./roles/common/vars/secrets.yml
+
+  tasks:
+    - name: Get droplet IP
+      community.digitalocean.digital_ocean_droplet_info:
+        oauth_token: "{{ DO_API_TOKEN }}"
+        name: "automated-builder-vps"
+      delegate_to: localhost
+      register: automated_builder_vps
+
+    - name: Set VPS_IP
+      set_fact:
+        VPS_IP: "{{ automated_builder_vps.data[0].networks.v4 | selectattr('type', 'equalto', 'public') | map(attribute='ip_address') | first }}"
+      delegate_to: localhost
+
+    - name: Gather facts
+      setup:
+
+    - name: Copy install_source log
+      fetch:
+        src: "/home/ansible/install_source.log"
+        dest: "./logs/install_source.log"
+
+    - name: Check logs
+      shell: "ls"
+      register: pwd
+      delegate_to: 127.0.0.1
+
+    - name: Copy gateway_build log
+      fetch:
+        src: "/home/ansible/gateway_build.log"
+        dest: "./logs/gateway_build.log"
+
+    - name: Copy workstation_build log
+      fetch:
+        src: "/home/ansible/workstation_build.log"
+        dest: "./logs/workstation_build.log"

automated_builder/main.yml

@@ -0,0 +1,22 @@
+---
+- name: Run automated_builder
+  hosts: vps_runner
+  gather_facts: false
+  vars_files:
+    - ./roles/common/vars/main.yml
+    - ./roles/common/vars/secrets.yml
+
+  tasks:
+    - name: Include common role
+      include_role:
+        name: common
+
+    - name: Include headless-build role
+      include_role:
+        name: headless-build
+      when: REF_TYPE != 'tag'
+
+    - name: Include gui-build role
+      include_role:
+        name: gui-build
+      when: REF_TYPE == 'tag'

automated_builder/roles/common/tasks/bootstrap_vps.yml

@@ -0,0 +1,35 @@
+---
+- name: Bootstrap VPS
+  become: true
+  vars:
+    ansible_ssh_user: root
+
+  block:
+    - name: Create ansible user
+      user:
+        name: ansible
+
+    - name: Create ansible user ssh directory
+      file:
+        path: /home/ansible/.ssh
+        state: directory
+        owner: ansible
+        group: ansible
+        mode: 0700
+
+    - name: Copy authorized_keys
+      copy:
+        src: /root/.ssh/authorized_keys
+        dest: /home/ansible/.ssh/authorized_keys
+        owner: ansible
+        group: ansible
+        mode: 0600
+        remote_src: true
+
+    - name: Allow passwordless sudo commands
+      community.general.sudoers:
+        name: ansible-passwordless-sudo
+        state: present
+        user: ansible
+        commands: ALL
+        nopassword: true

automated_builder/tasks/configure_local_environment.yml → automated_builder/roles/common/tasks/configure_local_environment.yml

@@ -1,12 +1,7 @@
 ---
 - name: Configure local environment
-  hosts: 127.0.0.1
-  connection: local
-  vars_files:
-    - ../vars/main.yml
-    - ../vars/secrets.yml
-
-  tasks:
+  delegate_to: 127.0.0.1
+  block:
     - name: Create local_ssh directory
       file:
         path: ~/.ssh

automated_builder/roles/common/tasks/delete_inventory.yml

@@ -0,0 +1,22 @@
+---
+- name: Destroy any existing droplets
+  hosts: 127.0.0.1
+  vars_files:
+    - ../vars/secrets.yml
+  tasks:
+    - name: Check for existing inventory
+      community.digitalocean.digital_ocean_droplet_info:
+        oauth_token: "{{ DO_API_TOKEN }}"
+      register: droplets
+
+    - name: Count existing droplets
+      set_fact:
+        droplet_count: "{{ droplets.data | length }}"
+
+    - name: Delete existing droplets
+      community.digitalocean.digital_ocean_droplet:
+        state: absent
+        oauth_token: "{{ DO_API_TOKEN }}"
+        id: "{{ item.id }}"
+      loop: "{{ droplets.data }}"
+      when: droplet_count != "0"

automated_builder/roles/common/tasks/generate_inventory.yml

@@ -0,0 +1,28 @@
+---
+- name: Create VPS resource
+  delegate_to: 127.0.0.1
+  block:
+    - name: Create SSH key
+      community.digitalocean.digital_ocean_sshkey:
+        oauth_token: "{{ DO_API_TOKEN }}"
+        name: "Ansible Key"
+        ssh_pub_key: "{{ SSH_PUBLIC_KEY }}"
+        state: present
+      register: public_key
+
+    - name: Create automated builder VPS
+      community.digitalocean.digital_ocean_droplet:
+        state: present
+        oauth_token: "{{ DO_API_TOKEN }}"
+        name: automated-builder-vps
+        size: s-4vcpu-8gb
+        region: nyc3
+        image: debian-11-x64
+        wait_timeout: 500
+        ssh_keys: ["{{ public_key.data.ssh_key.id }}"]
+        project: "Automated Builder"
+      register: automated_builder_vps
+
+    - name: set VPS_IP
+      set_fact:
+        VPS_IP: "{{ automated_builder_vps.data.droplet.networks.v4 | selectattr('type', 'equalto', 'public') | map(attribute='ip_address') | first }}"

automated_builder/roles/common/tasks/install_dependencies.yml

@@ -0,0 +1,36 @@
+---
+- name: Install dependencies
+  become: true
+  block:
+    - name: Install apt packages
+      apt:
+        pkg:
+          - git
+          - time
+          - curl
+          - lsof
+          - apt-cacher-ng
+          - lsb-release
+          - fakeroot
+          - dpkg-dev
+          - fasttrack-archive-keyring
+          - dnsutils
+          - software-properties-common
+        update_cache: true
+
+    - name: Install VirtualBox
+      include_tasks: install_virtualbox.yml
+
+    - name: Register LSB release
+      shell: "lsb_release -cs"
+      register: lsb_release
+
+    - name: Add VirtualBox apt repository
+      apt_repository:
+        repo: "deb [arch=amd64 signed-by=/usr/share/keyrings/{{ VBOX_ASC_KEY.name }}.asc.gpg] http://download.virtualbox.org/virtualbox/debian {{ lsb_release.stdout }} contrib"
+        state: present
+
+    - name: Install VirtualBox apt repository
+      apt:
+        name: "virtualbox-6.1"
+        update_cache: true

automated_builder/roles/common/tasks/install_virtualbox.yml

@@ -0,0 +1,18 @@
+---
+- name: Install VirtualBox
+  become: true
+
+  block:
+    - name: Import VirtualBox GPG key
+      get_url:
+        dest: "/root/{{ VBOX_ASC_KEY.name }}.asc"
+        url: "{{ VBOX_ASC_KEY.url }}"
+
+    - name: Create gpg file
+      shell: "gpg --dearmor /root/{{ VBOX_ASC_KEY.name }}.asc > {{ VBOX_ASC_KEY.name }}.gpg"
+
+    - name: Move gpg key to shared keyrings
+      copy:
+        src: "/root/{{ VBOX_ASC_KEY.name }}.asc.gpg"
+        dest: "/usr/share/keyrings/{{ VBOX_ASC_KEY.name }}.asc.gpg"
+        remote_src: true

automated_builder/roles/common/tasks/main.yml

@@ -0,0 +1,18 @@
+---
+- name: Configure local environment
+  include_tasks: configure_local_environment.yml
+
+- name: Generate inventory
+  include_tasks: generate_inventory.yml
+
+- name: Bootstrap VPS
+  include_tasks: bootstrap_vps.yml
+
+- name: Gather facts
+  setup:
+
+- name: Install dependencies
+  include_tasks: install_dependencies.yml
+
+- name: Clean existing VirtualBox VMs
+  include_tasks: clean_existing_vbox_vms.yml

automated_builder/vars/main.yml → automated_builder/roles/common/vars/main.yml

@@ -1,3 +1,6 @@
 REF_TYPE: "{{ lookup('env', 'GITHUB_REF_TYPE') }}"
 GIT_REPO: "{{ lookup('env', 'REPO_URL') }}"
 REF_NAME: "{{ lookup('env', 'GITHUB_REF_NAME') }}"
+VBOX_ASC_KEY:
+  name: 'oracle_vbox_2016'
+  url: 'https://www.virtualbox.org/download/oracle_vbox_2016.asc'

automated_builder/roles/common/vars/secrets.yml

@@ -0,0 +1,38 @@
+$ANSIBLE_VAULT;1.1;AES256
+36613161353462363665326163373733653932646233663737356636623834366566386235313435
+3365383733393738383863306661353339653261623533340a343537323738663234306265316238
+32646364633136353664356631613436616462343962336264613761366638363364383133386531
+3361303831616636340a663839653265353263323766353932333962343235343631653265316665
+64613665356538363036666336633633323765326338623637323335613932616536633262356134
+30653466336366646162313634326136366364656635303764336263313530373735653239393433
+64306236333365303466616263626437363635323037626662613665303834396539306230303530
+64383463653663613936626236353936616332383065643962353763313335353665333762313036
+37326164663239623233643364383238616134393537316566326165623363323161656435623965
+32386138666635323033383036643337663231363164306664366230323763626338366261373335
+37643064633734386535393430346564366638303763313165663765646135353737333430396166
+30613461343036376163636636353534613039616631313361333363383634643166643363633131
+31353437376639356138366662383565663535323833356163373134323934663938333134393466
+37383063336338656132376336623437626438316232313736313932646166383735303961636636
+62383635653832393030343433333464343935306161373632336338666136323132623036623833
+39373663633238623833643734623764313233306533393862653837326462666561366530626135
+62333365623336613163633538303765343931346230613231613462653734363566383739646236
+31656534613036646233306234336163616638366265643730346533343832613661386433326430
+34353964303937373864343433653439643239646630633863333561663266623735326437393532
+30636431643636623830643638363831393935616636386138343434353832653065663639363965
+30343139303866383130653261323538623033353237623262353463663236613965316631623533
+64356239323538616338323265613233666636376634613566643934356661393338343338333734
+38333763663366396266663530356361626664663832363232623130383631616530383337343662
+39623738386164663666386363653265386338616363373239386265306539346163643866666637
+36303161623038376536626432393264613736393032613464643238343837643664386538643564
+66386361393835613737363037636630613134333830653036643037343165663332356464623162
+35653938653066666432633563393933626437313237383837663038623062363739306635303964
+38333939626330396533613133623061343061343635643665393532326361323933646132386464
+31383630323362306366366638646432633466393963356232363438353866343665336364656534
+32326666653665663566636332366365613061613963376138336164656435386238353437666435
+62396134313661336163363838313939326438353232306339313762393964376164613336633663
+38353062663266326335633433336639633934333964376665663761636664643563316362313664
+37363834656534383330303465313237386264303661393862646532313431323030316331393436
+32656230326138663434336561663931633739393435383065363636626563353865616562353966
+66303738613332333631333965663934643961633839306466663338656461636634373439646130
+36393533316538646664363166303434343738373661393034373738313966663566363462393266
+6434

automated_builder/roles/gui-build/tasks/build_vms_from_tag.yml

@@ -0,0 +1,9 @@
+---
+- name: Install build_vms_from_tag script
+  template:
+    src: ../templates/build_vms_from_tag.sh
+    dest: /home/ansible/build_vms_from_tag.sh
+    mode: 0744
+
+- name: Run build_vms_from_tag scripts
+  shell: "/home/ansible/build_vms_from_tag.sh"

automated_builder/roles/gui-build/tasks/install_source.yml

@@ -0,0 +1,9 @@
+---
+- name: Create install_source script
+  template:
+    src: ./roles/common/templates/install_source.sh
+    dest: /home/ansible/install_source.sh
+    mode: 0744
+
+- name: Run install_source script for tag
+  shell: "/home/ansible/install_source.sh {{ GIT_REPO }} {{ REF_NAME }} {{ REF_NAME }} > /home/ansible/install_source.log 2>&1"

automated_builder/roles/gui-build/tasks/main.yml

@@ -0,0 +1,6 @@
+---
+- name: Install source code for commit
+  include_tasks: install_source.yml
+
+- name: Build VMs from tag
+  include_tasks: build_vms_from_tag.yml