From e60576b448e71faf3c38b7d431cc40c5f42df978 Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Tue, 15 Nov 2022 20:38:27 -0700 Subject: [PATCH 01/44] Add initial droplet creation strategy --- .github/workflows/run_automated_builder.yml | 26 +++---- .../scripts/run_automated_builder.sh | 6 +- .../tasks/generate_inventory.yml | 30 ++++++++ automated_builder/vars/secrets.yml | 70 ++++++++++--------- 4 files changed, 84 insertions(+), 48 deletions(-) create mode 100644 automated_builder/tasks/generate_inventory.yml diff --git a/.github/workflows/run_automated_builder.yml b/.github/workflows/run_automated_builder.yml index 99177f861..8f008ee05 100644 --- a/.github/workflows/run_automated_builder.yml +++ b/.github/workflows/run_automated_builder.yml @@ -1,4 +1,4 @@ -name: Build VMs +name: Run automated builder on: push @@ -9,7 +9,7 @@ concurrency: jobs: build: runs-on: ubuntu-latest - env: + env: ANSIBLE_VAULT_PASSWORD: ${{ secrets.ANSIBLE_VAULT_PASSWORD }} REPO_URL: ${{ github.repository }} steps: @@ -30,14 +30,14 @@ jobs: run: | ./automated_builder/scripts/run_automated_builder.sh $ANSIBLE_VAULT_PASSWORD - - name: Handle artifacts - if: always() - run: | - ./automated_builder/scripts/handle_artifacts.sh $ANSIBLE_VAULT_PASSWORD - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - if: always() - with: - name: logs - path: ./automated_builder/logs/ + # - name: Handle artifacts + # if: always() + # run: | + # ./automated_builder/scripts/handle_artifacts.sh $ANSIBLE_VAULT_PASSWORD + + # - name: Upload artifacts + # uses: actions/upload-artifact@v3 + # if: always() + # with: + # name: logs + # path: ./automated_builder/logs/ diff --git a/automated_builder/scripts/run_automated_builder.sh b/automated_builder/scripts/run_automated_builder.sh index 7bbbc129c..3aa009eae 100755 --- a/automated_builder/scripts/run_automated_builder.sh +++ b/automated_builder/scripts/run_automated_builder.sh @@ -12,8 +12,10 @@ main() { } run_builder() { - ansible-playbook -i automated_builder/inventory automated_builder/tasks/configure_local_environment.yml - ansible-playbook -i automated_builder/inventory automated_builder/tasks/build_vms.yml + ansible-playbook automated_builder/tasks/configure_local_environment.yml + ansible-galaxy collection install community.digitalocean + ansible-playbook automated_builder/tasks/generate_inventory.yml + # ansible-playbook -i automated_builder/inventory automated_builder/tasks/build_vms.yml } main diff --git a/automated_builder/tasks/generate_inventory.yml b/automated_builder/tasks/generate_inventory.yml new file mode 100644 index 000000000..92895f9ea --- /dev/null +++ b/automated_builder/tasks/generate_inventory.yml @@ -0,0 +1,30 @@ +--- +- name: Configure local environment + hosts: 127.0.0.1 + connection: local + vars_files: + - ../vars/main.yml + - ../vars/secrets.yml + + tasks: + - name: Check for existing inventory + community.digitalocean.digital_ocean_droplet_info: + oauth_token: "{{ DO_API_TOKEN }}" + register: droplets + + - name: Inspect Droplets + debug: + var: droplets + + - name: Count existing droplets + set_fact: + droplet_count: "{{ droplets.data | length }}" + + - name: Delete lingering droplets + debug: + msg: We should def delete droplets + when: droplet_count != 0 + + - name: Create droplet + debug: + msg: Ok make it now with SSH keys setup diff --git a/automated_builder/vars/secrets.yml b/automated_builder/vars/secrets.yml index 5bb0c7f28..90cc6c514 100644 --- a/automated_builder/vars/secrets.yml +++ b/automated_builder/vars/secrets.yml @@ -1,34 +1,38 @@ $ANSIBLE_VAULT;1.1;AES256 -30656233366135653137643536373830313836306562626634613037323432376561383662613434 -3733393535656662653337373161623635643537613133650a316630333434663861313335323932 -33336233396534323638323164353030346336373235303063666431653163366132613863333232 -3363653139376462350a613430326430316239616262626139643563666462353865666330326433 -36653665393537306438366332333462346436396662336233323233336466373637653861383635 -64626537326363356230393438623261313462363737343865376234666632303334303538393432 -38353531656439383663343662633333353734313435323834656563643534366638303131373933 -35343031343763623537366137643131643866393636376336363231336131643064666235336161 -62633639373838623734336564616437623161353834363631386335653236376431373934353739 -33653164313631316161313132663037643766336664633963643136616566613862376632376230 -31666461383531646237306463323536616562346233353435363866333561353831383861386431 -66313638643633626431333836383764663835326530323739363766376662633135323531643339 -63363539373264663261333632336532623531333337363134353832386661316662633531333136 -66633537626533643937633065333831653132316531313235313736393237373234383035643634 -35393838363636373031633634643364383465643633376462326564346165333165383666396333 -35376464323562623364346333663535303866366332346663383361636437643762363630623665 -66333931363033373362653034643964393539663635323365663561373235373038333632626534 -63643130333534356537306337306331643636313934333731353664643739396464643965346135 -62613263633565616132323136313066663335613361333464646236396533353633643363633637 -61343639366662316334366631353336613733303063303630393336353062346332323134376263 -35643266633064643162643962306566356335373233383236663839313838396264333162316262 -62393664336436666337356437666266613232653235633639656233626161303565623832373061 -65326161303038306534353865623561333561326339333865323338363662363866373862333832 -62373839623166363437333534326137373661623764383730363464326532333663313137636566 -38326563333339393566646665366165383733383134643463623065313262336130366232326261 -33616631396330356535633330373563333765363035333531323933313232646562313764396135 -61323562323033653731316364346136656339393434333965376238396263616135613562336333 -37613662663762393264643030393538376639343930636131336636636637626338643736303162 -33646337363063336133386634343336383263316439336430643663316138613230366436316533 -33636534663131613733336462396631326136336630333164386165366632333561313339386630 -65356361333666336432373239653762313363636635303363366235646163653261653233636463 -35623538313862333334393931353433336430613637393239616330323332616665663735623366 -32386334343166643566333931633933343062623336313037636565353235366263 +31363661653561386363626261393737666437333534613863623066313236643539393965366135 +3030633638653235333638373362383761333034363036610a383362303563393131363335363031 +62323862326336653066646533313535663865656662396339393836336334373938636433313733 +6565393663613866310a656362396231373337333938646265326663623761653062336363303039 +35306365373665366462306337626463613633663866666165636238636332313564643234393634 +32316363323866616430666262326436666465363930313239613630316534366332373438356662 +64306235663838326236376263376463663135613365653637643037366662373334393561393965 +66653734386335663335313934303164336634643330653266393033303330636137326563306630 +36376339643166306265623263383336303638643532396439333865623461303763633039663863 +62333261326133363664333936386539353131666666613836383135393934383737366565333261 +32316264353064633963323663373430353536633434623531616333306630396465303335653838 +32366661396664373761346162666434626637303939616539333230333763663532313734333063 +62353265366164346461626635613835303136353165653565643134643466346236373631396435 +62356635353939336335346338653137386533653361383466646462326239353238323635313065 +39633837396232383437636333616363353862373332643037643437326238623337626665363832 +30663434316133326261323134383865363964643130633835303231333663626536323037633938 +39386663323134653965363363633064313831613233643533303661636337383037313731383230 +34643036623765353166613832666265313033663262333638646533333133663736326463626530 +31353066613337666263326535663862646565646565353564613838386365623932613266363231 +39613034353463343136353664656539303433366432323761306330303034626361323461626438 +39313536613531373839316361623561613431346665663039663536643037316336306231663832 +63343533663363613231636636363234643238363135613062663531333837383762336566356363 +32376235353331306664356332386137613764353866363136356264663835656164623861323330 +65643765643762336138323435316330633032666262366436323264656463343364343432633565 +61353763613730376436373332326565333034336364383233393839616233666563666335373033 +37643564646564623437636164643537343165336134366337343236353331656162393763653633 +35663362353338306563386538333838636436313734323834326336373063336338633562636435 +63313264356634353936353966643464646634663937333235306435643136366237343463633537 +38353962373232633130326236623434366161653762353833373934376333633235653733623138 +63393531386431303336626664643935313639656265336133346436626434323636346364373830 +37383166303465653161363335346566373138366361313562396366393036663430343666396134 +35333330313863616363613963326232303066366162613434666232626130653239316165383961 +30343964653931373033303738613965363830383533623064303261616633326239346230363232 +65363533616265616163623164306434386639663130373135623938346238353230303133363434 +64303430656339303364346433633030326365646430653061643031386238656137313961616537 +31366464663236363333623339326666366139393734366436373539356633336633373764636639 +6338 From 2d93a90e3a862ade2f1515f0c2cbb9672760f5fd Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Tue, 15 Nov 2022 20:42:14 -0700 Subject: [PATCH 02/44] Add inventory for runs --- .github/workflows/run_automated_builder.yml | 12 ++++++++++++ automated_builder/scripts/run_automated_builder.sh | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_automated_builder.yml b/.github/workflows/run_automated_builder.yml index 8f008ee05..6fff171a2 100644 --- a/.github/workflows/run_automated_builder.yml +++ b/.github/workflows/run_automated_builder.yml @@ -41,3 +41,15 @@ jobs: # with: # name: logs # path: ./automated_builder/logs/ + + # - name: Destroy inventory + # if: always() + # run: | + # ./automated_builder/scripts/destroy_inventory.sh $ANSIBLE_VAULT_PASSWORD + + # - name: Upload artifacts + # uses: actions/upload-artifact@v3 + # if: always() + # with: + # name: logs + # path: ./automated_builder/logs/ diff --git a/automated_builder/scripts/run_automated_builder.sh b/automated_builder/scripts/run_automated_builder.sh index 3aa009eae..fd3bd5226 100755 --- a/automated_builder/scripts/run_automated_builder.sh +++ b/automated_builder/scripts/run_automated_builder.sh @@ -12,9 +12,9 @@ main() { } run_builder() { - ansible-playbook automated_builder/tasks/configure_local_environment.yml + ansible-playbook -i automated_builder/inventory automated_builder/tasks/configure_local_environment.yml ansible-galaxy collection install community.digitalocean - ansible-playbook automated_builder/tasks/generate_inventory.yml + ansible-playbook -i automated_builder/inventory automated_builder/tasks/generate_inventory.yml # ansible-playbook -i automated_builder/inventory automated_builder/tasks/build_vms.yml } From 683b5bc83ce7342344577c09c2528a7201b79ada Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Tue, 15 Nov 2022 20:54:48 -0700 Subject: [PATCH 03/44] Troubleshoot droplet counting conditionals --- .../tasks/generate_inventory.yml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/automated_builder/tasks/generate_inventory.yml b/automated_builder/tasks/generate_inventory.yml index 92895f9ea..dabd8b7c7 100644 --- a/automated_builder/tasks/generate_inventory.yml +++ b/automated_builder/tasks/generate_inventory.yml @@ -20,11 +20,24 @@ set_fact: droplet_count: "{{ droplets.data | length }}" + - name: Inspect droplet length + debug: + var: droplet_count + - name: Delete lingering droplets debug: msg: We should def delete droplets when: droplet_count != 0 - - name: Create droplet - debug: - msg: Ok make it now with SSH keys setup + # - name: Create a automated builder VPS + # community.digitalocean.digital_ocean_droplet: + # state: present + # oauth_token: "{{ DO_API_TOKEN }}" + # name: automated_builder_vps + # size: s-1vcpu-1gb + # region: nyc3 + # image: ubuntu-20-04-x64 + # wait_timeout: 500 + # ssh_keys: ["{{ SSH_KEY }}"] + # project: "Automated Builder" + # register: automated_builder_vps From 5d6bac77d5cabd4ad14a6c2cdd9f7761e8b51dcb Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Tue, 15 Nov 2022 21:10:22 -0700 Subject: [PATCH 04/44] Loop through existing droplets and delete --- automated_builder/tasks/generate_inventory.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/automated_builder/tasks/generate_inventory.yml b/automated_builder/tasks/generate_inventory.yml index dabd8b7c7..359761adc 100644 --- a/automated_builder/tasks/generate_inventory.yml +++ b/automated_builder/tasks/generate_inventory.yml @@ -20,14 +20,13 @@ set_fact: droplet_count: "{{ droplets.data | length }}" - - name: Inspect droplet length - debug: - var: droplet_count - - name: Delete lingering droplets - debug: - msg: We should def delete droplets - when: droplet_count != 0 + community.digitalocean.digital_ocean_droplet: + state: absent + oauth_token: "{{ DO_API_TOKEN }}" + id: "{{ item.id }}" + loop: "{{ droplets.data }}" + when: droplet_count != "0" # - name: Create a automated builder VPS # community.digitalocean.digital_ocean_droplet: From cadd194da3799f51a1ae1e24a2995a46ee7c2dcb Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Tue, 15 Nov 2022 21:18:11 -0700 Subject: [PATCH 05/44] Add automated_builder_vps creation --- .../tasks/generate_inventory.yml | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/automated_builder/tasks/generate_inventory.yml b/automated_builder/tasks/generate_inventory.yml index 359761adc..aa09202e7 100644 --- a/automated_builder/tasks/generate_inventory.yml +++ b/automated_builder/tasks/generate_inventory.yml @@ -28,15 +28,15 @@ loop: "{{ droplets.data }}" when: droplet_count != "0" - # - name: Create a automated builder VPS - # community.digitalocean.digital_ocean_droplet: - # state: present - # oauth_token: "{{ DO_API_TOKEN }}" - # name: automated_builder_vps - # size: s-1vcpu-1gb - # region: nyc3 - # image: ubuntu-20-04-x64 - # wait_timeout: 500 - # ssh_keys: ["{{ SSH_KEY }}"] - # project: "Automated Builder" - # register: automated_builder_vps + - name: Create automated builder VPS + community.digitalocean.digital_ocean_droplet: + state: present + oauth_token: "{{ DO_API_TOKEN }}" + name: automated_builder_vps + size: s-4vcpu-8gb + region: nyc3 + image: debian-11-x64 + wait_timeout: 500 + ssh_keys: ["{{ SSH_KEY }}"] + project: "Automated Builder" + register: automated_builder_vps From 46f1f1206dfc5992fa463fcdff78e0fc06519f08 Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Tue, 15 Nov 2022 21:22:25 -0700 Subject: [PATCH 06/44] Change ssh key to string --- automated_builder/tasks/generate_inventory.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated_builder/tasks/generate_inventory.yml b/automated_builder/tasks/generate_inventory.yml index aa09202e7..6414b5061 100644 --- a/automated_builder/tasks/generate_inventory.yml +++ b/automated_builder/tasks/generate_inventory.yml @@ -37,6 +37,6 @@ region: nyc3 image: debian-11-x64 wait_timeout: 500 - ssh_keys: ["{{ SSH_KEY }}"] + ssh_keys: "{{ SSH_KEY }}" project: "Automated Builder" register: automated_builder_vps From b19bfc8ace2c05f073e2e8594151404fdd2d4192 Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Tue, 15 Nov 2022 21:30:13 -0700 Subject: [PATCH 07/44] Create SSH key resource --- .../tasks/generate_inventory.yml | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/automated_builder/tasks/generate_inventory.yml b/automated_builder/tasks/generate_inventory.yml index 6414b5061..bf9d2d705 100644 --- a/automated_builder/tasks/generate_inventory.yml +++ b/automated_builder/tasks/generate_inventory.yml @@ -28,15 +28,27 @@ loop: "{{ droplets.data }}" when: droplet_count != "0" - - name: Create automated builder VPS - community.digitalocean.digital_ocean_droplet: - state: present + - name: Create SSH key + community.digitalocean.digital_ocean_sshkey: oauth_token: "{{ DO_API_TOKEN }}" - name: automated_builder_vps - size: s-4vcpu-8gb - region: nyc3 - image: debian-11-x64 - wait_timeout: 500 - ssh_keys: "{{ SSH_KEY }}" - project: "Automated Builder" - register: automated_builder_vps + name: "Ansible Key" + ssh_pub_key: "{{ SSH_PUBLIC_KEY }}" + state: present + register: public_key + + - name: Inspect SSH key + debug: + var: public_key + +# - name: Create automated builder VPS +# community.digitalocean.digital_ocean_droplet: +# state: present +# oauth_token: "{{ DO_API_TOKEN }}" +# name: automated_builder_vps +# size: s-4vcpu-8gb +# region: nyc3 +# image: debian-11-x64 +# wait_timeout: 500 +# ssh_keys: "{{ SSH_KEY }}" +# project: "Automated Builder" +# register: automated_builder_vps From c8b48217867d44694db6330a0923809e5b6b749a Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Tue, 15 Nov 2022 21:33:07 -0700 Subject: [PATCH 08/44] Create droplet with SSH key id --- .../tasks/generate_inventory.yml | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/automated_builder/tasks/generate_inventory.yml b/automated_builder/tasks/generate_inventory.yml index bf9d2d705..33b414b84 100644 --- a/automated_builder/tasks/generate_inventory.yml +++ b/automated_builder/tasks/generate_inventory.yml @@ -36,19 +36,15 @@ state: present register: public_key - - name: Inspect SSH key - debug: - var: public_key - -# - name: Create automated builder VPS -# community.digitalocean.digital_ocean_droplet: -# state: present -# oauth_token: "{{ DO_API_TOKEN }}" -# name: automated_builder_vps -# size: s-4vcpu-8gb -# region: nyc3 -# image: debian-11-x64 -# wait_timeout: 500 -# ssh_keys: "{{ SSH_KEY }}" -# project: "Automated Builder" -# register: automated_builder_vps + - name: Create automated builder VPS + community.digitalocean.digital_ocean_droplet: + state: present + oauth_token: "{{ DO_API_TOKEN }}" + name: automated_builder_vps + size: s-8vcpu-16gb + region: nyc3 + image: debian-11-x64 + wait_timeout: 500 + ssh_keys: ["{{ public_key.data.id }}"] + project: "Automated Builder" + register: automated_builder_vps From d1bff22411fa6c0ad917064259184e2194713e02 Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Tue, 15 Nov 2022 21:40:38 -0700 Subject: [PATCH 09/44] Create and include delete inventory task --- automated_builder/tasks/delete_inventory.yml | 25 +++++++++++++++++++ .../tasks/generate_inventory.yml | 24 +++--------------- 2 files changed, 28 insertions(+), 21 deletions(-) create mode 100644 automated_builder/tasks/delete_inventory.yml diff --git a/automated_builder/tasks/delete_inventory.yml b/automated_builder/tasks/delete_inventory.yml new file mode 100644 index 000000000..e0b498707 --- /dev/null +++ b/automated_builder/tasks/delete_inventory.yml @@ -0,0 +1,25 @@ +--- +- name: Configure local environment + hosts: 127.0.0.1 + connection: local + vars_files: + - ../vars/main.yml + - ../vars/secrets.yml + + tasks: + - name: Check for existing inventory + community.digitalocean.digital_ocean_droplet_info: + oauth_token: "{{ DO_API_TOKEN }}" + register: droplets + + - name: Count existing droplets + set_fact: + droplet_count: "{{ droplets.data | length }}" + + - name: Delete lingering droplets + community.digitalocean.digital_ocean_droplet: + state: absent + oauth_token: "{{ DO_API_TOKEN }}" + id: "{{ item.id }}" + loop: "{{ droplets.data }}" + when: droplet_count != "0" diff --git a/automated_builder/tasks/generate_inventory.yml b/automated_builder/tasks/generate_inventory.yml index 33b414b84..f4bd863e1 100644 --- a/automated_builder/tasks/generate_inventory.yml +++ b/automated_builder/tasks/generate_inventory.yml @@ -7,26 +7,8 @@ - ../vars/secrets.yml tasks: - - name: Check for existing inventory - community.digitalocean.digital_ocean_droplet_info: - oauth_token: "{{ DO_API_TOKEN }}" - register: droplets - - - name: Inspect Droplets - debug: - var: droplets - - - name: Count existing droplets - set_fact: - droplet_count: "{{ droplets.data | length }}" - - - name: Delete lingering droplets - community.digitalocean.digital_ocean_droplet: - state: absent - oauth_token: "{{ DO_API_TOKEN }}" - id: "{{ item.id }}" - loop: "{{ droplets.data }}" - when: droplet_count != "0" + - name: Delete lingering inventory + include_tasks: delete_inventory.yml - name: Create SSH key community.digitalocean.digital_ocean_sshkey: @@ -45,6 +27,6 @@ region: nyc3 image: debian-11-x64 wait_timeout: 500 - ssh_keys: ["{{ public_key.data.id }}"] + ssh_keys: ["{{ public_key.data.ssh_key.id }}"] project: "Automated Builder" register: automated_builder_vps From c4b4f82079fd38cc5195c92319e7a89ce2235e43 Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Tue, 15 Nov 2022 21:47:37 -0700 Subject: [PATCH 10/44] Change yml formatting --- automated_builder/tasks/delete_inventory.yml | 36 ++++++++----------- .../tasks/generate_inventory.yml | 4 +-- 2 files changed, 16 insertions(+), 24 deletions(-) diff --git a/automated_builder/tasks/delete_inventory.yml b/automated_builder/tasks/delete_inventory.yml index e0b498707..ec5f40158 100644 --- a/automated_builder/tasks/delete_inventory.yml +++ b/automated_builder/tasks/delete_inventory.yml @@ -1,25 +1,17 @@ --- -- name: Configure local environment - hosts: 127.0.0.1 - connection: local - vars_files: - - ../vars/main.yml - - ../vars/secrets.yml +- name: Check for existing inventory + community.digitalocean.digital_ocean_droplet_info: + oauth_token: "{{ DO_API_TOKEN }}" + register: droplets - tasks: - - name: Check for existing inventory - community.digitalocean.digital_ocean_droplet_info: - oauth_token: "{{ DO_API_TOKEN }}" - register: droplets +- name: Count existing droplets + set_fact: + droplet_count: "{{ droplets.data | length }}" - - name: Count existing droplets - set_fact: - droplet_count: "{{ droplets.data | length }}" - - - name: Delete lingering droplets - community.digitalocean.digital_ocean_droplet: - state: absent - oauth_token: "{{ DO_API_TOKEN }}" - id: "{{ item.id }}" - loop: "{{ droplets.data }}" - when: droplet_count != "0" +- name: Delete lingering droplets + community.digitalocean.digital_ocean_droplet: + state: absent + oauth_token: "{{ DO_API_TOKEN }}" + id: "{{ item.id }}" + loop: "{{ droplets.data }}" + when: droplet_count != "0" diff --git a/automated_builder/tasks/generate_inventory.yml b/automated_builder/tasks/generate_inventory.yml index f4bd863e1..05839f9ff 100644 --- a/automated_builder/tasks/generate_inventory.yml +++ b/automated_builder/tasks/generate_inventory.yml @@ -1,5 +1,5 @@ --- -- name: Configure local environment +- name: Generate DigitalOcean inventory hosts: 127.0.0.1 connection: local vars_files: @@ -7,7 +7,7 @@ - ../vars/secrets.yml tasks: - - name: Delete lingering inventory + - name: Delete inventory include_tasks: delete_inventory.yml - name: Create SSH key From 8bb5d9d6615ed5b5e09bf07d987157b3805710eb Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Tue, 15 Nov 2022 21:49:34 -0700 Subject: [PATCH 11/44] Rename VPS per DigitalOcean requirements --- automated_builder/tasks/generate_inventory.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated_builder/tasks/generate_inventory.yml b/automated_builder/tasks/generate_inventory.yml index 05839f9ff..39067719d 100644 --- a/automated_builder/tasks/generate_inventory.yml +++ b/automated_builder/tasks/generate_inventory.yml @@ -22,7 +22,7 @@ community.digitalocean.digital_ocean_droplet: state: present oauth_token: "{{ DO_API_TOKEN }}" - name: automated_builder_vps + name: automated-builder-vps size: s-8vcpu-16gb region: nyc3 image: debian-11-x64 From c89304b36c24c409cbb2d45ca3e4688d65cf2c77 Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Tue, 15 Nov 2022 22:07:29 -0700 Subject: [PATCH 12/44] Add teardown build script --- .github/workflows/run_automated_builder.yml | 15 ++++----------- automated_builder/scripts/teardown_build.sh | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 11 deletions(-) create mode 100644 automated_builder/scripts/teardown_build.sh diff --git a/.github/workflows/run_automated_builder.yml b/.github/workflows/run_automated_builder.yml index 6fff171a2..a7c5a2b58 100644 --- a/.github/workflows/run_automated_builder.yml +++ b/.github/workflows/run_automated_builder.yml @@ -42,14 +42,7 @@ jobs: # name: logs # path: ./automated_builder/logs/ - # - name: Destroy inventory - # if: always() - # run: | - # ./automated_builder/scripts/destroy_inventory.sh $ANSIBLE_VAULT_PASSWORD - - # - name: Upload artifacts - # uses: actions/upload-artifact@v3 - # if: always() - # with: - # name: logs - # path: ./automated_builder/logs/ + - name: Teardown build + if: always() + run: | + ./automated_builder/scripts/teardown_build.sh diff --git a/automated_builder/scripts/teardown_build.sh b/automated_builder/scripts/teardown_build.sh new file mode 100644 index 000000000..8efed42c5 --- /dev/null +++ b/automated_builder/scripts/teardown_build.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e +export ANSIBLE_VAULT_PASSWORD=$1 +export ANSIBLE_HOST_KEY_CHECKING=False +source ./automated_builder/scripts/functions.bash + +main() { + decrypt_vault + run_builder + encrypt_vault +} + +run_builder() { + ansible-playbook -i automated_builder/inventory automated_builder/tasks/build_vms.yml +} + +main From 99586761281eed4d6d9972b8f414da7337c1b340 Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Tue, 15 Nov 2022 22:10:26 -0700 Subject: [PATCH 13/44] Add execution permission to teardown script --- automated_builder/scripts/teardown_build.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 automated_builder/scripts/teardown_build.sh diff --git a/automated_builder/scripts/teardown_build.sh b/automated_builder/scripts/teardown_build.sh old mode 100644 new mode 100755 From efd96674ef6ae395194e15ebc8eae75c44f14035 Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Tue, 15 Nov 2022 22:15:57 -0700 Subject: [PATCH 14/44] Add local connection and vault password arguments to teardown functionality --- .github/workflows/run_automated_builder.yml | 2 +- automated_builder/scripts/teardown_build.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_automated_builder.yml b/.github/workflows/run_automated_builder.yml index a7c5a2b58..1f8f3c442 100644 --- a/.github/workflows/run_automated_builder.yml +++ b/.github/workflows/run_automated_builder.yml @@ -45,4 +45,4 @@ jobs: - name: Teardown build if: always() run: | - ./automated_builder/scripts/teardown_build.sh + ./automated_builder/scripts/teardown_build.sh $ANSIBLE_VAULT_PASSWORD diff --git a/automated_builder/scripts/teardown_build.sh b/automated_builder/scripts/teardown_build.sh index 8efed42c5..a08d21ca8 100755 --- a/automated_builder/scripts/teardown_build.sh +++ b/automated_builder/scripts/teardown_build.sh @@ -12,7 +12,7 @@ main() { } run_builder() { - ansible-playbook -i automated_builder/inventory automated_builder/tasks/build_vms.yml + ansible-playbook -i automated_builder/inventory automated_builder/tasks/build_vms.yml --connection=local } main From 4eacbcc441aaf7fda26fb9907dfd6bb2c77610f4 Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Tue, 15 Nov 2022 22:21:38 -0700 Subject: [PATCH 15/44] Add correct teardown task --- automated_builder/scripts/teardown_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated_builder/scripts/teardown_build.sh b/automated_builder/scripts/teardown_build.sh index a08d21ca8..f78001400 100755 --- a/automated_builder/scripts/teardown_build.sh +++ b/automated_builder/scripts/teardown_build.sh @@ -12,7 +12,7 @@ main() { } run_builder() { - ansible-playbook -i automated_builder/inventory automated_builder/tasks/build_vms.yml --connection=local + ansible-playbook -i automated_builder/tasks/delete_inventory.yml --connection=local } main From 9bad0e9725825f572165fe0da943cd6d1723a7da Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Tue, 15 Nov 2022 22:26:02 -0700 Subject: [PATCH 16/44] Remove unnecessary inventory flag --- automated_builder/scripts/teardown_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated_builder/scripts/teardown_build.sh b/automated_builder/scripts/teardown_build.sh index f78001400..54c46e2dd 100755 --- a/automated_builder/scripts/teardown_build.sh +++ b/automated_builder/scripts/teardown_build.sh @@ -12,7 +12,7 @@ main() { } run_builder() { - ansible-playbook -i automated_builder/tasks/delete_inventory.yml --connection=local + ansible-playbook automated_builder/tasks/delete_inventory.yml --connection=local } main From d76bae9bd39aea65931c2fe065737f83fe9525b2 Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Tue, 15 Nov 2022 22:32:04 -0700 Subject: [PATCH 17/44] Restructure delete inventory playbook --- .../scripts/run_automated_builder.sh | 3 +- automated_builder/scripts/teardown_build.sh | 2 +- automated_builder/tasks/delete_inventory.yml | 36 +++++++++++-------- .../tasks/generate_inventory.yml | 3 -- 4 files changed, 25 insertions(+), 19 deletions(-) diff --git a/automated_builder/scripts/run_automated_builder.sh b/automated_builder/scripts/run_automated_builder.sh index fd3bd5226..e19606484 100755 --- a/automated_builder/scripts/run_automated_builder.sh +++ b/automated_builder/scripts/run_automated_builder.sh @@ -12,7 +12,8 @@ main() { } run_builder() { - ansible-playbook -i automated_builder/inventory automated_builder/tasks/configure_local_environment.yml + ansible-playbook automated_builder/tasks/delete_inventory.yml + ansible-playbook automated_builder/tasks/configure_local_environment.yml ansible-galaxy collection install community.digitalocean ansible-playbook -i automated_builder/inventory automated_builder/tasks/generate_inventory.yml # ansible-playbook -i automated_builder/inventory automated_builder/tasks/build_vms.yml diff --git a/automated_builder/scripts/teardown_build.sh b/automated_builder/scripts/teardown_build.sh index 54c46e2dd..7aca708bc 100755 --- a/automated_builder/scripts/teardown_build.sh +++ b/automated_builder/scripts/teardown_build.sh @@ -12,7 +12,7 @@ main() { } run_builder() { - ansible-playbook automated_builder/tasks/delete_inventory.yml --connection=local + ansible-playbook automated_builder/tasks/delete_inventory.yml } main diff --git a/automated_builder/tasks/delete_inventory.yml b/automated_builder/tasks/delete_inventory.yml index ec5f40158..c27d13a28 100644 --- a/automated_builder/tasks/delete_inventory.yml +++ b/automated_builder/tasks/delete_inventory.yml @@ -1,17 +1,25 @@ --- -- name: Check for existing inventory - community.digitalocean.digital_ocean_droplet_info: - oauth_token: "{{ DO_API_TOKEN }}" - register: droplets +- name: Generate DigitalOcean inventory + hosts: 127.0.0.1 + connection: local + vars_files: + - ../vars/main.yml + - ../vars/secrets.yml -- name: Count existing droplets - set_fact: - droplet_count: "{{ droplets.data | length }}" + tasks: + - name: Check for existing inventory + community.digitalocean.digital_ocean_droplet_info: + oauth_token: "{{ DO_API_TOKEN }}" + register: droplets -- name: Delete lingering droplets - community.digitalocean.digital_ocean_droplet: - state: absent - oauth_token: "{{ DO_API_TOKEN }}" - id: "{{ item.id }}" - loop: "{{ droplets.data }}" - when: droplet_count != "0" + - name: Count existing droplets + set_fact: + droplet_count: "{{ droplets.data | length }}" + + - name: Delete lingering droplets + community.digitalocean.digital_ocean_droplet: + state: absent + oauth_token: "{{ DO_API_TOKEN }}" + id: "{{ item.id }}" + loop: "{{ droplets.data }}" + when: droplet_count != "0" diff --git a/automated_builder/tasks/generate_inventory.yml b/automated_builder/tasks/generate_inventory.yml index 39067719d..5cc84c376 100644 --- a/automated_builder/tasks/generate_inventory.yml +++ b/automated_builder/tasks/generate_inventory.yml @@ -7,9 +7,6 @@ - ../vars/secrets.yml tasks: - - name: Delete inventory - include_tasks: delete_inventory.yml - - name: Create SSH key community.digitalocean.digital_ocean_sshkey: oauth_token: "{{ DO_API_TOKEN }}" From 98e70106153fa6d198e6bd98a903ad7bf9a2cfa1 Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Wed, 16 Nov 2022 08:30:21 -0700 Subject: [PATCH 18/44] Add VPS_IP variable definition --- .../scripts/run_automated_builder.sh | 2 + automated_builder/tasks/bootstrap_vps.yml | 43 +++++++++++++++++++ .../tasks/generate_inventory.yml | 8 ++++ 3 files changed, 53 insertions(+) create mode 100644 automated_builder/tasks/bootstrap_vps.yml diff --git a/automated_builder/scripts/run_automated_builder.sh b/automated_builder/scripts/run_automated_builder.sh index e19606484..9e0ce2915 100755 --- a/automated_builder/scripts/run_automated_builder.sh +++ b/automated_builder/scripts/run_automated_builder.sh @@ -13,7 +13,9 @@ main() { run_builder() { ansible-playbook automated_builder/tasks/delete_inventory.yml + ansible-playbook automated_builder/tasks/generate_inventory.yml ansible-playbook automated_builder/tasks/configure_local_environment.yml + ansible-playbook automated_builder/tasks/bootstrap_vps.yml ansible-galaxy collection install community.digitalocean ansible-playbook -i automated_builder/inventory automated_builder/tasks/generate_inventory.yml # ansible-playbook -i automated_builder/inventory automated_builder/tasks/build_vms.yml diff --git a/automated_builder/tasks/bootstrap_vps.yml b/automated_builder/tasks/bootstrap_vps.yml new file mode 100644 index 000000000..5c247d10c --- /dev/null +++ b/automated_builder/tasks/bootstrap_vps.yml @@ -0,0 +1,43 @@ +# --- +# - name: Build VMs +# hosts: vps_runner +# gather_facts: true +# vars_files: +# - ../vars/secrets.yml +# - ../vars/main.yml + +# tasks: +# - name: Install dependencies +# block: +# - name: Install dependencies +# include_tasks: install_dependencies.yml +# become: true + +# - name: Install source and submodules +# block: +# - name: Install source +# include_tasks: install_source.yml + +# - name: Clean existing VirtualBox VMs +# block: +# - name: Clean existing vbox VMs +# include_tasks: clean_existing_vbox_vms.yml + +# - name: Create VMs from tag +# block: +# - name: Create VMs from tag +# include_tasks: create_vms_from_tag.yml +# when: REF_TYPE == 'tag' + +# - name: Create VMs from commit +# block: +# - name: Create VMs from commit +# include_tasks: create_vms_from_commit.yml +# when: REF_TYPE != 'tag' + +# - name: Start new VirtualBox VMs +# block: +# - name: Start new vbox VMs +# include_tasks: start_new_vbox_vms.yml +# environment: +# DISPLAY: "localhost:1" diff --git a/automated_builder/tasks/generate_inventory.yml b/automated_builder/tasks/generate_inventory.yml index 5cc84c376..cf8c4fe11 100644 --- a/automated_builder/tasks/generate_inventory.yml +++ b/automated_builder/tasks/generate_inventory.yml @@ -27,3 +27,11 @@ ssh_keys: ["{{ public_key.data.ssh_key.id }}"] project: "Automated Builder" register: automated_builder_vps + + - name: set VPS IP + set_fact: + VPS_IP: "{{ automated_builder_vps.data.droplet.networks.v4 | selectattr('type', 'equalto', 'public') | map(attribute='ip_address') | first }}" + + - name: Debug VPS_IP + debug: + var: VPS_IP From ce0315e0d2fd31130c4d8bd589f2b27b94fbd30f Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Wed, 16 Nov 2022 18:47:51 -0700 Subject: [PATCH 19/44] Create ansible user, allow ssh, and passwordless sudo --- .../scripts/run_automated_builder.sh | 5 +- automated_builder/tasks/bootstrap_vps.yml | 63 +++++++------------ .../tasks/generate_inventory.yml | 6 +- 3 files changed, 26 insertions(+), 48 deletions(-) diff --git a/automated_builder/scripts/run_automated_builder.sh b/automated_builder/scripts/run_automated_builder.sh index 9e0ce2915..1d4a26272 100755 --- a/automated_builder/scripts/run_automated_builder.sh +++ b/automated_builder/scripts/run_automated_builder.sh @@ -12,13 +12,12 @@ main() { } run_builder() { + ansible-galaxy collection install community.digitalocean ansible-playbook automated_builder/tasks/delete_inventory.yml ansible-playbook automated_builder/tasks/generate_inventory.yml ansible-playbook automated_builder/tasks/configure_local_environment.yml ansible-playbook automated_builder/tasks/bootstrap_vps.yml - ansible-galaxy collection install community.digitalocean - ansible-playbook -i automated_builder/inventory automated_builder/tasks/generate_inventory.yml - # ansible-playbook -i automated_builder/inventory automated_builder/tasks/build_vms.yml + ansible-playbook -i automated_builder/inventory automated_builder/tasks/build_vms.yml } main diff --git a/automated_builder/tasks/bootstrap_vps.yml b/automated_builder/tasks/bootstrap_vps.yml index 5c247d10c..41754376b 100644 --- a/automated_builder/tasks/bootstrap_vps.yml +++ b/automated_builder/tasks/bootstrap_vps.yml @@ -1,43 +1,26 @@ -# --- -# - name: Build VMs -# hosts: vps_runner -# gather_facts: true -# vars_files: -# - ../vars/secrets.yml -# - ../vars/main.yml +--- +- name: Bootstrap VPS + hosts: vps_runner + gather_facts: true + vars_files: + - ../vars/secrets.yml + - ../vars/main.yml -# tasks: -# - name: Install dependencies -# block: -# - name: Install dependencies -# include_tasks: install_dependencies.yml -# become: true + tasks: + - name: Create ansible user with SSH permissions + block: + - name: Create ansible user + user: + name: ansible + group: sudo -# - name: Install source and submodules -# block: -# - name: Install source -# include_tasks: install_source.yml + - name: Copy authorized_keys + copy: + remote_src: /root/.ssh/authorized_keys + dest: /home/ansible/authorized_keys -# - name: Clean existing VirtualBox VMs -# block: -# - name: Clean existing vbox VMs -# include_tasks: clean_existing_vbox_vms.yml - -# - name: Create VMs from tag -# block: -# - name: Create VMs from tag -# include_tasks: create_vms_from_tag.yml -# when: REF_TYPE == 'tag' - -# - name: Create VMs from commit -# block: -# - name: Create VMs from commit -# include_tasks: create_vms_from_commit.yml -# when: REF_TYPE != 'tag' - -# - name: Start new VirtualBox VMs -# block: -# - name: Start new vbox VMs -# include_tasks: start_new_vbox_vms.yml -# environment: -# DISPLAY: "localhost:1" + - name: Allow passwordless sudo commands + sudoers: + name: ansible + commands: all + become: true diff --git a/automated_builder/tasks/generate_inventory.yml b/automated_builder/tasks/generate_inventory.yml index cf8c4fe11..258a0e1dd 100644 --- a/automated_builder/tasks/generate_inventory.yml +++ b/automated_builder/tasks/generate_inventory.yml @@ -28,10 +28,6 @@ project: "Automated Builder" register: automated_builder_vps - - name: set VPS IP + - name: set VPS_IP set_fact: VPS_IP: "{{ automated_builder_vps.data.droplet.networks.v4 | selectattr('type', 'equalto', 'public') | map(attribute='ip_address') | first }}" - - - name: Debug VPS_IP - debug: - var: VPS_IP From b2aca0476570cfcc998699aebb9a4b8e46449da8 Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Wed, 16 Nov 2022 18:57:42 -0700 Subject: [PATCH 20/44] Temporarily change droplet size to account for account restrictions --- automated_builder/tasks/generate_inventory.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated_builder/tasks/generate_inventory.yml b/automated_builder/tasks/generate_inventory.yml index 258a0e1dd..dcbf16a49 100644 --- a/automated_builder/tasks/generate_inventory.yml +++ b/automated_builder/tasks/generate_inventory.yml @@ -20,7 +20,7 @@ state: present oauth_token: "{{ DO_API_TOKEN }}" name: automated-builder-vps - size: s-8vcpu-16gb + size: s-4vcpu-8gb region: nyc3 image: debian-11-x64 wait_timeout: 500 From 43631c5cd68d10e947df3c19137ac0e4f4963516 Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Wed, 16 Nov 2022 19:08:36 -0700 Subject: [PATCH 21/44] Add formatting for localhost and create user block --- automated_builder/tasks/bootstrap_vps.yml | 4 ++-- automated_builder/tasks/delete_inventory.yml | 3 +-- automated_builder/tasks/generate_inventory.yml | 1 - 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/automated_builder/tasks/bootstrap_vps.yml b/automated_builder/tasks/bootstrap_vps.yml index 41754376b..bd7914410 100644 --- a/automated_builder/tasks/bootstrap_vps.yml +++ b/automated_builder/tasks/bootstrap_vps.yml @@ -22,5 +22,5 @@ - name: Allow passwordless sudo commands sudoers: name: ansible - commands: all - become: true + commands: ALL + become_user: root diff --git a/automated_builder/tasks/delete_inventory.yml b/automated_builder/tasks/delete_inventory.yml index c27d13a28..7164f4e7e 100644 --- a/automated_builder/tasks/delete_inventory.yml +++ b/automated_builder/tasks/delete_inventory.yml @@ -1,7 +1,6 @@ --- -- name: Generate DigitalOcean inventory +- name: Delete DigitalOcean inventory hosts: 127.0.0.1 - connection: local vars_files: - ../vars/main.yml - ../vars/secrets.yml diff --git a/automated_builder/tasks/generate_inventory.yml b/automated_builder/tasks/generate_inventory.yml index dcbf16a49..8439355e0 100644 --- a/automated_builder/tasks/generate_inventory.yml +++ b/automated_builder/tasks/generate_inventory.yml @@ -1,7 +1,6 @@ --- - name: Generate DigitalOcean inventory hosts: 127.0.0.1 - connection: local vars_files: - ../vars/main.yml - ../vars/secrets.yml From 5e8e9eaf00d7f0efe0577175ea192f0cc52e3b1c Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Wed, 16 Nov 2022 19:22:26 -0700 Subject: [PATCH 22/44] Add community general install --- automated_builder/scripts/run_automated_builder.sh | 2 +- automated_builder/tasks/bootstrap_vps.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/automated_builder/scripts/run_automated_builder.sh b/automated_builder/scripts/run_automated_builder.sh index 1d4a26272..ce55ec6c6 100755 --- a/automated_builder/scripts/run_automated_builder.sh +++ b/automated_builder/scripts/run_automated_builder.sh @@ -12,7 +12,7 @@ main() { } run_builder() { - ansible-galaxy collection install community.digitalocean + ansible-galaxy collection install community.digitalocean community.general ansible-playbook automated_builder/tasks/delete_inventory.yml ansible-playbook automated_builder/tasks/generate_inventory.yml ansible-playbook automated_builder/tasks/configure_local_environment.yml diff --git a/automated_builder/tasks/bootstrap_vps.yml b/automated_builder/tasks/bootstrap_vps.yml index bd7914410..be67533cd 100644 --- a/automated_builder/tasks/bootstrap_vps.yml +++ b/automated_builder/tasks/bootstrap_vps.yml @@ -18,6 +18,8 @@ copy: remote_src: /root/.ssh/authorized_keys dest: /home/ansible/authorized_keys + owner: ansible + group: ansible - name: Allow passwordless sudo commands sudoers: From a0959479d67560d24c308b97ee8b0ec881c76e55 Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Sat, 19 Nov 2022 19:18:48 -0700 Subject: [PATCH 23/44] Restructure tasks to common role --- automated_builder/main.yml | 12 ++++++ .../roles/common/tasks/bootstrap_vps.yml | 21 +++++++++ .../common}/tasks/clean_existing_vbox_vms.yml | 0 .../tasks/configure_local_environment.yml | 13 ++---- .../common}/tasks/create_vms_from_commit.yml | 0 .../common}/tasks/create_vms_from_tag.yml | 0 .../common}/tasks/delete_inventory.yml | 10 ++--- .../common}/tasks/gather_build_logs.yml | 0 .../common}/tasks/generate_inventory.yml | 10 ++--- .../common/tasks/install_dependencies.yml | 22 ++++++++++ .../common}/tasks/install_source.yml | 0 automated_builder/roles/common/tasks/main.yml | 35 +++++++++++++++ .../roles/common/tasks/start_new_vbox_vms.yml | 16 +++++++ .../common}/templates/ansible_key.j2 | 0 .../common}/templates/ansible_key.pub.j2 | 0 .../templates/install_verified_source.sh | 0 .../{ => roles/common}/vars/main.yml | 0 .../{ => roles/common}/vars/secrets.yml | 0 .../scripts/run_automated_builder.sh | 6 +-- automated_builder/tasks/bootstrap_vps.yml | 28 ------------ automated_builder/tasks/build_vms.yml | 43 ------------------- .../tasks/install_dependencies.yml | 19 -------- .../tasks/start_new_vbox_vms.yml | 12 ------ 23 files changed, 117 insertions(+), 130 deletions(-) create mode 100644 automated_builder/main.yml create mode 100644 automated_builder/roles/common/tasks/bootstrap_vps.yml rename automated_builder/{ => roles/common}/tasks/clean_existing_vbox_vms.yml (100%) rename automated_builder/{ => roles/common}/tasks/configure_local_environment.yml (80%) rename automated_builder/{ => roles/common}/tasks/create_vms_from_commit.yml (100%) rename automated_builder/{ => roles/common}/tasks/create_vms_from_tag.yml (100%) rename automated_builder/{ => roles/common}/tasks/delete_inventory.yml (80%) rename automated_builder/{ => roles/common}/tasks/gather_build_logs.yml (100%) rename automated_builder/{ => roles/common}/tasks/generate_inventory.yml (86%) create mode 100644 automated_builder/roles/common/tasks/install_dependencies.yml rename automated_builder/{ => roles/common}/tasks/install_source.yml (100%) create mode 100644 automated_builder/roles/common/tasks/main.yml create mode 100644 automated_builder/roles/common/tasks/start_new_vbox_vms.yml rename automated_builder/{ => roles/common}/templates/ansible_key.j2 (100%) rename automated_builder/{ => roles/common}/templates/ansible_key.pub.j2 (100%) rename automated_builder/{ => roles/common}/templates/install_verified_source.sh (100%) rename automated_builder/{ => roles/common}/vars/main.yml (100%) rename automated_builder/{ => roles/common}/vars/secrets.yml (100%) delete mode 100644 automated_builder/tasks/bootstrap_vps.yml delete mode 100644 automated_builder/tasks/build_vms.yml delete mode 100644 automated_builder/tasks/install_dependencies.yml delete mode 100644 automated_builder/tasks/start_new_vbox_vms.yml diff --git a/automated_builder/main.yml b/automated_builder/main.yml new file mode 100644 index 000000000..27fb75cc3 --- /dev/null +++ b/automated_builder/main.yml @@ -0,0 +1,12 @@ +--- +- name: Run automated_builder + hosts: vps_runner + gather_facts: false + vars_files: + - ./roles/common/vars/main.yml + - ./roles/common/vars/secrets.yml + + tasks: + - name: Include common role + include_role: + name: common diff --git a/automated_builder/roles/common/tasks/bootstrap_vps.yml b/automated_builder/roles/common/tasks/bootstrap_vps.yml new file mode 100644 index 000000000..929ecb7bf --- /dev/null +++ b/automated_builder/roles/common/tasks/bootstrap_vps.yml @@ -0,0 +1,21 @@ +--- +- name: Bootstrap VPS + become: true + remote_user: root + block: + - name: Create ansible user + user: + name: ansible + group: sudo + + - name: Copy authorized_keys + copy: + remote_src: /root/.ssh/authorized_keys + dest: /home/ansible/authorized_keys + owner: ansible + group: ansible + + - name: Allow passwordless sudo commands + community.general.sudoers: + name: ansible + commands: ALL diff --git a/automated_builder/tasks/clean_existing_vbox_vms.yml b/automated_builder/roles/common/tasks/clean_existing_vbox_vms.yml similarity index 100% rename from automated_builder/tasks/clean_existing_vbox_vms.yml rename to automated_builder/roles/common/tasks/clean_existing_vbox_vms.yml diff --git a/automated_builder/tasks/configure_local_environment.yml b/automated_builder/roles/common/tasks/configure_local_environment.yml similarity index 80% rename from automated_builder/tasks/configure_local_environment.yml rename to automated_builder/roles/common/tasks/configure_local_environment.yml index 9de979d45..9e6ad81ee 100644 --- a/automated_builder/tasks/configure_local_environment.yml +++ b/automated_builder/roles/common/tasks/configure_local_environment.yml @@ -1,24 +1,19 @@ --- - name: Configure local environment - hosts: 127.0.0.1 - connection: local - vars_files: - - ../vars/main.yml - - ../vars/secrets.yml - - tasks: + delegate_to: 127.0.0.1 + block: - name: Create local_ssh directory file: path: ~/.ssh state: directory mode: 0700 - + - name: Add ansible user SSH key template: src: ../templates/ansible_key.j2 dest: ~/.ssh/ansible_key mode: 0600 - + - name: Add ansible user public key template: src: ../templates/ansible_key.pub.j2 diff --git a/automated_builder/tasks/create_vms_from_commit.yml b/automated_builder/roles/common/tasks/create_vms_from_commit.yml similarity index 100% rename from automated_builder/tasks/create_vms_from_commit.yml rename to automated_builder/roles/common/tasks/create_vms_from_commit.yml diff --git a/automated_builder/tasks/create_vms_from_tag.yml b/automated_builder/roles/common/tasks/create_vms_from_tag.yml similarity index 100% rename from automated_builder/tasks/create_vms_from_tag.yml rename to automated_builder/roles/common/tasks/create_vms_from_tag.yml diff --git a/automated_builder/tasks/delete_inventory.yml b/automated_builder/roles/common/tasks/delete_inventory.yml similarity index 80% rename from automated_builder/tasks/delete_inventory.yml rename to automated_builder/roles/common/tasks/delete_inventory.yml index 7164f4e7e..a11a6164e 100644 --- a/automated_builder/tasks/delete_inventory.yml +++ b/automated_builder/roles/common/tasks/delete_inventory.yml @@ -1,11 +1,7 @@ --- -- name: Delete DigitalOcean inventory - hosts: 127.0.0.1 - vars_files: - - ../vars/main.yml - - ../vars/secrets.yml - - tasks: +- name: Destroy any lingering droplets + delegate_to: 127.0.0.1 + block: - name: Check for existing inventory community.digitalocean.digital_ocean_droplet_info: oauth_token: "{{ DO_API_TOKEN }}" diff --git a/automated_builder/tasks/gather_build_logs.yml b/automated_builder/roles/common/tasks/gather_build_logs.yml similarity index 100% rename from automated_builder/tasks/gather_build_logs.yml rename to automated_builder/roles/common/tasks/gather_build_logs.yml diff --git a/automated_builder/tasks/generate_inventory.yml b/automated_builder/roles/common/tasks/generate_inventory.yml similarity index 86% rename from automated_builder/tasks/generate_inventory.yml rename to automated_builder/roles/common/tasks/generate_inventory.yml index 8439355e0..32ad54cbe 100644 --- a/automated_builder/tasks/generate_inventory.yml +++ b/automated_builder/roles/common/tasks/generate_inventory.yml @@ -1,11 +1,7 @@ --- -- name: Generate DigitalOcean inventory - hosts: 127.0.0.1 - vars_files: - - ../vars/main.yml - - ../vars/secrets.yml - - tasks: +- name: Create VPS resource + delegate_to: 127.0.0.1 + block: - name: Create SSH key community.digitalocean.digital_ocean_sshkey: oauth_token: "{{ DO_API_TOKEN }}" diff --git a/automated_builder/roles/common/tasks/install_dependencies.yml b/automated_builder/roles/common/tasks/install_dependencies.yml new file mode 100644 index 000000000..c5db637f6 --- /dev/null +++ b/automated_builder/roles/common/tasks/install_dependencies.yml @@ -0,0 +1,22 @@ +--- +- name: Install dependencies + become: true + block: + - name: Install apt packages + apt: + pkg: + - git + - time + - curl + - lsof + - apt-cacher-ng + - lsb-release + - fakeroot + - dpkg-dev + - fasttrack-archive-keyring + - dnsutils + update_cache: true + + - name: Upgrade apt packages + apt: + upgrade: full diff --git a/automated_builder/tasks/install_source.yml b/automated_builder/roles/common/tasks/install_source.yml similarity index 100% rename from automated_builder/tasks/install_source.yml rename to automated_builder/roles/common/tasks/install_source.yml diff --git a/automated_builder/roles/common/tasks/main.yml b/automated_builder/roles/common/tasks/main.yml new file mode 100644 index 000000000..655ae1593 --- /dev/null +++ b/automated_builder/roles/common/tasks/main.yml @@ -0,0 +1,35 @@ +--- +- name: Delete existing inventory + include_tasks: delete_inventory.yml + +- name: Generate inventory + include_tasks: generate_inventory.yml + +- name: Configure local environment + include_tasks: configure_local_environment.yml + +- name: Bootstrap VPS + include_tasks: bootstrap_vps.yml + +- name: Gathering facts + setup: + +- name: Install dependencies + include_tasks: install_dependencies.yml + +- name: Install source and submodules + include_tasks: install_source.yml + +- name: Clean existing VirtualBox VMs + include_tasks: clean_existing_vbox_vms.yml + +- name: Create VMs from tag + include_tasks: create_vms_from_tag.yml + when: REF_TYPE == 'tag' + +- name: Create VMs from commit + include_tasks: create_vms_from_commit.yml + when: REF_TYPE != 'tag' + +- name: Start new VirtualBox VMs + include_tasks: start_new_vbox_vms.yml diff --git a/automated_builder/roles/common/tasks/start_new_vbox_vms.yml b/automated_builder/roles/common/tasks/start_new_vbox_vms.yml new file mode 100644 index 000000000..1eb3248e4 --- /dev/null +++ b/automated_builder/roles/common/tasks/start_new_vbox_vms.yml @@ -0,0 +1,16 @@ +--- +- name: Start new VirtualBox VMs + environment: + DISPLAY: "localhost:1" + block: + - name: Start VNC display + shell: "vncserver" + + - name: Add xhost + shell: "xhost +" + + - name: Start gateway VM + shell: "VBoxManage startvm Whonix-Gateway-XFCE" + + - name: Start workstation VM + shell: "VBoxManage startvm Whonix-Workstation-XFCE" diff --git a/automated_builder/templates/ansible_key.j2 b/automated_builder/roles/common/templates/ansible_key.j2 similarity index 100% rename from automated_builder/templates/ansible_key.j2 rename to automated_builder/roles/common/templates/ansible_key.j2 diff --git a/automated_builder/templates/ansible_key.pub.j2 b/automated_builder/roles/common/templates/ansible_key.pub.j2 similarity index 100% rename from automated_builder/templates/ansible_key.pub.j2 rename to automated_builder/roles/common/templates/ansible_key.pub.j2 diff --git a/automated_builder/templates/install_verified_source.sh b/automated_builder/roles/common/templates/install_verified_source.sh similarity index 100% rename from automated_builder/templates/install_verified_source.sh rename to automated_builder/roles/common/templates/install_verified_source.sh diff --git a/automated_builder/vars/main.yml b/automated_builder/roles/common/vars/main.yml similarity index 100% rename from automated_builder/vars/main.yml rename to automated_builder/roles/common/vars/main.yml diff --git a/automated_builder/vars/secrets.yml b/automated_builder/roles/common/vars/secrets.yml similarity index 100% rename from automated_builder/vars/secrets.yml rename to automated_builder/roles/common/vars/secrets.yml diff --git a/automated_builder/scripts/run_automated_builder.sh b/automated_builder/scripts/run_automated_builder.sh index ce55ec6c6..14d270a03 100755 --- a/automated_builder/scripts/run_automated_builder.sh +++ b/automated_builder/scripts/run_automated_builder.sh @@ -13,11 +13,7 @@ main() { run_builder() { ansible-galaxy collection install community.digitalocean community.general - ansible-playbook automated_builder/tasks/delete_inventory.yml - ansible-playbook automated_builder/tasks/generate_inventory.yml - ansible-playbook automated_builder/tasks/configure_local_environment.yml - ansible-playbook automated_builder/tasks/bootstrap_vps.yml - ansible-playbook -i automated_builder/inventory automated_builder/tasks/build_vms.yml + ansible-playbook -i automated_builder/inventory automated_builder/main.yml } main diff --git a/automated_builder/tasks/bootstrap_vps.yml b/automated_builder/tasks/bootstrap_vps.yml deleted file mode 100644 index be67533cd..000000000 --- a/automated_builder/tasks/bootstrap_vps.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -- name: Bootstrap VPS - hosts: vps_runner - gather_facts: true - vars_files: - - ../vars/secrets.yml - - ../vars/main.yml - - tasks: - - name: Create ansible user with SSH permissions - block: - - name: Create ansible user - user: - name: ansible - group: sudo - - - name: Copy authorized_keys - copy: - remote_src: /root/.ssh/authorized_keys - dest: /home/ansible/authorized_keys - owner: ansible - group: ansible - - - name: Allow passwordless sudo commands - sudoers: - name: ansible - commands: ALL - become_user: root diff --git a/automated_builder/tasks/build_vms.yml b/automated_builder/tasks/build_vms.yml deleted file mode 100644 index 993e1a204..000000000 --- a/automated_builder/tasks/build_vms.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -- name: Build VMs - hosts: vps_runner - gather_facts: true - vars_files: - - ../vars/secrets.yml - - ../vars/main.yml - - tasks: - - name: Install dependencies - block: - - name: Install dependencies - include_tasks: install_dependencies.yml - become: true - - - name: Install source and submodules - block: - - name: Install source - include_tasks: install_source.yml - - - name: Clean existing VirtualBox VMs - block: - - name: Clean existing vbox VMs - include_tasks: clean_existing_vbox_vms.yml - - - name: Create VMs from tag - block: - - name: Create VMs from tag - include_tasks: create_vms_from_tag.yml - when: REF_TYPE == 'tag' - - - name: Create VMs from commit - block: - - name: Create VMs from commit - include_tasks: create_vms_from_commit.yml - when: REF_TYPE != 'tag' - - - name: Start new VirtualBox VMs - block: - - name: Start new vbox VMs - include_tasks: start_new_vbox_vms.yml - environment: - DISPLAY: "localhost:1" diff --git a/automated_builder/tasks/install_dependencies.yml b/automated_builder/tasks/install_dependencies.yml deleted file mode 100644 index 06fdb4b38..000000000 --- a/automated_builder/tasks/install_dependencies.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -- name: Install apt packages - apt: - pkg: - - git - - time - - curl - - lsof - - apt-cacher-ng - - lsb-release - - fakeroot - - dpkg-dev - - fasttrack-archive-keyring - - dnsutils - update_cache: true - -- name: Upgrade apt packages - apt: - upgrade: full diff --git a/automated_builder/tasks/start_new_vbox_vms.yml b/automated_builder/tasks/start_new_vbox_vms.yml deleted file mode 100644 index a90f5321b..000000000 --- a/automated_builder/tasks/start_new_vbox_vms.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- name: Start VNC display - shell: "vncserver" - -- name: Add xhost - shell: "xhost +" - -- name: Start gateway VM - shell: "VBoxManage startvm Whonix-Gateway-XFCE" - -- name: Start workstation VM - shell: "VBoxManage startvm Whonix-Workstation-XFCE" From 8e3ac54b91073e82f652c72279d2330776db8999 Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Sat, 19 Nov 2022 20:52:13 -0700 Subject: [PATCH 24/44] Refactor vars decryption and encryption paths --- automated_builder/scripts/functions.bash | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/automated_builder/scripts/functions.bash b/automated_builder/scripts/functions.bash index 1b8ab2002..e505bae26 100755 --- a/automated_builder/scripts/functions.bash +++ b/automated_builder/scripts/functions.bash @@ -4,7 +4,7 @@ decrypt_vault() { check_vault_value if [ "$ANSIBLE_VAULT_VALUE" == "ANSIBLE_VAULT" ]; then write_password - ansible-vault decrypt --vault-password-file ansible_vault_password automated_builder/vars/secrets.yml + ansible-vault decrypt --vault-password-file ansible_vault_password automated_builder/roles/common/vars/secrets.yml rm ansible_vault_password fi } @@ -13,7 +13,7 @@ encrypt_vault() { check_vault_value if [ "$ANSIBLE_VAULT_VALUE" != "ANSIBLE_VAULT" ]; then write_password - ansible-vault encrypt --vault-password-file ansible_vault_password automated_builder/vars/secrets.yml + ansible-vault encrypt --vault-password-file ansible_vault_password automated_builder/roles/common/vars/secrets.yml rm ansible_vault_password fi } @@ -23,5 +23,5 @@ write_password() { } check_vault_value() { - ANSIBLE_VAULT_VALUE=$(head -n 1 automated_builder/vars/secrets.yml | cut -d ';' -f1 | sed 's/\$//g') + ANSIBLE_VAULT_VALUE=$(head -n 1 automated_builder/roles/common/vars/secrets.yml | cut -d ';' -f1 | sed 's/\$//g') } From d0fceac2d9f5188128a5a98990aa1175750a1e10 Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Sat, 19 Nov 2022 20:59:26 -0700 Subject: [PATCH 25/44] Change ssh user for bootstrap play --- automated_builder/roles/common/tasks/bootstrap_vps.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/automated_builder/roles/common/tasks/bootstrap_vps.yml b/automated_builder/roles/common/tasks/bootstrap_vps.yml index 929ecb7bf..b537d6fbb 100644 --- a/automated_builder/roles/common/tasks/bootstrap_vps.yml +++ b/automated_builder/roles/common/tasks/bootstrap_vps.yml @@ -1,7 +1,8 @@ --- - name: Bootstrap VPS become: true - remote_user: root + vars: + ansible_ssh_user: root block: - name: Create ansible user user: From b92857314108b431c90175ba230ab0086749972b Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Sun, 20 Nov 2022 08:23:03 -0700 Subject: [PATCH 26/44] Restructure delete inventory play --- .github/workflows/run_automated_builder.yml | 66 +++++++++---------- .../common/tasks => }/delete_inventory.yml | 10 +-- .../roles/common/tasks/bootstrap_vps.yml | 7 ++ automated_builder/roles/common/tasks/main.yml | 3 - .../scripts/run_automated_builder.sh | 1 + automated_builder/scripts/teardown_build.sh | 2 +- 6 files changed, 48 insertions(+), 41 deletions(-) rename automated_builder/{roles/common/tasks => }/delete_inventory.yml (75%) diff --git a/.github/workflows/run_automated_builder.yml b/.github/workflows/run_automated_builder.yml index 1f8f3c442..ed3914cd8 100644 --- a/.github/workflows/run_automated_builder.yml +++ b/.github/workflows/run_automated_builder.yml @@ -13,36 +13,36 @@ jobs: ANSIBLE_VAULT_PASSWORD: ${{ secrets.ANSIBLE_VAULT_PASSWORD }} REPO_URL: ${{ github.repository }} steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Set up Python 3.8 - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install ansible - - - name: Run automated builder - run: | - ./automated_builder/scripts/run_automated_builder.sh $ANSIBLE_VAULT_PASSWORD - - # - name: Handle artifacts - # if: always() - # run: | - # ./automated_builder/scripts/handle_artifacts.sh $ANSIBLE_VAULT_PASSWORD - - # - name: Upload artifacts - # uses: actions/upload-artifact@v3 - # if: always() - # with: - # name: logs - # path: ./automated_builder/logs/ - - - name: Teardown build - if: always() - run: | - ./automated_builder/scripts/teardown_build.sh $ANSIBLE_VAULT_PASSWORD + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Python 3.8 + uses: actions/setup-python@v4 + with: + python-version: 3.8 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install ansible + + - name: Run automated builder + run: | + ./automated_builder/scripts/run_automated_builder.sh $ANSIBLE_VAULT_PASSWORD + + - name: Handle artifacts + if: always() + run: | + ./automated_builder/scripts/handle_artifacts.sh $ANSIBLE_VAULT_PASSWORD + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + if: always() + with: + name: logs + path: ./automated_builder/logs/ + + - name: Teardown build + if: always() + run: | + ./automated_builder/scripts/teardown_build.sh $ANSIBLE_VAULT_PASSWORD diff --git a/automated_builder/roles/common/tasks/delete_inventory.yml b/automated_builder/delete_inventory.yml similarity index 75% rename from automated_builder/roles/common/tasks/delete_inventory.yml rename to automated_builder/delete_inventory.yml index a11a6164e..24d42f4e7 100644 --- a/automated_builder/roles/common/tasks/delete_inventory.yml +++ b/automated_builder/delete_inventory.yml @@ -1,7 +1,9 @@ --- -- name: Destroy any lingering droplets - delegate_to: 127.0.0.1 - block: +- name: Destroy any existing droplets + hosts: 127.0.0.1 + vars_files: + - ./roles/common/vars/secrets.yml + tasks: - name: Check for existing inventory community.digitalocean.digital_ocean_droplet_info: oauth_token: "{{ DO_API_TOKEN }}" @@ -11,7 +13,7 @@ set_fact: droplet_count: "{{ droplets.data | length }}" - - name: Delete lingering droplets + - name: Delete existing droplets community.digitalocean.digital_ocean_droplet: state: absent oauth_token: "{{ DO_API_TOKEN }}" diff --git a/automated_builder/roles/common/tasks/bootstrap_vps.yml b/automated_builder/roles/common/tasks/bootstrap_vps.yml index b537d6fbb..ec0eab29f 100644 --- a/automated_builder/roles/common/tasks/bootstrap_vps.yml +++ b/automated_builder/roles/common/tasks/bootstrap_vps.yml @@ -9,6 +9,13 @@ name: ansible group: sudo + - name: register ssh directory + shell: "ls -la /root/.ssh" + register: ssh_contents + + - name: inspect ssh contents + debug: + var: ssh_contents - name: Copy authorized_keys copy: remote_src: /root/.ssh/authorized_keys diff --git a/automated_builder/roles/common/tasks/main.yml b/automated_builder/roles/common/tasks/main.yml index 655ae1593..290aaaa02 100644 --- a/automated_builder/roles/common/tasks/main.yml +++ b/automated_builder/roles/common/tasks/main.yml @@ -1,7 +1,4 @@ --- -- name: Delete existing inventory - include_tasks: delete_inventory.yml - - name: Generate inventory include_tasks: generate_inventory.yml diff --git a/automated_builder/scripts/run_automated_builder.sh b/automated_builder/scripts/run_automated_builder.sh index 14d270a03..c92a31704 100755 --- a/automated_builder/scripts/run_automated_builder.sh +++ b/automated_builder/scripts/run_automated_builder.sh @@ -13,6 +13,7 @@ main() { run_builder() { ansible-galaxy collection install community.digitalocean community.general + ansible-playbook automated_builder/delete_inventory.yml ansible-playbook -i automated_builder/inventory automated_builder/main.yml } diff --git a/automated_builder/scripts/teardown_build.sh b/automated_builder/scripts/teardown_build.sh index 7aca708bc..5d446c99a 100755 --- a/automated_builder/scripts/teardown_build.sh +++ b/automated_builder/scripts/teardown_build.sh @@ -12,7 +12,7 @@ main() { } run_builder() { - ansible-playbook automated_builder/tasks/delete_inventory.yml + ansible-playbook automated_builder/delete_inventory.yml } main From b97509b55b5c4cd8546042b7837ba1feef7dc209 Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Sun, 20 Nov 2022 09:01:40 -0700 Subject: [PATCH 27/44] Restructure ansible user authorized_keys copy --- .github/workflows/run_automated_builder.yml | 3 +-- .../roles/common/tasks/bootstrap_vps.yml | 26 ++++++++++++------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/.github/workflows/run_automated_builder.yml b/.github/workflows/run_automated_builder.yml index ed3914cd8..bc54a0cf1 100644 --- a/.github/workflows/run_automated_builder.yml +++ b/.github/workflows/run_automated_builder.yml @@ -1,7 +1,6 @@ +--- name: Run automated builder - on: push - concurrency: group: ${{ github.workflow }} cancel-in-progress: true diff --git a/automated_builder/roles/common/tasks/bootstrap_vps.yml b/automated_builder/roles/common/tasks/bootstrap_vps.yml index ec0eab29f..7fe0011d0 100644 --- a/automated_builder/roles/common/tasks/bootstrap_vps.yml +++ b/automated_builder/roles/common/tasks/bootstrap_vps.yml @@ -3,27 +3,33 @@ become: true vars: ansible_ssh_user: root + block: - name: Create ansible user user: name: ansible - group: sudo - - name: register ssh directory - shell: "ls -la /root/.ssh" - register: ssh_contents + - name: Create ansible user ssh directory + file: + path: /home/ansible/.ssh + state: directory + owner: ansible + group: ansible + mode: 0700 - - name: inspect ssh contents - debug: - var: ssh_contents - name: Copy authorized_keys copy: - remote_src: /root/.ssh/authorized_keys - dest: /home/ansible/authorized_keys + src: /root/.ssh/authorized_keys + dest: /home/ansible/.ssh/authorized_keys owner: ansible group: ansible + mode: 0600 + remote_src: true - name: Allow passwordless sudo commands community.general.sudoers: - name: ansible + name: ansible-passwordless-sudo + state: present + user: ansible commands: ALL + nopassword: true From 011458c65398bb30ff2c15ccb6c8d4fe4c5cc60b Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Sun, 20 Nov 2022 10:57:09 -0700 Subject: [PATCH 28/44] Install VirtualBox keys and repository --- .github/workflows/run_automated_builder.yml | 8 ++++---- .../common/tasks/install_dependencies.yml | 18 ++++++++++++++++-- .../roles/common/tasks/install_virtualbox.yml | 18 ++++++++++++++++++ automated_builder/roles/common/vars/main.yml | 3 +++ 4 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 automated_builder/roles/common/tasks/install_virtualbox.yml diff --git a/.github/workflows/run_automated_builder.yml b/.github/workflows/run_automated_builder.yml index bc54a0cf1..101dfef63 100644 --- a/.github/workflows/run_automated_builder.yml +++ b/.github/workflows/run_automated_builder.yml @@ -41,7 +41,7 @@ jobs: name: logs path: ./automated_builder/logs/ - - name: Teardown build - if: always() - run: | - ./automated_builder/scripts/teardown_build.sh $ANSIBLE_VAULT_PASSWORD + # - name: Teardown build + # if: always() + # run: | + # ./automated_builder/scripts/teardown_build.sh $ANSIBLE_VAULT_PASSWORD diff --git a/automated_builder/roles/common/tasks/install_dependencies.yml b/automated_builder/roles/common/tasks/install_dependencies.yml index c5db637f6..1863929df 100644 --- a/automated_builder/roles/common/tasks/install_dependencies.yml +++ b/automated_builder/roles/common/tasks/install_dependencies.yml @@ -15,8 +15,22 @@ - dpkg-dev - fasttrack-archive-keyring - dnsutils + - software-properties-common update_cache: true - - name: Upgrade apt packages + - name: Install VirtualBox + include_tasks: install_virtualbox.yml + + - name: Register LSB release + shell: "lsb_release -cs" + register: lsb_release + + - name: Add VirtualBox apt repository + apt_repository: + repo: "deb [arch=amd64 signed-by=/usr/share/keyrings/{{ VBOX_ASC_KEY.name }}.asc.gpg] http://download.virtualbox.org/virtualbox/debian {{ lsb_release.stdout }} contrib" + state: present + + - name: Install VirtualBox apt repository apt: - upgrade: full + name: "virtualbox-6.1" + update_cache: true diff --git a/automated_builder/roles/common/tasks/install_virtualbox.yml b/automated_builder/roles/common/tasks/install_virtualbox.yml new file mode 100644 index 000000000..155ce7588 --- /dev/null +++ b/automated_builder/roles/common/tasks/install_virtualbox.yml @@ -0,0 +1,18 @@ +--- +- name: Install VirtualBox + become: true + + block: + - name: Import VirtualBox GPG key + get_url: + dest: "/root/{{ VBOX_ASC_KEY.name }}.asc" + url: "{{ VBOX_ASC_KEY.url }}" + + - name: Create gpg file + shell: "gpg --dearmor /root/{{ VBOX_ASC_KEY.name }}.asc > {{ VBOX_ASC_KEY.name }}.gpg" + + - name: Move gpg key to shared keyrings + copy: + src: "/root/{{ VBOX_ASC_KEY.name }}.asc.gpg" + dest: "/usr/share/keyrings/{{ VBOX_ASC_KEY.name }}.asc.gpg" + remote_src: true diff --git a/automated_builder/roles/common/vars/main.yml b/automated_builder/roles/common/vars/main.yml index 67210bcac..4a704165d 100644 --- a/automated_builder/roles/common/vars/main.yml +++ b/automated_builder/roles/common/vars/main.yml @@ -1,3 +1,6 @@ REF_TYPE: "{{ lookup('env', 'GITHUB_REF_TYPE') }}" GIT_REPO: "{{ lookup('env', 'REPO_URL') }}" REF_NAME: "{{ lookup('env', 'GITHUB_REF_NAME') }}" +VBOX_ASC_KEY: + name: 'oracle_vbox_2016' + url: 'https://www.virtualbox.org/download/oracle_vbox_2016.asc' From 28ecdb1519e9e9303f81fc5512541f769fc67f0b Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Sun, 20 Nov 2022 13:31:52 -0700 Subject: [PATCH 29/44] Allow untagged builds for commit pipelines --- .../roles/common/tasks/create_vms_from_commit.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated_builder/roles/common/tasks/create_vms_from_commit.yml b/automated_builder/roles/common/tasks/create_vms_from_commit.yml index 2b2560665..dc4ca8f13 100644 --- a/automated_builder/roles/common/tasks/create_vms_from_commit.yml +++ b/automated_builder/roles/common/tasks/create_vms_from_commit.yml @@ -11,7 +11,7 @@ become: true - name: Build new gateway VM - shell: "dist_build_non_interactive=true /home/ansible/derivative-maker/derivative-maker --flavor whonix-gateway-xfce --target virtualbox --build --remote-derivative-packages true >> /home/ansible/build.log 2>&1" + shell: "dist_build_non_interactive=true /home/ansible/derivative-maker/derivative-maker --flavor whonix-gateway-xfce --target virtualbox --build --remote-derivative-packages true --allow-untagged true >> /home/ansible/build.log 2>&1" - name: Build new workstation VM - shell: "dist_build_non_interactive=true /home/ansible/derivative-maker/derivative-maker --flavor whonix-workstation-xfce --target virtualbox --build --remote-derivative-packages true >> /home/ansible/build.log 2>&1" + shell: "dist_build_non_interactive=true /home/ansible/derivative-maker/derivative-maker --flavor whonix-workstation-xfce --target virtualbox --build --remote-derivative-packages true --allow-untagged true >> /home/ansible/build.log 2>&1" From 0559050a0a264209bb2678417e661efbaea984bd Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Sun, 20 Nov 2022 13:52:51 -0700 Subject: [PATCH 30/44] Restructure gather build logs task --- .github/workflows/run_automated_builder.yml | 8 ++++---- .../roles/common/tasks/gather_build_logs.yml | 18 +++++++++++++++++- automated_builder/roles/common/tasks/main.yml | 2 +- automated_builder/scripts/handle_artifacts.sh | 2 +- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/.github/workflows/run_automated_builder.yml b/.github/workflows/run_automated_builder.yml index 101dfef63..bc54a0cf1 100644 --- a/.github/workflows/run_automated_builder.yml +++ b/.github/workflows/run_automated_builder.yml @@ -41,7 +41,7 @@ jobs: name: logs path: ./automated_builder/logs/ - # - name: Teardown build - # if: always() - # run: | - # ./automated_builder/scripts/teardown_build.sh $ANSIBLE_VAULT_PASSWORD + - name: Teardown build + if: always() + run: | + ./automated_builder/scripts/teardown_build.sh $ANSIBLE_VAULT_PASSWORD diff --git a/automated_builder/roles/common/tasks/gather_build_logs.yml b/automated_builder/roles/common/tasks/gather_build_logs.yml index 89765a056..96a511154 100644 --- a/automated_builder/roles/common/tasks/gather_build_logs.yml +++ b/automated_builder/roles/common/tasks/gather_build_logs.yml @@ -1,12 +1,28 @@ --- - name: Gather build logs hosts: vps_runner - gather_facts: true + gather_facts: false + vars: + VPS_IP: "{{ lookup('env', 'VPS_IP' )}}" vars_files: - ../vars/main.yml - ../vars/secrets.yml tasks: + - name: Get droplet IP + community.digitalocean.digital_ocean_droplet_info: + oauth_token: "{{ DO_API_TOKEN }}" + name: "automated-builder-vps" + delegate_to: localhost + register: automated_builder_vps + + - name: Set VPS_IP + set_fact: + VPS_IP: "{{ automated_builder_vps.data.networks.v4 | selectattr('type', 'equalto', 'public') | map(attribute='ip_address') | first }}" + + - name: Gather facts + setup: + - name: Copy logs fetch: src: "/home/ansible/{{ item }}" diff --git a/automated_builder/roles/common/tasks/main.yml b/automated_builder/roles/common/tasks/main.yml index 290aaaa02..320f2d61b 100644 --- a/automated_builder/roles/common/tasks/main.yml +++ b/automated_builder/roles/common/tasks/main.yml @@ -8,7 +8,7 @@ - name: Bootstrap VPS include_tasks: bootstrap_vps.yml -- name: Gathering facts +- name: Gather facts setup: - name: Install dependencies diff --git a/automated_builder/scripts/handle_artifacts.sh b/automated_builder/scripts/handle_artifacts.sh index b0a20264b..a7f294afb 100755 --- a/automated_builder/scripts/handle_artifacts.sh +++ b/automated_builder/scripts/handle_artifacts.sh @@ -12,7 +12,7 @@ main() { } gather_logs() { - ansible-playbook -i automated_builder/inventory automated_builder/tasks/gather_build_logs.yml + ansible-playbook -i automated_builder/inventory automated_builder/roles/common/tasks/gather_build_logs.yml } main From c436024294549e931c4b57a3cf1571bb4ef73686 Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Sun, 20 Nov 2022 16:53:04 -0700 Subject: [PATCH 31/44] Add folder structure for different build types --- automated_builder/main.yml | 20 ++++++++++++++++++ .../common/tasks}/delete_inventory.yml | 0 automated_builder/roles/common/tasks/main.yml | 11 ---------- .../tasks/create_vms_from_tag.yml | 0 .../roles/large-resource/tasks/main.yml | 21 +++++++++++++++++++ .../tasks/start_new_vbox_vms.yml | 0 .../tasks/create_vms_from_commit.yml | 0 .../roles/small-resource/tasks/main.yml | 21 +++++++++++++++++++ .../templates/build_small_resource_vms.sh | 0 .../scripts/run_automated_builder.sh | 2 +- automated_builder/scripts/teardown_build.sh | 2 +- 11 files changed, 64 insertions(+), 13 deletions(-) rename automated_builder/{ => roles/common/tasks}/delete_inventory.yml (100%) rename automated_builder/roles/{common => large-resource}/tasks/create_vms_from_tag.yml (100%) create mode 100644 automated_builder/roles/large-resource/tasks/main.yml rename automated_builder/roles/{common => large-resource}/tasks/start_new_vbox_vms.yml (100%) rename automated_builder/roles/{common => small-resource}/tasks/create_vms_from_commit.yml (100%) create mode 100644 automated_builder/roles/small-resource/tasks/main.yml create mode 100644 automated_builder/roles/small-resource/templates/build_small_resource_vms.sh diff --git a/automated_builder/main.yml b/automated_builder/main.yml index 27fb75cc3..bdf57fc57 100644 --- a/automated_builder/main.yml +++ b/automated_builder/main.yml @@ -10,3 +10,23 @@ - name: Include common role include_role: name: common + + - name: Include small-resource role + include_role: + name: small-resource + when: REF_TYPE != 'tag' + + - name: Include large-resource role + include_role: + name: small-resource + when: REF_TYPE == 'tag' +# - name: Create VMs from tag +# include_tasks: create_vms_from_tag.yml +# when: REF_TYPE == 'tag' + +# - name: Create VMs from commit +# include_tasks: create_vms_from_commit.yml +# when: REF_TYPE != 'tag' + +# - name: Start new VirtualBox VMs +# include_tasks: start_new_vbox_vms.yml diff --git a/automated_builder/delete_inventory.yml b/automated_builder/roles/common/tasks/delete_inventory.yml similarity index 100% rename from automated_builder/delete_inventory.yml rename to automated_builder/roles/common/tasks/delete_inventory.yml diff --git a/automated_builder/roles/common/tasks/main.yml b/automated_builder/roles/common/tasks/main.yml index 320f2d61b..52299665b 100644 --- a/automated_builder/roles/common/tasks/main.yml +++ b/automated_builder/roles/common/tasks/main.yml @@ -19,14 +19,3 @@ - name: Clean existing VirtualBox VMs include_tasks: clean_existing_vbox_vms.yml - -- name: Create VMs from tag - include_tasks: create_vms_from_tag.yml - when: REF_TYPE == 'tag' - -- name: Create VMs from commit - include_tasks: create_vms_from_commit.yml - when: REF_TYPE != 'tag' - -- name: Start new VirtualBox VMs - include_tasks: start_new_vbox_vms.yml diff --git a/automated_builder/roles/common/tasks/create_vms_from_tag.yml b/automated_builder/roles/large-resource/tasks/create_vms_from_tag.yml similarity index 100% rename from automated_builder/roles/common/tasks/create_vms_from_tag.yml rename to automated_builder/roles/large-resource/tasks/create_vms_from_tag.yml diff --git a/automated_builder/roles/large-resource/tasks/main.yml b/automated_builder/roles/large-resource/tasks/main.yml new file mode 100644 index 000000000..52299665b --- /dev/null +++ b/automated_builder/roles/large-resource/tasks/main.yml @@ -0,0 +1,21 @@ +--- +- name: Generate inventory + include_tasks: generate_inventory.yml + +- name: Configure local environment + include_tasks: configure_local_environment.yml + +- name: Bootstrap VPS + include_tasks: bootstrap_vps.yml + +- name: Gather facts + setup: + +- name: Install dependencies + include_tasks: install_dependencies.yml + +- name: Install source and submodules + include_tasks: install_source.yml + +- name: Clean existing VirtualBox VMs + include_tasks: clean_existing_vbox_vms.yml diff --git a/automated_builder/roles/common/tasks/start_new_vbox_vms.yml b/automated_builder/roles/large-resource/tasks/start_new_vbox_vms.yml similarity index 100% rename from automated_builder/roles/common/tasks/start_new_vbox_vms.yml rename to automated_builder/roles/large-resource/tasks/start_new_vbox_vms.yml diff --git a/automated_builder/roles/common/tasks/create_vms_from_commit.yml b/automated_builder/roles/small-resource/tasks/create_vms_from_commit.yml similarity index 100% rename from automated_builder/roles/common/tasks/create_vms_from_commit.yml rename to automated_builder/roles/small-resource/tasks/create_vms_from_commit.yml diff --git a/automated_builder/roles/small-resource/tasks/main.yml b/automated_builder/roles/small-resource/tasks/main.yml new file mode 100644 index 000000000..52299665b --- /dev/null +++ b/automated_builder/roles/small-resource/tasks/main.yml @@ -0,0 +1,21 @@ +--- +- name: Generate inventory + include_tasks: generate_inventory.yml + +- name: Configure local environment + include_tasks: configure_local_environment.yml + +- name: Bootstrap VPS + include_tasks: bootstrap_vps.yml + +- name: Gather facts + setup: + +- name: Install dependencies + include_tasks: install_dependencies.yml + +- name: Install source and submodules + include_tasks: install_source.yml + +- name: Clean existing VirtualBox VMs + include_tasks: clean_existing_vbox_vms.yml diff --git a/automated_builder/roles/small-resource/templates/build_small_resource_vms.sh b/automated_builder/roles/small-resource/templates/build_small_resource_vms.sh new file mode 100644 index 000000000..e69de29bb diff --git a/automated_builder/scripts/run_automated_builder.sh b/automated_builder/scripts/run_automated_builder.sh index c92a31704..32194d8a9 100755 --- a/automated_builder/scripts/run_automated_builder.sh +++ b/automated_builder/scripts/run_automated_builder.sh @@ -13,7 +13,7 @@ main() { run_builder() { ansible-galaxy collection install community.digitalocean community.general - ansible-playbook automated_builder/delete_inventory.yml + ansible-playbook automated_builder/roles/common/tasks/delete_inventory.yml ansible-playbook -i automated_builder/inventory automated_builder/main.yml } diff --git a/automated_builder/scripts/teardown_build.sh b/automated_builder/scripts/teardown_build.sh index 5d446c99a..9799f1d1e 100755 --- a/automated_builder/scripts/teardown_build.sh +++ b/automated_builder/scripts/teardown_build.sh @@ -12,7 +12,7 @@ main() { } run_builder() { - ansible-playbook automated_builder/delete_inventory.yml + ansible-playbook automated_builder/roles/common/tasks/delete_inventory.yml } main From 749a59849aa44aec02960b9509802d31d5040cb3 Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Sun, 20 Nov 2022 17:11:23 -0700 Subject: [PATCH 32/44] Fix broken VPS_IP assignment --- automated_builder/roles/common/tasks/delete_inventory.yml | 2 +- automated_builder/roles/common/tasks/gather_build_logs.yml | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/automated_builder/roles/common/tasks/delete_inventory.yml b/automated_builder/roles/common/tasks/delete_inventory.yml index 24d42f4e7..8605f67b6 100644 --- a/automated_builder/roles/common/tasks/delete_inventory.yml +++ b/automated_builder/roles/common/tasks/delete_inventory.yml @@ -2,7 +2,7 @@ - name: Destroy any existing droplets hosts: 127.0.0.1 vars_files: - - ./roles/common/vars/secrets.yml + - ../vars/secrets.yml tasks: - name: Check for existing inventory community.digitalocean.digital_ocean_droplet_info: diff --git a/automated_builder/roles/common/tasks/gather_build_logs.yml b/automated_builder/roles/common/tasks/gather_build_logs.yml index 96a511154..79d903ed5 100644 --- a/automated_builder/roles/common/tasks/gather_build_logs.yml +++ b/automated_builder/roles/common/tasks/gather_build_logs.yml @@ -16,9 +16,13 @@ delegate_to: localhost register: automated_builder_vps + - name: Debug VPS_IP setting + debug: + var: automated_builder_vps + - name: Set VPS_IP set_fact: - VPS_IP: "{{ automated_builder_vps.data.networks.v4 | selectattr('type', 'equalto', 'public') | map(attribute='ip_address') | first }}" + VPS_IP: "{{ automated_builder_vps.data[0].networks.v4 | selectattr('type', 'equalto', 'public') | map(attribute='ip_address') | first }}" - name: Gather facts setup: From 8cc580497284d12a528110b872fbc140b5c1e684 Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Sun, 20 Nov 2022 17:33:10 -0700 Subject: [PATCH 33/44] Restructure roles for small-resource builds --- .../roles/common/tasks/gather_build_logs.yml | 4 ++- .../tasks/create_vms_from_commit.yml | 11 +++++--- .../roles/small-resource/tasks/main.yml | 21 +++------------ .../templates/build_small_resource_vms.sh | 26 +++++++++++++++++++ 4 files changed, 39 insertions(+), 23 deletions(-) mode change 100644 => 100755 automated_builder/roles/small-resource/templates/build_small_resource_vms.sh diff --git a/automated_builder/roles/common/tasks/gather_build_logs.yml b/automated_builder/roles/common/tasks/gather_build_logs.yml index 79d903ed5..52142c2a0 100644 --- a/automated_builder/roles/common/tasks/gather_build_logs.yml +++ b/automated_builder/roles/common/tasks/gather_build_logs.yml @@ -28,9 +28,11 @@ setup: - name: Copy logs + ignore_errors: true fetch: src: "/home/ansible/{{ item }}" dest: "../logs/{{ item }}" with_items: - 'install_verified_source.log' - - 'build.log' + - 'gateway_build.log' + - 'workstation_build.log' diff --git a/automated_builder/roles/small-resource/tasks/create_vms_from_commit.yml b/automated_builder/roles/small-resource/tasks/create_vms_from_commit.yml index dc4ca8f13..1ed74a948 100644 --- a/automated_builder/roles/small-resource/tasks/create_vms_from_commit.yml +++ b/automated_builder/roles/small-resource/tasks/create_vms_from_commit.yml @@ -10,8 +10,11 @@ reboot_timeout: 60 become: true -- name: Build new gateway VM - shell: "dist_build_non_interactive=true /home/ansible/derivative-maker/derivative-maker --flavor whonix-gateway-xfce --target virtualbox --build --remote-derivative-packages true --allow-untagged true >> /home/ansible/build.log 2>&1" +- name: Install build_small_resource_vms script + template: + src: ../templates/build_small_resource_vms.sh + dest: /home/ansible/build_small_resource_vms.sh + mode: 0744 -- name: Build new workstation VM - shell: "dist_build_non_interactive=true /home/ansible/derivative-maker/derivative-maker --flavor whonix-workstation-xfce --target virtualbox --build --remote-derivative-packages true --allow-untagged true >> /home/ansible/build.log 2>&1" +- name: Run build_small_resource_vms scripts + shell: "/home/ansible/build_small_resource_vms.sh" diff --git a/automated_builder/roles/small-resource/tasks/main.yml b/automated_builder/roles/small-resource/tasks/main.yml index 52299665b..cc0722b3c 100644 --- a/automated_builder/roles/small-resource/tasks/main.yml +++ b/automated_builder/roles/small-resource/tasks/main.yml @@ -1,21 +1,6 @@ --- -- name: Generate inventory - include_tasks: generate_inventory.yml - -- name: Configure local environment - include_tasks: configure_local_environment.yml - -- name: Bootstrap VPS - include_tasks: bootstrap_vps.yml - -- name: Gather facts - setup: - -- name: Install dependencies - include_tasks: install_dependencies.yml - -- name: Install source and submodules - include_tasks: install_source.yml - - name: Clean existing VirtualBox VMs include_tasks: clean_existing_vbox_vms.yml + +- name: Copy build script + include_tasks: build_small_resource_vms.yml diff --git a/automated_builder/roles/small-resource/templates/build_small_resource_vms.sh b/automated_builder/roles/small-resource/templates/build_small_resource_vms.sh old mode 100644 new mode 100755 index e69de29bb..45d746e0d --- a/automated_builder/roles/small-resource/templates/build_small_resource_vms.sh +++ b/automated_builder/roles/small-resource/templates/build_small_resource_vms.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +export dist_build_non_interactive=true + +main() { + build_gateway_vm >> /home/ansible/build_gateway.log 2>&1 + build_workstation_vm >> /home/ansible/build_gateway.log 2>&1 +} + +build_gateway_vm() { + /home/ansible/derivative-maker/derivative-maker \ + --flavor whonix-gateway-xfce \ + --target virtualbox \ + --remote-derivative-packages true \ + --allow-untagged true \ + --build +} + +build_workstation_vm() { + /home/ansible/derivative-maker/derivative-maker \ + --flavor whonix-workstation-xfce \ + --target virtualbox \ + --remote-derivative-packages true \ + --allow-untagged true \ + --build +} From acbbff38a65da3cdb0dcfa02e16e2e671bc58ecb Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Fri, 25 Nov 2022 14:00:16 -0700 Subject: [PATCH 34/44] Remove redundant call of common role --- automated_builder/main.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/automated_builder/main.yml b/automated_builder/main.yml index bdf57fc57..7e1c7ec82 100644 --- a/automated_builder/main.yml +++ b/automated_builder/main.yml @@ -7,10 +7,6 @@ - ./roles/common/vars/secrets.yml tasks: - - name: Include common role - include_role: - name: common - - name: Include small-resource role include_role: name: small-resource From 5ccbd2529649c0c62c15ee5ca7a2f741d9e7135f Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Fri, 25 Nov 2022 14:05:16 -0700 Subject: [PATCH 35/44] Restructure small resource build roles --- automated_builder/main.yml | 14 ++++---------- .../roles/small-resource/tasks/main.yml | 5 +---- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/automated_builder/main.yml b/automated_builder/main.yml index 7e1c7ec82..33e8bc92a 100644 --- a/automated_builder/main.yml +++ b/automated_builder/main.yml @@ -7,6 +7,10 @@ - ./roles/common/vars/secrets.yml tasks: + - name: Include common role + include_role: + name: common + - name: Include small-resource role include_role: name: small-resource @@ -16,13 +20,3 @@ include_role: name: small-resource when: REF_TYPE == 'tag' -# - name: Create VMs from tag -# include_tasks: create_vms_from_tag.yml -# when: REF_TYPE == 'tag' - -# - name: Create VMs from commit -# include_tasks: create_vms_from_commit.yml -# when: REF_TYPE != 'tag' - -# - name: Start new VirtualBox VMs -# include_tasks: start_new_vbox_vms.yml diff --git a/automated_builder/roles/small-resource/tasks/main.yml b/automated_builder/roles/small-resource/tasks/main.yml index cc0722b3c..ee0c34440 100644 --- a/automated_builder/roles/small-resource/tasks/main.yml +++ b/automated_builder/roles/small-resource/tasks/main.yml @@ -1,6 +1,3 @@ --- -- name: Clean existing VirtualBox VMs - include_tasks: clean_existing_vbox_vms.yml - -- name: Copy build script +- name: Build small resource VMs include_tasks: build_small_resource_vms.yml From b77702ae590b5eacf73ca320b0d77948a3fa8cd2 Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Fri, 25 Nov 2022 14:10:17 -0700 Subject: [PATCH 36/44] Remove redundant cleanup logic and shutdown for loopbacks --- .../small-resource/tasks/create_vms_from_commit.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/automated_builder/roles/small-resource/tasks/create_vms_from_commit.yml b/automated_builder/roles/small-resource/tasks/create_vms_from_commit.yml index 1ed74a948..d85d6fcc9 100644 --- a/automated_builder/roles/small-resource/tasks/create_vms_from_commit.yml +++ b/automated_builder/roles/small-resource/tasks/create_vms_from_commit.yml @@ -1,15 +1,4 @@ --- -- name: Clean existing gateway VM - shell: "dist_build_non_interactive=true /home/ansible/derivative-maker/derivative-maker --flavor whonix-gateway-xfce --target virtualbox --clean > /home/ansible/build.log 2>&1" - -- name: Clean existing workstation VM - shell: "dist_build_non_interactive=true /home/ansible/derivative-maker/derivative-maker --flavor whonix-workstation-xfce --target virtualbox --clean >> /home/ansible/build.log 2>&1" - -- name: Reboot VPS for stray loop devices - reboot: - reboot_timeout: 60 - become: true - - name: Install build_small_resource_vms script template: src: ../templates/build_small_resource_vms.sh From e27fa4898dc5353f537020cc8a8d8b96611d291a Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Fri, 25 Nov 2022 14:20:22 -0700 Subject: [PATCH 37/44] Restructure log logic --- automated_builder/gather_build_logs.yml | 43 +++++++++++++++++++ .../roles/common/tasks/gather_build_logs.yml | 38 ---------------- .../templates/build_small_resource_vms.sh | 4 +- automated_builder/scripts/handle_artifacts.sh | 2 +- 4 files changed, 46 insertions(+), 41 deletions(-) create mode 100644 automated_builder/gather_build_logs.yml delete mode 100644 automated_builder/roles/common/tasks/gather_build_logs.yml diff --git a/automated_builder/gather_build_logs.yml b/automated_builder/gather_build_logs.yml new file mode 100644 index 000000000..f93a11865 --- /dev/null +++ b/automated_builder/gather_build_logs.yml @@ -0,0 +1,43 @@ +--- +- name: Gather build logs + hosts: vps_runner + gather_facts: false + vars_files: + - ./roles/common/vars/main.yml + - ./roles/common/vars/secrets.yml + + tasks: + - name: Get droplet IP + community.digitalocean.digital_ocean_droplet_info: + oauth_token: "{{ DO_API_TOKEN }}" + name: "automated-builder-vps" + delegate_to: localhost + register: automated_builder_vps + + - name: Set VPS_IP + set_fact: + VPS_IP: "{{ automated_builder_vps.data[0].networks.v4 | selectattr('type', 'equalto', 'public') | map(attribute='ip_address') | first }}" + delegate_to: localhost + + - name: Gather facts + setup: + + - name: Copy install_verified_source log + fetch: + src: "/home/ansible/install_verified_source.log" + dest: "./logs/install_verified_source.log" + + - name: Check logs + shell: "ls" + register: pwd + delegate_to: 127.0.0.1 + + - name: Copy gateway_build log + fetch: + src: "/home/ansible/gateway_build.log" + dest: "./logs/gateway_build.log" + + - name: Copy workstation_build log + fetch: + src: "/home/ansible/workstation_build.log" + dest: "./logs/workstation_build.log" diff --git a/automated_builder/roles/common/tasks/gather_build_logs.yml b/automated_builder/roles/common/tasks/gather_build_logs.yml deleted file mode 100644 index 52142c2a0..000000000 --- a/automated_builder/roles/common/tasks/gather_build_logs.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- -- name: Gather build logs - hosts: vps_runner - gather_facts: false - vars: - VPS_IP: "{{ lookup('env', 'VPS_IP' )}}" - vars_files: - - ../vars/main.yml - - ../vars/secrets.yml - - tasks: - - name: Get droplet IP - community.digitalocean.digital_ocean_droplet_info: - oauth_token: "{{ DO_API_TOKEN }}" - name: "automated-builder-vps" - delegate_to: localhost - register: automated_builder_vps - - - name: Debug VPS_IP setting - debug: - var: automated_builder_vps - - - name: Set VPS_IP - set_fact: - VPS_IP: "{{ automated_builder_vps.data[0].networks.v4 | selectattr('type', 'equalto', 'public') | map(attribute='ip_address') | first }}" - - - name: Gather facts - setup: - - - name: Copy logs - ignore_errors: true - fetch: - src: "/home/ansible/{{ item }}" - dest: "../logs/{{ item }}" - with_items: - - 'install_verified_source.log' - - 'gateway_build.log' - - 'workstation_build.log' diff --git a/automated_builder/roles/small-resource/templates/build_small_resource_vms.sh b/automated_builder/roles/small-resource/templates/build_small_resource_vms.sh index 45d746e0d..0c17b8195 100755 --- a/automated_builder/roles/small-resource/templates/build_small_resource_vms.sh +++ b/automated_builder/roles/small-resource/templates/build_small_resource_vms.sh @@ -3,8 +3,8 @@ export dist_build_non_interactive=true main() { - build_gateway_vm >> /home/ansible/build_gateway.log 2>&1 - build_workstation_vm >> /home/ansible/build_gateway.log 2>&1 + build_gateway_vm >> /home/ansible/gateway_build.log 2>&1 + build_workstation_vm >> /home/ansible/workstation_build.log 2>&1 } build_gateway_vm() { diff --git a/automated_builder/scripts/handle_artifacts.sh b/automated_builder/scripts/handle_artifacts.sh index a7f294afb..a4e62fb36 100755 --- a/automated_builder/scripts/handle_artifacts.sh +++ b/automated_builder/scripts/handle_artifacts.sh @@ -12,7 +12,7 @@ main() { } gather_logs() { - ansible-playbook -i automated_builder/inventory automated_builder/roles/common/tasks/gather_build_logs.yml + ansible-playbook -i automated_builder/inventory automated_builder/gather_build_logs.yml } main From 485e51cf800781a00cdc832937e93a24de501837 Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Fri, 25 Nov 2022 19:57:08 -0700 Subject: [PATCH 38/44] Rename build roles for clarity --- automated_builder/main.yml | 6 +++--- automated_builder/roles/common/tasks/main.yml | 1 + .../tasks/create_vms_from_tag.yml | 0 .../roles/{large-resource => gui-build}/tasks/main.yml | 0 .../tasks/start_new_vbox_vms.yml | 0 .../roles/headless-build/tasks/build_vms_from_commit.yml | 9 +++++++++ automated_builder/roles/headless-build/tasks/main.yml | 3 +++ .../templates/build_vms_from_commit.sh} | 0 .../small-resource/tasks/create_vms_from_commit.yml | 9 --------- automated_builder/roles/small-resource/tasks/main.yml | 3 --- 10 files changed, 16 insertions(+), 15 deletions(-) rename automated_builder/roles/{large-resource => gui-build}/tasks/create_vms_from_tag.yml (100%) rename automated_builder/roles/{large-resource => gui-build}/tasks/main.yml (100%) rename automated_builder/roles/{large-resource => gui-build}/tasks/start_new_vbox_vms.yml (100%) create mode 100644 automated_builder/roles/headless-build/tasks/build_vms_from_commit.yml create mode 100644 automated_builder/roles/headless-build/tasks/main.yml rename automated_builder/roles/{small-resource/templates/build_small_resource_vms.sh => headless-build/templates/build_vms_from_commit.sh} (100%) delete mode 100644 automated_builder/roles/small-resource/tasks/create_vms_from_commit.yml delete mode 100644 automated_builder/roles/small-resource/tasks/main.yml diff --git a/automated_builder/main.yml b/automated_builder/main.yml index 33e8bc92a..80dbd2e32 100644 --- a/automated_builder/main.yml +++ b/automated_builder/main.yml @@ -11,12 +11,12 @@ include_role: name: common - - name: Include small-resource role + - name: Include headless-build role include_role: - name: small-resource + name: headless-build when: REF_TYPE != 'tag' - name: Include large-resource role include_role: - name: small-resource + name: gui-build when: REF_TYPE == 'tag' diff --git a/automated_builder/roles/common/tasks/main.yml b/automated_builder/roles/common/tasks/main.yml index 52299665b..9ebef39cc 100644 --- a/automated_builder/roles/common/tasks/main.yml +++ b/automated_builder/roles/common/tasks/main.yml @@ -14,6 +14,7 @@ - name: Install dependencies include_tasks: install_dependencies.yml +# TODO: break in to small resource and large resource roles - name: Install source and submodules include_tasks: install_source.yml diff --git a/automated_builder/roles/large-resource/tasks/create_vms_from_tag.yml b/automated_builder/roles/gui-build/tasks/create_vms_from_tag.yml similarity index 100% rename from automated_builder/roles/large-resource/tasks/create_vms_from_tag.yml rename to automated_builder/roles/gui-build/tasks/create_vms_from_tag.yml diff --git a/automated_builder/roles/large-resource/tasks/main.yml b/automated_builder/roles/gui-build/tasks/main.yml similarity index 100% rename from automated_builder/roles/large-resource/tasks/main.yml rename to automated_builder/roles/gui-build/tasks/main.yml diff --git a/automated_builder/roles/large-resource/tasks/start_new_vbox_vms.yml b/automated_builder/roles/gui-build/tasks/start_new_vbox_vms.yml similarity index 100% rename from automated_builder/roles/large-resource/tasks/start_new_vbox_vms.yml rename to automated_builder/roles/gui-build/tasks/start_new_vbox_vms.yml diff --git a/automated_builder/roles/headless-build/tasks/build_vms_from_commit.yml b/automated_builder/roles/headless-build/tasks/build_vms_from_commit.yml new file mode 100644 index 000000000..d2a55a1ac --- /dev/null +++ b/automated_builder/roles/headless-build/tasks/build_vms_from_commit.yml @@ -0,0 +1,9 @@ +--- +- name: Install build_vms_from_commit script + template: + src: ../templates/build_vms_from_commit.sh + dest: /home/ansible/build_vms_from_commit.sh + mode: 0744 + +- name: Run build_vms_from_commit scripts + shell: "/home/ansible/build_vms_from_commit.sh" diff --git a/automated_builder/roles/headless-build/tasks/main.yml b/automated_builder/roles/headless-build/tasks/main.yml new file mode 100644 index 000000000..f53d410a1 --- /dev/null +++ b/automated_builder/roles/headless-build/tasks/main.yml @@ -0,0 +1,3 @@ +--- +- name: Build VMs from commit + include_tasks: build_vms_from_commit.yml diff --git a/automated_builder/roles/small-resource/templates/build_small_resource_vms.sh b/automated_builder/roles/headless-build/templates/build_vms_from_commit.sh similarity index 100% rename from automated_builder/roles/small-resource/templates/build_small_resource_vms.sh rename to automated_builder/roles/headless-build/templates/build_vms_from_commit.sh diff --git a/automated_builder/roles/small-resource/tasks/create_vms_from_commit.yml b/automated_builder/roles/small-resource/tasks/create_vms_from_commit.yml deleted file mode 100644 index d85d6fcc9..000000000 --- a/automated_builder/roles/small-resource/tasks/create_vms_from_commit.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- name: Install build_small_resource_vms script - template: - src: ../templates/build_small_resource_vms.sh - dest: /home/ansible/build_small_resource_vms.sh - mode: 0744 - -- name: Run build_small_resource_vms scripts - shell: "/home/ansible/build_small_resource_vms.sh" diff --git a/automated_builder/roles/small-resource/tasks/main.yml b/automated_builder/roles/small-resource/tasks/main.yml deleted file mode 100644 index ee0c34440..000000000 --- a/automated_builder/roles/small-resource/tasks/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -- name: Build small resource VMs - include_tasks: build_small_resource_vms.yml From 977c8fdc5759041219b2dd62cc09d5a4fa612203 Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Sat, 26 Nov 2022 10:01:43 -0700 Subject: [PATCH 39/44] Call build VM script main function --- .../roles/headless-build/templates/build_vms_from_commit.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/automated_builder/roles/headless-build/templates/build_vms_from_commit.sh b/automated_builder/roles/headless-build/templates/build_vms_from_commit.sh index 0c17b8195..b88ef2688 100755 --- a/automated_builder/roles/headless-build/templates/build_vms_from_commit.sh +++ b/automated_builder/roles/headless-build/templates/build_vms_from_commit.sh @@ -24,3 +24,5 @@ build_workstation_vm() { --allow-untagged true \ --build } + +main From 18cae247e01e0561b9dfe2021d4d7f4e074f6c03 Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Sat, 26 Nov 2022 10:18:35 -0700 Subject: [PATCH 40/44] Remove whitespace --- .../roles/common/tasks/configure_local_environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated_builder/roles/common/tasks/configure_local_environment.yml b/automated_builder/roles/common/tasks/configure_local_environment.yml index 9e6ad81ee..e640dff24 100644 --- a/automated_builder/roles/common/tasks/configure_local_environment.yml +++ b/automated_builder/roles/common/tasks/configure_local_environment.yml @@ -7,13 +7,13 @@ path: ~/.ssh state: directory mode: 0700 - + - name: Add ansible user SSH key template: src: ../templates/ansible_key.j2 dest: ~/.ssh/ansible_key mode: 0600 - + - name: Add ansible user public key template: src: ../templates/ansible_key.pub.j2 From 95e0cbbb1dddd3b8079073b3b61c1b4503e3440e Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Sat, 26 Nov 2022 17:04:45 -0700 Subject: [PATCH 41/44] Refactor source install --- automated_builder/gather_build_logs.yml | 6 ++--- .../roles/common/tasks/generate_inventory.yml | 2 +- .../roles/common/tasks/install_source.yml | 14 ----------- automated_builder/roles/common/tasks/main.yml | 10 +++----- ...l_verified_source.sh => install_source.sh} | 0 .../gui-build/tasks/build_vms_from_tag.yml | 9 +++++++ .../gui-build/tasks/create_vms_from_tag.yml | 17 ------------- .../roles/gui-build/tasks/install_source.yml | 9 +++++++ .../roles/gui-build/tasks/main.yml | 21 +++------------- .../gui-build/templates/build_vms_from_tag.sh | 24 +++++++++++++++++++ .../headless-build/tasks/install_source.yml | 9 +++++++ .../roles/headless-build/tasks/main.yml | 3 +++ 12 files changed, 64 insertions(+), 60 deletions(-) delete mode 100644 automated_builder/roles/common/tasks/install_source.yml rename automated_builder/roles/common/templates/{install_verified_source.sh => install_source.sh} (100%) create mode 100644 automated_builder/roles/gui-build/tasks/build_vms_from_tag.yml delete mode 100644 automated_builder/roles/gui-build/tasks/create_vms_from_tag.yml create mode 100644 automated_builder/roles/gui-build/tasks/install_source.yml create mode 100755 automated_builder/roles/gui-build/templates/build_vms_from_tag.sh create mode 100644 automated_builder/roles/headless-build/tasks/install_source.yml diff --git a/automated_builder/gather_build_logs.yml b/automated_builder/gather_build_logs.yml index f93a11865..28a8651c3 100644 --- a/automated_builder/gather_build_logs.yml +++ b/automated_builder/gather_build_logs.yml @@ -22,10 +22,10 @@ - name: Gather facts setup: - - name: Copy install_verified_source log + - name: Copy install_source log fetch: - src: "/home/ansible/install_verified_source.log" - dest: "./logs/install_verified_source.log" + src: "/home/ansible/install_source.log" + dest: "./logs/install_source.log" - name: Check logs shell: "ls" diff --git a/automated_builder/roles/common/tasks/generate_inventory.yml b/automated_builder/roles/common/tasks/generate_inventory.yml index 32ad54cbe..ad055956e 100644 --- a/automated_builder/roles/common/tasks/generate_inventory.yml +++ b/automated_builder/roles/common/tasks/generate_inventory.yml @@ -15,7 +15,7 @@ state: present oauth_token: "{{ DO_API_TOKEN }}" name: automated-builder-vps - size: s-4vcpu-8gb + size: s-8vcpu-16gb region: nyc3 image: debian-11-x64 wait_timeout: 500 diff --git a/automated_builder/roles/common/tasks/install_source.yml b/automated_builder/roles/common/tasks/install_source.yml deleted file mode 100644 index fb1f9937b..000000000 --- a/automated_builder/roles/common/tasks/install_source.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -- name: Create install_verified_source script - template: - src: ../templates/install_verified_source.sh - dest: /home/ansible/install_verified_source.sh - mode: 0744 - -- name: Run install_verified_source script for tag - shell: "/home/ansible/install_verified_source.sh {{ GIT_REPO }} {{ REF_NAME }} {{ REF_NAME }} > /home/ansible/install_verified_source.log 2>&1" - when: REF_TYPE == 'tag' - -- name: Run install_verified_source script for commit - shell: "/home/ansible/install_verified_source.sh {{ GIT_REPO }} {{ REF_NAME }} > /home/ansible/install_verified_source.log 2>&1" - when: REF_TYPE != 'tag' diff --git a/automated_builder/roles/common/tasks/main.yml b/automated_builder/roles/common/tasks/main.yml index 9ebef39cc..c525c069c 100644 --- a/automated_builder/roles/common/tasks/main.yml +++ b/automated_builder/roles/common/tasks/main.yml @@ -1,10 +1,10 @@ --- -- name: Generate inventory - include_tasks: generate_inventory.yml - - name: Configure local environment include_tasks: configure_local_environment.yml +- name: Generate inventory + include_tasks: generate_inventory.yml + - name: Bootstrap VPS include_tasks: bootstrap_vps.yml @@ -14,9 +14,5 @@ - name: Install dependencies include_tasks: install_dependencies.yml -# TODO: break in to small resource and large resource roles -- name: Install source and submodules - include_tasks: install_source.yml - - name: Clean existing VirtualBox VMs include_tasks: clean_existing_vbox_vms.yml diff --git a/automated_builder/roles/common/templates/install_verified_source.sh b/automated_builder/roles/common/templates/install_source.sh similarity index 100% rename from automated_builder/roles/common/templates/install_verified_source.sh rename to automated_builder/roles/common/templates/install_source.sh diff --git a/automated_builder/roles/gui-build/tasks/build_vms_from_tag.yml b/automated_builder/roles/gui-build/tasks/build_vms_from_tag.yml new file mode 100644 index 000000000..340c9d575 --- /dev/null +++ b/automated_builder/roles/gui-build/tasks/build_vms_from_tag.yml @@ -0,0 +1,9 @@ +--- +- name: Install build_vms_from_tag script + template: + src: ../templates/build_vms_from_tag.sh + dest: /home/ansible/build_vms_from_tag.sh + mode: 0744 + +- name: Run build_vms_from_tag scripts + shell: "/home/ansible/build_vms_from_tag.sh" diff --git a/automated_builder/roles/gui-build/tasks/create_vms_from_tag.yml b/automated_builder/roles/gui-build/tasks/create_vms_from_tag.yml deleted file mode 100644 index df01cd544..000000000 --- a/automated_builder/roles/gui-build/tasks/create_vms_from_tag.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -- name: Clean existing gateway VM - shell: "dist_build_non_interactive=true /home/ansible/derivative-maker/derivative-maker --flavor whonix-gateway-xfce --target virtualbox --clean > /home/ansible/build.log 2>&1" - -- name: Clean existing workstation VM - shell: "dist_build_non_interactive=true /home/ansible/derivative-maker/derivative-maker --flavor whonix-workstation-xfce --target virtualbox --clean >> /home/ansible/build.log 2>&1" - -- name: Reboot VPS for stray loop devices - reboot: - reboot_timeout: 60 - become: true - -- name: Build new gateway VM - shell: "dist_build_non_interactive=true /home/ansible/derivative-maker/derivative-maker --flavor whonix-gateway-xfce --target virtualbox --build >> /home/ansible/build.log 2>&1" - -- name: Build new workstation VM - shell: "dist_build_non_interactive=true /home/ansible/derivative-maker/derivative-maker --flavor whonix-workstation-xfce --target virtualbox --build >> /home/ansible/build.log 2>&1" diff --git a/automated_builder/roles/gui-build/tasks/install_source.yml b/automated_builder/roles/gui-build/tasks/install_source.yml new file mode 100644 index 000000000..d42c2c814 --- /dev/null +++ b/automated_builder/roles/gui-build/tasks/install_source.yml @@ -0,0 +1,9 @@ +--- +- name: Create install_source script + template: + src: ./roles/common/templates/install_source.sh + dest: /home/ansible/install_source.sh + mode: 0744 + +- name: Run install_source script for tag + shell: "/home/ansible/install_source.sh {{ GIT_REPO }} {{ REF_NAME }} {{ REF_NAME }} > /home/ansible/install_source.log 2>&1" diff --git a/automated_builder/roles/gui-build/tasks/main.yml b/automated_builder/roles/gui-build/tasks/main.yml index 52299665b..cd383d1ce 100644 --- a/automated_builder/roles/gui-build/tasks/main.yml +++ b/automated_builder/roles/gui-build/tasks/main.yml @@ -1,21 +1,6 @@ --- -- name: Generate inventory - include_tasks: generate_inventory.yml - -- name: Configure local environment - include_tasks: configure_local_environment.yml - -- name: Bootstrap VPS - include_tasks: bootstrap_vps.yml - -- name: Gather facts - setup: - -- name: Install dependencies - include_tasks: install_dependencies.yml - -- name: Install source and submodules +- name: Install source code for commit include_tasks: install_source.yml -- name: Clean existing VirtualBox VMs - include_tasks: clean_existing_vbox_vms.yml +- name: Build VMs from tag + include_tasks: build_vms_from_tag.yml diff --git a/automated_builder/roles/gui-build/templates/build_vms_from_tag.sh b/automated_builder/roles/gui-build/templates/build_vms_from_tag.sh new file mode 100755 index 000000000..b1619665b --- /dev/null +++ b/automated_builder/roles/gui-build/templates/build_vms_from_tag.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +export dist_build_non_interactive=true + +main() { + build_gateway_vm >> /home/ansible/gateway_build.log 2>&1 + build_workstation_vm >> /home/ansible/workstation_build.log 2>&1 +} + +build_gateway_vm() { + /home/ansible/derivative-maker/derivative-maker \ + --flavor whonix-gateway-xfce \ + --target virtualbox \ + --build +} + +build_workstation_vm() { + /home/ansible/derivative-maker/derivative-maker \ + --flavor whonix-workstation-xfce \ + --target virtualbox \ + --build +} + +main diff --git a/automated_builder/roles/headless-build/tasks/install_source.yml b/automated_builder/roles/headless-build/tasks/install_source.yml new file mode 100644 index 000000000..693a55651 --- /dev/null +++ b/automated_builder/roles/headless-build/tasks/install_source.yml @@ -0,0 +1,9 @@ +--- +- name: Create install_source script + template: + src: ./roles/common/templates/install_source.sh + dest: /home/ansible/install_source.sh + mode: 0744 + +- name: Run install_source script for commit + shell: "/home/ansible/install_source.sh {{ GIT_REPO }} {{ REF_NAME }} > /home/ansible/install_source.log 2>&1" diff --git a/automated_builder/roles/headless-build/tasks/main.yml b/automated_builder/roles/headless-build/tasks/main.yml index f53d410a1..9f94ecd3f 100644 --- a/automated_builder/roles/headless-build/tasks/main.yml +++ b/automated_builder/roles/headless-build/tasks/main.yml @@ -1,3 +1,6 @@ --- +- name: Install source code for commit + include_tasks: install_source.yml + - name: Build VMs from commit include_tasks: build_vms_from_commit.yml From 0ac041d590177846336597a59d1f7e5944e5e4a8 Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Sat, 26 Nov 2022 18:17:15 -0700 Subject: [PATCH 42/44] Rotate SSH keys after accidental commit --- .../roles/common/vars/secrets.yml | 74 +++++++++---------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/automated_builder/roles/common/vars/secrets.yml b/automated_builder/roles/common/vars/secrets.yml index 90cc6c514..f298bfef7 100644 --- a/automated_builder/roles/common/vars/secrets.yml +++ b/automated_builder/roles/common/vars/secrets.yml @@ -1,38 +1,38 @@ $ANSIBLE_VAULT;1.1;AES256 -31363661653561386363626261393737666437333534613863623066313236643539393965366135 -3030633638653235333638373362383761333034363036610a383362303563393131363335363031 -62323862326336653066646533313535663865656662396339393836336334373938636433313733 -6565393663613866310a656362396231373337333938646265326663623761653062336363303039 -35306365373665366462306337626463613633663866666165636238636332313564643234393634 -32316363323866616430666262326436666465363930313239613630316534366332373438356662 -64306235663838326236376263376463663135613365653637643037366662373334393561393965 -66653734386335663335313934303164336634643330653266393033303330636137326563306630 -36376339643166306265623263383336303638643532396439333865623461303763633039663863 -62333261326133363664333936386539353131666666613836383135393934383737366565333261 -32316264353064633963323663373430353536633434623531616333306630396465303335653838 -32366661396664373761346162666434626637303939616539333230333763663532313734333063 -62353265366164346461626635613835303136353165653565643134643466346236373631396435 -62356635353939336335346338653137386533653361383466646462326239353238323635313065 -39633837396232383437636333616363353862373332643037643437326238623337626665363832 -30663434316133326261323134383865363964643130633835303231333663626536323037633938 -39386663323134653965363363633064313831613233643533303661636337383037313731383230 -34643036623765353166613832666265313033663262333638646533333133663736326463626530 -31353066613337666263326535663862646565646565353564613838386365623932613266363231 -39613034353463343136353664656539303433366432323761306330303034626361323461626438 -39313536613531373839316361623561613431346665663039663536643037316336306231663832 -63343533663363613231636636363234643238363135613062663531333837383762336566356363 -32376235353331306664356332386137613764353866363136356264663835656164623861323330 -65643765643762336138323435316330633032666262366436323264656463343364343432633565 -61353763613730376436373332326565333034336364383233393839616233666563666335373033 -37643564646564623437636164643537343165336134366337343236353331656162393763653633 -35663362353338306563386538333838636436313734323834326336373063336338633562636435 -63313264356634353936353966643464646634663937333235306435643136366237343463633537 -38353962373232633130326236623434366161653762353833373934376333633235653733623138 -63393531386431303336626664643935313639656265336133346436626434323636346364373830 -37383166303465653161363335346566373138366361313562396366393036663430343666396134 -35333330313863616363613963326232303066366162613434666232626130653239316165383961 -30343964653931373033303738613965363830383533623064303261616633326239346230363232 -65363533616265616163623164306434386639663130373135623938346238353230303133363434 -64303430656339303364346433633030326365646430653061643031386238656137313961616537 -31366464663236363333623339326666366139393734366436373539356633336633373764636639 -6338 +36613161353462363665326163373733653932646233663737356636623834366566386235313435 +3365383733393738383863306661353339653261623533340a343537323738663234306265316238 +32646364633136353664356631613436616462343962336264613761366638363364383133386531 +3361303831616636340a663839653265353263323766353932333962343235343631653265316665 +64613665356538363036666336633633323765326338623637323335613932616536633262356134 +30653466336366646162313634326136366364656635303764336263313530373735653239393433 +64306236333365303466616263626437363635323037626662613665303834396539306230303530 +64383463653663613936626236353936616332383065643962353763313335353665333762313036 +37326164663239623233643364383238616134393537316566326165623363323161656435623965 +32386138666635323033383036643337663231363164306664366230323763626338366261373335 +37643064633734386535393430346564366638303763313165663765646135353737333430396166 +30613461343036376163636636353534613039616631313361333363383634643166643363633131 +31353437376639356138366662383565663535323833356163373134323934663938333134393466 +37383063336338656132376336623437626438316232313736313932646166383735303961636636 +62383635653832393030343433333464343935306161373632336338666136323132623036623833 +39373663633238623833643734623764313233306533393862653837326462666561366530626135 +62333365623336613163633538303765343931346230613231613462653734363566383739646236 +31656534613036646233306234336163616638366265643730346533343832613661386433326430 +34353964303937373864343433653439643239646630633863333561663266623735326437393532 +30636431643636623830643638363831393935616636386138343434353832653065663639363965 +30343139303866383130653261323538623033353237623262353463663236613965316631623533 +64356239323538616338323265613233666636376634613566643934356661393338343338333734 +38333763663366396266663530356361626664663832363232623130383631616530383337343662 +39623738386164663666386363653265386338616363373239386265306539346163643866666637 +36303161623038376536626432393264613736393032613464643238343837643664386538643564 +66386361393835613737363037636630613134333830653036643037343165663332356464623162 +35653938653066666432633563393933626437313237383837663038623062363739306635303964 +38333939626330396533613133623061343061343635643665393532326361323933646132386464 +31383630323362306366366638646432633466393963356232363438353866343665336364656534 +32326666653665663566636332366365613061613963376138336164656435386238353437666435 +62396134313661336163363838313939326438353232306339313762393964376164613336633663 +38353062663266326335633433336639633934333964376665663761636664643563316362313664 +37363834656534383330303465313237386264303661393862646532313431323030316331393436 +32656230326138663434336561663931633739393435383065363636626563353865616562353966 +66303738613332333631333965663934643961633839306466663338656461636634373439646130 +36393533316538646664363166303434343738373661393034373738313966663566363462393266 +6434 From e2a23cb1d3fa42d6595af1f94d5e3b65007c43b7 Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Sat, 26 Nov 2022 18:28:14 -0700 Subject: [PATCH 43/44] Change droplet size for tier restrictions --- automated_builder/roles/common/tasks/generate_inventory.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated_builder/roles/common/tasks/generate_inventory.yml b/automated_builder/roles/common/tasks/generate_inventory.yml index ad055956e..32ad54cbe 100644 --- a/automated_builder/roles/common/tasks/generate_inventory.yml +++ b/automated_builder/roles/common/tasks/generate_inventory.yml @@ -15,7 +15,7 @@ state: present oauth_token: "{{ DO_API_TOKEN }}" name: automated-builder-vps - size: s-8vcpu-16gb + size: s-4vcpu-8gb region: nyc3 image: debian-11-x64 wait_timeout: 500 From cc8cfaba9e22b8bc80a05bb3d2439eab62a3f8ab Mon Sep 17 00:00:00 2001 From: Rob Stringer <41843577+Mycobee@users.noreply.github.com> Date: Sat, 26 Nov 2022 18:55:46 -0700 Subject: [PATCH 44/44] Rename include gui-build role task --- automated_builder/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated_builder/main.yml b/automated_builder/main.yml index 80dbd2e32..d83abcdd0 100644 --- a/automated_builder/main.yml +++ b/automated_builder/main.yml @@ -16,7 +16,7 @@ name: headless-build when: REF_TYPE != 'tag' - - name: Include large-resource role + - name: Include gui-build role include_role: name: gui-build when: REF_TYPE == 'tag'