From b62c66d4648bdf0133b54258609013ba52b2aec5 Mon Sep 17 00:00:00 2001 From: Olivier Bilodeau Date: Tue, 9 Oct 2018 12:23:20 -0400 Subject: [PATCH 01/10] Installing VirtualBox 5.2.x hoping it fixes #108 Required adjustments to dependencies --- tests/smoke/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/smoke/Dockerfile b/tests/smoke/Dockerfile index a1d14a9..232b8f3 100644 --- a/tests/smoke/Dockerfile +++ b/tests/smoke/Dockerfile @@ -1,6 +1,9 @@ FROM ubuntu:16.04 -RUN apt-get update && apt-get install -y --no-install-recommends virtualbox python3 python3-pip python3-setuptools wget unzip bash git && \ +RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates gnupg && \ + curl -sSL https://www.virtualbox.org/download/oracle_vbox_2016.asc | apt-key add - && \ + echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list.d/virtualbox.list && \ + apt-get update && apt-get install -y --no-install-recommends virtualbox-5.2 module-init-tools python3 python3-pip python3-setuptools wget unzip bash git && \ wget https://releases.hashicorp.com/packer/1.3.1/packer_1.3.1_linux_amd64.zip && unzip packer_1.3.1_linux_amd64.zip -d packer && \ mv packer/packer /usr/local/bin/ && chmod a+x /usr/local/bin/packer && rm packer_1.3.1_linux_amd64.zip && rmdir packer From dc7cc0a3dd37d72735b1bf765cb69e55412f0258 Mon Sep 17 00:00:00 2001 From: Olivier Bilodeau Date: Tue, 9 Oct 2018 16:31:19 -0400 Subject: [PATCH 02/10] Trying packer 1.2.5 (latest of 1.2.x) see #108 --- tests/smoke/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/smoke/Dockerfile b/tests/smoke/Dockerfile index 232b8f3..ae31ded 100644 --- a/tests/smoke/Dockerfile +++ b/tests/smoke/Dockerfile @@ -4,8 +4,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certifi curl -sSL https://www.virtualbox.org/download/oracle_vbox_2016.asc | apt-key add - && \ echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list.d/virtualbox.list && \ apt-get update && apt-get install -y --no-install-recommends virtualbox-5.2 module-init-tools python3 python3-pip python3-setuptools wget unzip bash git && \ - wget https://releases.hashicorp.com/packer/1.3.1/packer_1.3.1_linux_amd64.zip && unzip packer_1.3.1_linux_amd64.zip -d packer && \ - mv packer/packer /usr/local/bin/ && chmod a+x /usr/local/bin/packer && rm packer_1.3.1_linux_amd64.zip && rmdir packer + wget https://releases.hashicorp.com/packer/1.2.5/packer_1.2.5_linux_amd64.zip && unzip packer_1.2.5_linux_amd64.zip -d packer && \ + mv packer/packer /usr/local/bin/ && chmod a+x /usr/local/bin/packer && rm packer_1.2.5_linux_amd64.zip && rmdir packer # Config between delimiters taken from https://github.com/jenkinsci/docker/blob/master/Dockerfile # -- From df323a4505e37d133666098daa99cfcea1004ce8 Mon Sep 17 00:00:00 2001 From: Olivier Bilodeau Date: Wed, 10 Oct 2018 09:09:04 -0400 Subject: [PATCH 03/10] Trying packer 1.0.3 (latest of 1.0.x) see #108 --- tests/smoke/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/smoke/Dockerfile b/tests/smoke/Dockerfile index ae31ded..6bf9586 100644 --- a/tests/smoke/Dockerfile +++ b/tests/smoke/Dockerfile @@ -4,8 +4,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certifi curl -sSL https://www.virtualbox.org/download/oracle_vbox_2016.asc | apt-key add - && \ echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list.d/virtualbox.list && \ apt-get update && apt-get install -y --no-install-recommends virtualbox-5.2 module-init-tools python3 python3-pip python3-setuptools wget unzip bash git && \ - wget https://releases.hashicorp.com/packer/1.2.5/packer_1.2.5_linux_amd64.zip && unzip packer_1.2.5_linux_amd64.zip -d packer && \ - mv packer/packer /usr/local/bin/ && chmod a+x /usr/local/bin/packer && rm packer_1.2.5_linux_amd64.zip && rmdir packer + wget https://releases.hashicorp.com/packer/1.0.3/packer_1.0.3_linux_amd64.zip && unzip packer_1.0.3_linux_amd64.zip -d packer && \ + mv packer/packer /usr/local/bin/ && chmod a+x /usr/local/bin/packer && rm packer_1.0.3_linux_amd64.zip && rmdir packer + # Config between delimiters taken from https://github.com/jenkinsci/docker/blob/master/Dockerfile # -- From 5622495414e1ba4f84966cd503e2d5a5dfd03a76 Mon Sep 17 00:00:00 2001 From: Olivier Bilodeau Date: Wed, 10 Oct 2018 11:36:58 -0400 Subject: [PATCH 04/10] CI Troubleshooting: Installing VBox extensions for VRDP support --- tests/smoke/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/smoke/Dockerfile b/tests/smoke/Dockerfile index 6bf9586..7064ce6 100644 --- a/tests/smoke/Dockerfile +++ b/tests/smoke/Dockerfile @@ -5,7 +5,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certifi echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list.d/virtualbox.list && \ apt-get update && apt-get install -y --no-install-recommends virtualbox-5.2 module-init-tools python3 python3-pip python3-setuptools wget unzip bash git && \ wget https://releases.hashicorp.com/packer/1.0.3/packer_1.0.3_linux_amd64.zip && unzip packer_1.0.3_linux_amd64.zip -d packer && \ - mv packer/packer /usr/local/bin/ && chmod a+x /usr/local/bin/packer && rm packer_1.0.3_linux_amd64.zip && rmdir packer + mv packer/packer /usr/local/bin/ && chmod a+x /usr/local/bin/packer && rm packer_1.0.3_linux_amd64.zip && rmdir packer && \ + VBOXVER=$(wget -qO - https://download.virtualbox.org/virtualbox/LATEST.TXT) && \ + wget "https://download.virtualbox.org/virtualbox/${VBOXVER}/Oracle_VM_VirtualBox_Extension_Pack-${VBOXVER}.vbox-extpack" && \ + VBoxManage extpack install --accept-license=56be48f923303c8cababb0bb4c478284b688ed23f16d775d729b89a2e8e5f9eb Oracle_VM_VirtualBox_Extension_Pack-${VBOXVER}.vbox-extpack && \ + rm Oracle_VM_VirtualBox_Extension_Pack-${VBOXVER}.vbox-extpack # Config between delimiters taken from https://github.com/jenkinsci/docker/blob/master/Dockerfile From 315439e0a5f51d8fc81a80eaeada0dab9788b497 Mon Sep 17 00:00:00 2001 From: Olivier Bilodeau Date: Wed, 10 Oct 2018 15:11:49 -0400 Subject: [PATCH 05/10] Accelerated Windows 7 build (see #108) Turns out slow build time and/or heavy logging might be the culprit as to why Windows 7 builds were failing. Acceleration only impacts smoke test config so we can merge this w/o regressions. --- tests/smoke/config.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/smoke/config.js b/tests/smoke/config.js index a5d8fab..1f18546 100644 --- a/tests/smoke/config.js +++ b/tests/smoke/config.js @@ -41,8 +41,8 @@ //"hypervisor": "kvm", // Chocolatey packages to install on the VM - // TODO re-add dependencywalker and regshot once upstream choco package provides a checksum - "choco_packages": "sysinternals windbg wireshark 7zip putty apm", + // FIXME made install go faster to prevent jenkins / packer bug when building Windows 7 machines (see #108) + //"choco_packages": "sysinternals windbg wireshark 7zip putty apm", // Setting the IDA Path will copy the IDA remote debugging tools into the guest //"ida_path": "/path/to/your/ida", From 5b277432d660814475fe4e83c77d085ed78da5f5 Mon Sep 17 00:00:00 2001 From: Olivier Bilodeau Date: Wed, 10 Oct 2018 15:58:26 -0400 Subject: [PATCH 06/10] split VBox extension in a separate RUN cmd disabled by default --- tests/smoke/Dockerfile | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/smoke/Dockerfile b/tests/smoke/Dockerfile index 7064ce6..24350db 100644 --- a/tests/smoke/Dockerfile +++ b/tests/smoke/Dockerfile @@ -5,11 +5,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certifi echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list.d/virtualbox.list && \ apt-get update && apt-get install -y --no-install-recommends virtualbox-5.2 module-init-tools python3 python3-pip python3-setuptools wget unzip bash git && \ wget https://releases.hashicorp.com/packer/1.0.3/packer_1.0.3_linux_amd64.zip && unzip packer_1.0.3_linux_amd64.zip -d packer && \ - mv packer/packer /usr/local/bin/ && chmod a+x /usr/local/bin/packer && rm packer_1.0.3_linux_amd64.zip && rmdir packer && \ - VBOXVER=$(wget -qO - https://download.virtualbox.org/virtualbox/LATEST.TXT) && \ - wget "https://download.virtualbox.org/virtualbox/${VBOXVER}/Oracle_VM_VirtualBox_Extension_Pack-${VBOXVER}.vbox-extpack" && \ - VBoxManage extpack install --accept-license=56be48f923303c8cababb0bb4c478284b688ed23f16d775d729b89a2e8e5f9eb Oracle_VM_VirtualBox_Extension_Pack-${VBOXVER}.vbox-extpack && \ - rm Oracle_VM_VirtualBox_Extension_Pack-${VBOXVER}.vbox-extpack + mv packer/packer /usr/local/bin/ && chmod a+x /usr/local/bin/packer && rm packer_1.0.3_linux_amd64.zip && rmdir packer + + +# Enable this RUN statement when you need to connect to the VRDP server of the VM to troubleshoot issues +#RUN VBOXVER=$(wget -qO - https://download.virtualbox.org/virtualbox/LATEST.TXT) && \ +# wget "https://download.virtualbox.org/virtualbox/${VBOXVER}/Oracle_VM_VirtualBox_Extension_Pack-${VBOXVER}.vbox-extpack" && \ +# VBoxManage extpack install --accept-license=56be48f923303c8cababb0bb4c478284b688ed23f16d775d729b89a2e8e5f9eb Oracle_VM_VirtualBox_Extension_Pack-${VBOXVER}.vbox-extpack && \ +# rm Oracle_VM_VirtualBox_Extension_Pack-${VBOXVER}.vbox-extpack # Config between delimiters taken from https://github.com/jenkinsci/docker/blob/master/Dockerfile From 6b0831de4f7f424b7c9a5a4729213d2dff37cf26 Mon Sep 17 00:00:00 2001 From: Olivier Bilodeau Date: Wed, 10 Oct 2018 15:59:37 -0400 Subject: [PATCH 07/10] Back to Packer 1.3.5 (#108) Since downgrading didn't resolve build issues in itself we are back to the latest and greatest packer version. --- tests/smoke/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/smoke/Dockerfile b/tests/smoke/Dockerfile index 24350db..521e937 100644 --- a/tests/smoke/Dockerfile +++ b/tests/smoke/Dockerfile @@ -4,8 +4,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certifi curl -sSL https://www.virtualbox.org/download/oracle_vbox_2016.asc | apt-key add - && \ echo "deb http://download.virtualbox.org/virtualbox/debian xenial contrib" >> /etc/apt/sources.list.d/virtualbox.list && \ apt-get update && apt-get install -y --no-install-recommends virtualbox-5.2 module-init-tools python3 python3-pip python3-setuptools wget unzip bash git && \ - wget https://releases.hashicorp.com/packer/1.0.3/packer_1.0.3_linux_amd64.zip && unzip packer_1.0.3_linux_amd64.zip -d packer && \ - mv packer/packer /usr/local/bin/ && chmod a+x /usr/local/bin/packer && rm packer_1.0.3_linux_amd64.zip && rmdir packer + wget https://releases.hashicorp.com/packer/1.3.1/packer_1.3.1_linux_amd64.zip && unzip packer_1.3.1_linux_amd64.zip -d packer && \ + mv packer/packer /usr/local/bin/ && chmod a+x /usr/local/bin/packer && rm packer_1.3.1_linux_amd64.zip && rmdir packer # Enable this RUN statement when you need to connect to the VRDP server of the VM to troubleshoot issues From 747d4d158ec32b7c287c8e56d3de9a72cc3b71e3 Mon Sep 17 00:00:00 2001 From: Olivier Bilodeau Date: Wed, 10 Oct 2018 15:06:57 -0400 Subject: [PATCH 08/10] More disk space, Windows 7 can't install SP1 due to lack of space --- tests/smoke/config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/smoke/config.js b/tests/smoke/config.js index 1f18546..bf22649 100644 --- a/tests/smoke/config.js +++ b/tests/smoke/config.js @@ -25,7 +25,7 @@ "password": "malboxes", "computername": "smoketest", // disk size is in megabytes - "disk_size": "20480", + "disk_size": "40960", // Windows Defender: true means enabled, false means disabled. Default is false. //"windows_defender": "false", From a722bb36b43ae8f773cbbe02ba69b9feda0bb812 Mon Sep 17 00:00:00 2001 From: Olivier Bilodeau Date: Wed, 10 Oct 2018 16:23:25 -0400 Subject: [PATCH 09/10] CI Docker Setup: host network access and /tmp access host network: to be able to connect to the VRDP to troubleshoot builds /tmp access: VirtualBox uses /tmp to store locks, this will prevent concurrent access to the same VMs --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6ac2a8a..36dff4d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -9,7 +9,7 @@ pipeline { dockerfile { filename 'Dockerfile' dir 'tests/smoke' - args '-v /dev/vboxdrv:/dev/vboxdrv --privileged' + args '-v /dev/vboxdrv:/dev/vboxdrv -v /tmp:/tmp --network host --privileged' } } steps { From 90a6e9e0de88ad0a348afa30d793fb008b30f5e0 Mon Sep 17 00:00:00 2001 From: Olivier Bilodeau Date: Wed, 10 Oct 2018 21:30:12 -0400 Subject: [PATCH 10/10] Travis-CI: Remove large generated files in a clean step --- Jenkinsfile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 36dff4d..7336aac 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -16,5 +16,13 @@ pipeline { sh 'tests/smoke/build-all-templates.sh' } } + stage('Clean') { + agent any + steps { + dir('/var/jenkins_home/.cache/malboxes/') { + deleteDir() + } + } + } } }