Skip to content
Permalink
Browse files

Merge pull request #249 from clong/vagrant_cloud

Move boxes to Vagrant Cloud, Add simple tests for logger
  • Loading branch information...
clong committed Apr 11, 2019
2 parents d3e00c8 + 65dbb8d commit c53ce5681c8cfc5edb83437f5e4dc1d2ff671af1
Showing with 123 additions and 160 deletions.
  1. +3 −1 README.md
  2. +3 −6 Vagrant/Vagrantfile
  3. +91 −9 Vagrant/bootstrap.sh
  4. +2 −67 build.ps1
  5. +24 −77 build.sh
@@ -64,7 +64,9 @@ Windows users will want to use the following script:
1. Determine which Vagrant provider you want to use.
* Note: Virtualbox is free, the [VMWare Desktop vagrant plugin](https://www.vagrantup.com/vmware/#buy-now) is $80.

**NOTE:** If you'd like to save time, you can skip the building of the Packer boxes and download the boxes directly from https://detectionlab.network and put them into the `Boxes` directory:
#### Warning: As of May 1, 2019 the boxes will only be available from [Vagrant Cloud](https://app.vagrantup.com/detectionlab/)

If you'd like to save time, you can skip the building of the Packer boxes and download the boxes directly from https://detectionlab.network and put them into the `Boxes` directory:

Provider | Box | URL | MD5 | Size
------------|-----|-----|----|----
@@ -26,7 +26,7 @@ Vagrant.configure("2") do |config|
end

config.vm.define "dc" do |cfg|
cfg.vm.box = "../Boxes/windows_2016_virtualbox.box"
cfg.vm.box = "detectionlab/win2016"
cfg.vm.hostname = "dc"
cfg.vm.boot_timeout = 600
# use the plaintext WinRM transport and force it to use basic authentication.
@@ -63,7 +63,6 @@ Vagrant.configure("2") do |config|
cfg.vm.provision "shell", inline: "Set-SmbServerConfiguration -AuditSmb1Access $true -Force", privileged: false

cfg.vm.provider "vmware_desktop" do |v, override|
override.vm.box = "../Boxes/windows_2016_vmware.box"
v.vmx["displayname"] = "dc.windomain.local"
v.memory = 2560
v.cpus = 2
@@ -84,7 +83,7 @@ Vagrant.configure("2") do |config|
end

config.vm.define "wef" do |cfg|
cfg.vm.box = "../Boxes/windows_2016_virtualbox.box"
cfg.vm.box = "detectionlab/win2016"
cfg.vm.hostname = "wef"
cfg.vm.boot_timeout = 600
cfg.vm.communicator = "winrm"
@@ -116,7 +115,6 @@ Vagrant.configure("2") do |config|
cfg.vm.provision "shell", path: "scripts/install-microsoft-ata.ps1", privileged: false

cfg.vm.provider "vmware_desktop" do |v, override|
override.vm.box = "../Boxes/windows_2016_vmware.box"
v.vmx["displayname"] = "wef.windomain.local"
v.memory = 2048
v.cpus = 2
@@ -137,7 +135,7 @@ Vagrant.configure("2") do |config|
end

config.vm.define "win10" do |cfg|
cfg.vm.box = "../Boxes/windows_10_virtualbox.box"
cfg.vm.box = "detectionlab/win10"
cfg.vm.hostname = "win10"
cfg.vm.boot_timeout = 600
cfg.vm.communicator = "winrm"
@@ -165,7 +163,6 @@ Vagrant.configure("2") do |config|
cfg.vm.provision "shell", path: "scripts/install-autorunstowineventlog.ps1", privileged: false

cfg.vm.provider "vmware_desktop" do |v, override|
override.vm.box = "../Boxes/windows_10_vmware.box"
v.vmx["displayname"] = "win10.windomain.local"
v.vmx["gui.fullscreenatpoweron"] = "FALSE"
v.vmx["gui.viewModeAtPowerOn"] = "windowed"
@@ -19,13 +19,55 @@ install_python_apt_source() {

apt_install_prerequisites() {
# Install prerequisites and useful tools
apt-get update
apt-get install -y apt-fast
apt-fast install -y jq whois build-essential git docker docker-compose unzip mongodb-org python3.6 python3.6-dev
echo "Running apt-get update..."
apt-get -qq update
apt-get -qq install -y apt-fast
echo "Running apt-fast install..."
apt-fast -qq install -y jq whois build-essential git docker docker-compose unzip mongodb-org python3.6 python3.6-dev
# Install pip for Python 3.6
echo "Installing Pip3.6..."
curl https://bootstrap.pypa.io/get-pip.py | sudo -H python3.6
}

test_prerequisites() {
for package in jq whois build-essential git docker docker-compose unzip mongodb-org python3.6 python3.6-dev
do
echo "[TEST] Validating that $package is correctly installed..."
# Loop through each package using dpkg
if ! dpkg -S $package > /dev/null; then
# If which returns a non-zero return code, try to re-install the package
echo "[-] $package was not found. Attempting to reinstall."
apt-get -qq update && apt-get install -y $package
if ! which $package > /dev/null; then
# If the reinstall fails, give up
echo "[X] Unable to install $package even after a retry. Exiting."
exit 1
fi
else
echo "[+] $package was successfully installed!"
fi
done

# One-off support for packages which aren't installed via dpkg
for package in "pip3.6"
do
echo "[TEST] Validating that $package is correctly installed..."
# Loop through each package using which
if ! which $package > /dev/null; then
# If which returns a non-zero return code, try to re-install the package
echo "[-] $package was not found. Attempting to reinstall."
curl https://bootstrap.pypa.io/get-pip.py | sudo -H python3.6
if ! which $package > /dev/null; then
# If the reinstall fails, give up
echo "[X] Unable to install $package even after a retry. Exiting."
exit 1
fi
else
echo "[+] $package was successfully installed!"
fi
done
}

fix_eth1_static_ip() {
# There's a fun issue where dhclient keeps messing with eth1 despite the fact
# that eth1 has a static IP set. We workaround this by setting a static DHCP lease.
@@ -55,7 +97,7 @@ install_golang() {
if ! which go > /dev/null; then
echo "Installing Golang v.1.12..."
cd /home/vagrant || exit
wget https://dl.google.com/go/go1.12.linux-amd64.tar.gz
wget --progress=bar:force https://dl.google.com/go/go1.12.linux-amd64.tar.gz
tar -C /usr/local -xzf go1.12.linux-amd64.tar.gz
mkdir /root/go
else
@@ -124,6 +166,7 @@ install_fleet() {
else
echo "Installing Fleet..."
echo -e "\n127.0.0.1 kolide" >> /etc/hosts
echo -e "\n127.0.0.1 logger" >> /etc/hosts
git clone https://github.com/kolide/kolide-quickstart.git
cd kolide-quickstart || echo "Something went wrong while trying to clone the kolide-quickstart repository"
cp /vagrant/resources/fleet/server.* .
@@ -149,7 +192,7 @@ download_palantir_osquery_config() {
}

import_osquery_config_into_fleet() {
wget https://github.com/kolide/fleet/releases/download/2.0.1/fleet_2.0.1.zip
wget --progress=bar:force https://github.com/kolide/fleet/releases/download/2.0.1/fleet_2.0.1.zip
unzip fleet_2.0.1.zip -d fleet_2.0.1
cp fleet_2.0.1/linux/fleetctl /usr/local/bin/fleetctl && chmod +x /usr/local/bin/fleetctl
fleetctl config set --address https://192.168.38.105:8412
@@ -192,8 +235,8 @@ install_caldera() {
systemctl enable mongod.service
cd /home/vagrant/caldera || exit
mkdir -p dep/crater/crater
wget https://github.com/mitre/caldera-crater/releases/download/v0.1.0/CraterMainWin8up.exe -O /home/vagrant/caldera/dep/crater/crater/CraterMain.exe
cp /vagrant/resources/caldera/cert.pem /vagrant/resources/caldera/key.pem /vagrant/resources/caldera/settings.yml /home/vagrant/caldera/caldera/conf
wget --progress=bar:force https://github.com/mitre/caldera-crater/releases/download/v0.1.0/CraterMainWin8up.exe -O /home/vagrant/caldera/dep/crater/crater/CraterMain.exe
cp /vagrant/resources/caldera/cert.pem /vagrant/resources/caldera/key.pem /vagrant/resources/caldera/settings.yml /home/vagrant/caldera/caldera/conf
service caldera start
systemctl enable caldera.service
fi
@@ -283,17 +326,19 @@ install_suricata() {
# Run iwr -Uri testmyids.com -UserAgent "BlackSun" in Powershell to generate test alerts

# Install yq to maniuplate the suricata.yaml inline
/usr/local/go/bin/go get -u github.com/mikefarah/yq
/usr/local/go/bin/go get -u github.com/mikefarah/yq

# Install suricata
add-apt-repository -y ppa:oisf/suricata-stable
apt-get -qq -y update && apt-get -qq -y install suricata crudini
test_suricata_prerequisites
# Install suricata-update
cd /home/vagrant || exit 1
git clone https://github.com/OISF/suricata-update.git
cd /home/vagrant/suricata-update || exit 1
python setup.py install
# Add DC_SERVERS variable to suricata.yaml in support et-open signatures
/root/go/bin/yq w -i /etc/suricata/suricata.yaml vars.address-groups.DC_SERVERS '$HOME_NET'
/root/go/bin/yq w -i /etc/suricata/suricata.yaml vars.address-groups.DC_SERVERS '$HOME_NET'

# It may make sense to store the suricata.yaml file as a resource file if this begins to become too complex
# Add more verbose alert logging
@@ -341,10 +386,47 @@ install_suricata() {
fi
}

test_suricata_prerequisites() {
for package in suricata crudini
do
echo "[TEST] Validating that $package is correctly installed..."
# Loop through each package using dpkg
if ! dpkg -S $package > /dev/null; then
# If which returns a non-zero return code, try to re-install the package
echo "[-] $package was not found. Attempting to reinstall."
apt-get -qq update && apt-get install -y $package
if ! which $package > /dev/null; then
# If the reinstall fails, give up
echo "[X] Unable to install $package even after a retry. Exiting."
exit 1
fi
else
echo "[+] $package was successfully installed!"
fi
done

# One-off support for packages which aren't installed via dpkg
echo "[TEST] Validating that yq is correctly installed..."
# Check if the binary exists
if ! [ -f /root/go/bin/yq ]; then
# If it doesn't exist, try to re-install the package
echo "[-] yq was not found. Attempting to reinstall."
/usr/local/go/bin/go get -u github.com/mikefarah/yq
if ! [ -f /root/go/bin/yq ]; then
# If the reinstall fails, give up
echo "[X] Unable to install yq even after a retry. Exiting."
exit 1
fi
else
echo "[+] yq was successfully installed!"
fi
}

main() {
install_mongo_db_apt_key
install_python_apt_source
apt_install_prerequisites
test_prerequisites
fix_eth1_static_ip
install_golang
install_splunk
@@ -26,7 +26,7 @@
The full path to the packer executable. Default is C:\Hashicorp\packer.exe
.PARAMETER VagrantOnly
This switch skips building packer boxes and instead downloads from www.detectionlab.network
This switch skips building packer boxes and instead downloads from Vagrant Cloud
.EXAMPLE
build.ps1 -ProviderName virtualbox
@@ -54,14 +54,6 @@ Param(
$DL_DIR = Split-Path -Parent -Path $MyInvocation.MyCommand.Definition
$LAB_HOSTS = ('logger', 'dc', 'wef', 'win10')

# Register-EngineEvent PowerShell.Exiting -SupportEvent -Action {
# Set-Location $DL_DIR
# }

# Register-ObjectEvent -InputObject ([System.Console]) -EventName CancelKeyPress -Action {
# Set-Location $DL_DIR
# }

function install_checker {
param(
[string]$Name
@@ -177,55 +169,6 @@ function list_providers {
return $ProviderName
}

function download_boxes {
Write-Host '[download_boxes] Running..'
if ($PackerProvider -eq 'virtualbox') {
$win10Hash = 'c03f10f21b8d79e6acca2b2965b23046'
$win2016Hash = '231b54077d4396cad01e4cd60651b1e0'
}
if ($PackerProvider -eq 'vmware') {
$win10Hash = 'b334c3ba5be3b29840567ffe368db5fe'
$win2016Hash = '2bbaf5a1177e0499dc3aacdb0246eb38'
}

$win10Filename = "windows_10_$PackerProvider.box"
$win2016Filename = "windows_2016_$PackerProvider.box"

$wc = New-Object System.Net.WebClient
Write-Host "[download_boxes] Downloading $win10Filename"
$wc.DownloadFile("https://www.detectionlab.network/$win10Filename", "$DL_DIR\Boxes\$win10Filename")
Write-Host "[download_boxes] Downloading $win2016Filename"
$wc.DownloadFile("https://www.detectionlab.network/$win2016Filename", "$DL_DIR\Boxes\$win2016Filename")
$wc.Dispose()

if (-Not (Test-Path "$DL_DIR\Boxes\$win2016Filename")) {
Write-Error 'Windows 2016 box is missing from the Boxes directory. Qutting.'
break
}
if (-Not (Test-Path "$DL_DIR\Boxes\$win10Filename")) {
Write-Error 'Windows 10 box is missing from the Boxes directory. Qutting.'
break
}

Write-Host "[download_boxes] Getting filehash for: $win10Filename"
$win10Filehash = (Get-FileHash -Path "$DL_DIR\Boxes\$win10Filename" -Algorithm MD5).Hash
Write-Host "[download_boxes] Getting filehash for: $win2016Filename"
$win2016Filehash = (Get-FileHash -Path "$DL_DIR\Boxes\$win2016Filename" -Algorithm MD5).Hash

Write-Host '[download_boxes] Checking Filehashes..'
if ($win10hash -ne $win10Filehash) {
Write-Error 'Hash mismatch on windows_10_virtualbox.box'
Write-Error 'The boxes may have been updated since you last ran the build script. Try updating the git repository to retrieve the latest hashes.'
break
}
if ($win2016hash -ne $win2016Filehash) {
Write-Error 'Hash mismatch on windows_2016_virtualbox.box'
Write-Error 'The boxes may have been updated since you last ran the build script. Try updating the git repository to retrieve the latest hashes.'
break
}
Write-Host '[download_boxes] Finished.'
}

function preflight_checks {
Write-Host '[preflight_checks] Running..'
# Check to see that no boxes exist
@@ -390,7 +333,6 @@ function download {
Write-Host "Error occured on webrequest: $_"
return $false
}

}
}

@@ -412,7 +354,6 @@ function post_build_checks {
$ATA_CHECK = download -URL 'https://192.168.38.103' -SuccessOn401
Write-Host "[post_build_checks] ATA Result: $ATA_CHECK"


if ($CALDERA_CHECK -eq $false) {
Write-Warning 'Caldera failed post-build tests and may not be functioning correctly.'
}
@@ -427,7 +368,6 @@ function post_build_checks {
}
}


# If no ProviderName was provided, get a provider
if ($ProviderName -eq $Null -or $ProviderName -eq "") {
$ProviderName = list_providers
@@ -441,15 +381,11 @@ else {
$PackerProvider = 'virtualbox'
}


# Run check functions
preflight_checks

# Build Packer Boxes
if ($VagrantOnly) {
download_boxes
}
else {
if (!($VagrantOnly)) {
packer_build_box -Box 'windows_2016'
packer_build_box -Box 'windows_10'
# Move Packer Boxes
@@ -477,7 +413,6 @@ forEach ($VAGRANT_HOST in $LAB_HOSTS) {
Write-Host "[main] Finished for: $VAGRANT_HOST"
}


Write-Host "[main] Running post_build_checks"
post_build_checks
Write-Host "[main] Finished post_build_checks"
Oops, something went wrong.

0 comments on commit c53ce56

Please sign in to comment.
You can’t perform that action at this time.