Getting things ready for deployment

Docker/BuildDockerSkadi.sh

@@ -28,15 +28,14 @@ echo "==> Giving ${SKADI_USER} sudo powers"
 echo "${SKADI_USER}        ALL=(ALL)       NOPASSWD: ALL" > /etc/sudoers.d/$SKADI_USER
 chmod 440 /etc/sudoers.d/$SKADI_USER
 
-
 # Update
 sudo apt-get update && sudo apt-get dist-upgrade -y
 
 # Install deps
 sudo apt-get install apt-transport-https ca-certificates curl software-properties-common python-pip glances unzip vim htop -y
 
 # Ensure pip is on 9.0.3 for installation
-sudo -H pip install pip==9.0.3
+sudo -H pip install pip==9.0.3 --no-cache-dir
 
 # Disable Swap
 sudo swapoff -a
@@ -66,12 +65,10 @@ sudo apt-get -y autoclean
 sudo usermod -aG docker $SKADI_USER
 
 # Install Docker-Compose
-# sudo -H pip install docker-compose
 sudo curl -L "https://github.com/docker/compose/releases/download/1.23.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
 sudo chmod +x /usr/local/bin/docker-compose
 sudo curl -L https://raw.githubusercontent.com/docker/compose/1.23.1/contrib/completion/bash/docker-compose -o /etc/bash_completion.d/docker-compose
 
-
 # Set the vm.max_map_count kernel setting needs to be set to at least 262144 for production use
 sudo sysctl -w vm.max_map_count=262144
 echo vm.max_map_count=262144 | sudo tee -a /etc/sysctl.conf
@@ -82,18 +79,17 @@ sudo chown -R skadi:skadi /opt/skadi
 sudo mkdir -p /etc/nginx/conf.d
 sudo mkdir -p /usr/share/nginx/html
 
-
 # Copy Nginx configuration files to required locations
-sudo cp ./nginx/.skadi_auth /etc/nginx/
-sudo cp ./nginx/skadi_default.conf /etc/nginx/conf.d
-sudo cp -r ./nginx/www/* /usr/share/nginx/html
+sudo git clone https://github.com/orlikoski/Skadi.git /opt/Skadi
+sudo cp /opt/Skadi/Docker/nginx/.skadi_auth /etc/nginx/
+sudo cp /opt/Skadi/Docker/nginx/skadi_default.conf /etc/nginx/conf.d
+sudo cp -r /opt/Skadi/Docker/nginx/www/* /usr/share/nginx/html
 
-# Install Things Required for TimeSketch on Host
-
-# Install TimeSketch on the Host
+# Install TimeSketch on the Host (required for psort.py to output in timesketch format)
 sudo -H pip install timesketch
 
 # Write TS and Postgres creds to .env file
+cd /opt/Skadi/Docker/
 echo TIMESKETCH_USER=$TIMESKETCH_USER > ./.env
 echo TIMESKETCH_PASSWORD=$TIMESKETCH_PASSWORD >> ./.env
 echo POSTGRES_USER=$POSTGRES_USER >> ./.env
@@ -114,19 +110,16 @@ sudo sed -i "s@NEO4J_USERNAME = u'neo4j'@NEO4J_USERNAME = u'$neo4juser'@g" /etc/
 sudo sed -i "s@NEO4J_PASSWORD = u'<N4J_PASSWORD>'@NEO4J_PASSWORD = u'$neo4jpassword'@g" /etc/timesketch.conf
 sudo sed -i "s/UPLOAD_ENABLED = False/UPLOAD_ENABLED = True/g" /etc/timesketch.conf
 sudo sed -i "s/GRAPH_BACKEND_ENABLED = False/GRAPH_BACKEND_ENABLED = True/g" /etc/timesketch.conf
-
 sudo sed -i "s#@localhost/timesketch#@postgres/timesketch#g" /etc/timesketch.conf
 sudo sed -i "s/ELASTIC_HOST = u'127.0.0.1'/ELASTIC_HOST = u'elasticsearch'/g" /etc/timesketch.conf
 sudo sed -i "s@'redis://127.0.0.1:6379'@'redis://redis:6379'@g" /etc/timesketch.conf
 sudo sed -i "s/NEO4J_HOST = u'127.0.0.1'/NEO4J_HOST = u'neo4j'/g" /etc/timesketch.conf
 
-# sudo useradd -r -s /bin/false timesketch
-
 # To build TimeSketch and CyberChef Docker Images Locally, uncomment the following lines
 # sudo docker build -t aorlikoski/skadi_timesketch:1.0 ./timesketch/
 # sudo docker build -t aorlikoski/skadi_cyberchef:1.0 ./cyberchef/
 
-# Deploy all the things
+# Deploy the Skadi solution defined in ./docker-compose.yml
 sudo docker-compose up -d
 
 # Create a template in ES that sets the number of replicas for all indexes to 0
@@ -141,16 +134,19 @@ curl -XPUT 'localhost:9200/_template/number_of_replicas' \
     -d '{"template": "*","settings": {"number_of_replicas": 0}}' \
     -H'Content-Type: application/json'
 
+# The TimeSketch container needs to be running before continuing and this
+# requires the other containers to be up and running too. This can take time
+# so this loop ensures all the parts are running and timesketch is responding
+# to web requets before continuing
 echo "Waiting for TimeSketch to become available"
 echo "Press CTRL-C at any time to stop installation"
 until $(curl --output /dev/null --silent --head --fail http://localhost/timesketch); do
-    echo "No response, attempting to restart the TimeSketch container"
+    echo "No response, restarting the TimeSketch container and waiting 10 seconds to try again"
     sudo docker restart timesketch
     sleep 10
 done
 echo "TimeSketch available. Continuing"
 
-
 # Install Glances as a Service
 glances_service="W1VuaXRdCkRlc2NyaXB0aW9uPUdsYW5jZXMKQWZ0ZXI9bmV0d29yay50YXJnZXQKCltTZXJ2aWNlXQpFeGVjU3RhcnQ9L3Vzci9iaW4vZ2xhbmNlcyAtdwpSZXN0YXJ0PW9uLWFib3J0CgpbSW5zdGFsbF0KV2FudGVkQnk9bXVsdGktdXNlci50YXJnZXQK"
 echo $glances_service |base64 -d | sudo tee /etc/systemd/system/glances.service
@@ -167,10 +163,9 @@ cd skadi_dockprom
 echo ADMIN_USER=$GRAFANA_USER > ./.env
 echo ADMIN_PASSWORD=$GRAFANA_PASSWORD >> ./.env
 
+# This uses the docker-compose.yml found in the skadi_dockprom repo
 sudo docker-compose up -d
 
-
-
 # Installs and Configures CDQR and CyLR
 echo "Updating CDQR"
 wget -O /tmp/cdqr.py https://raw.githubusercontent.com/orlikoski/CDQR/master/src/cdqr.py

Docker/README.md

@@ -1,7 +1,13 @@
 # Docker Build Steps
 
 ## Setup the Host
-Install Ubuntu 16.04 on the host machine. This can be a VM, Cloud instance, or bare iron machine.
+Install Ubuntu 16.04 on the host machine. This can be a VM, Cloud, or bare iron endpoint.
+
+### Machine Specifications
+The following are the minimum / recommended configurations to run Skadi
+ * CPU Cores:  4 / 8
+ * RAM: 8 GB / 16 BB
+ * Disk: 50 GB / 200 GB
 
 ## Install all the Components
 ```

Packer/GCP/skadi_server.json

@@ -49,7 +49,7 @@
     "cleanup_pause": "",
     "cpus": "4",
     "memory": "8192",
-    "custom_script": "script/skadibuild.sh",
+    "custom_script": "../../Docker/BuildDockerSkadi.sh",
     "desktop": "true",
     "disk_size": "10240",
     "ftp_proxy": "{{env `ftp_proxy`}}",
@@ -72,7 +72,7 @@
     "ssh_password": "vagrant",
     "ssh_username": "vagrant",
     "update": "false",
-    "vagrantfile_template": "../../Vagrant/skadi_desktop/virtualbox/Vagrantfile",
+    "vagrantfile_template": "../../Vagrant/skadi_server/virtualbox/Vagrantfile",
     "version": "2018.3.1",
     "virtualbox_guest_os_type": "Ubuntu_64",
     "vm_name": "skadi_desktop",

Packer/script/cleanup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-SSH_USER="skadi"
+SKADI_USER="skadi"
 DISK_USAGE_BEFORE_CLEANUP=$(df -h)
 
 # Make sure udev does not block our network - http://6.ptmc.org/?p=164
@@ -35,6 +35,7 @@ apt-get -y autoclean
 unset HISTFILE
 rm -f /root/.bash_history
 rm -f /home/${SSH_USER}/.bash_history
+rm -f /home/$SKADI_USER/.bash_history
 
 # Clean up log files
 find /var/log -type f | while read f; do echo -ne '' > "${f}"; done;

Packer/skadi_build/http/preseed.cfg

@@ -0,0 +1,43 @@
+choose-mirror-bin mirror/http/proxy string
+d-i base-installer/kernel/override-image string linux-server
+d-i clock-setup/utc boolean true
+d-i clock-setup/utc-auto boolean true
+d-i finish-install/reboot_in_progress note
+d-i grub-installer/only_debian boolean true
+d-i grub-installer/with_other_os boolean true
+d-i partman-auto/disk string /dev/sda
+d-i partman-auto-lvm/guided_size string max
+d-i partman-auto/choose_recipe select atomic
+d-i partman-auto/method string lvm
+d-i partman-lvm/confirm boolean true
+d-i partman-lvm/confirm boolean true
+d-i partman-lvm/confirm_nooverwrite boolean true
+d-i partman-lvm/device_remove_lvm boolean true
+d-i partman/choose_partition select finish
+d-i partman/confirm boolean true
+d-i partman/confirm_nooverwrite boolean true
+d-i partman/confirm_write_new_label boolean true
+d-i pkgsel/include string openssh-server cryptsetup build-essential libssl-dev libreadline-dev zlib1g-dev linux-source dkms nfs-common
+d-i pkgsel/install-language-support boolean false
+d-i pkgsel/update-policy select none
+d-i pkgsel/upgrade select full-upgrade
+d-i time/zone string UTC
+tasksel tasksel/first multiselect standard, ubuntu-server
+
+d-i console-setup/ask_detect boolean false
+d-i keyboard-configuration/layoutcode string us
+d-i keyboard-configuration/modelcode string pc105
+d-i debian-installer/locale string en_US
+
+# Create vagrant user account.
+d-i passwd/user-fullname string vagrant
+d-i passwd/username string vagrant
+d-i passwd/user-password password vagrant
+d-i passwd/user-password-again password vagrant
+d-i user-setup/allow-password-weak boolean true
+d-i user-setup/encrypt-home boolean false
+d-i passwd/user-default-groups vagrant sudo
+d-i passwd/user-uid string 900
+
+# Set root passwords
+d-i rootpw vagrant