#12: First working containerization setup.

docker/boot-infrastructure.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+set -e
+
+# The full path to this script, no matter where it is called from
+SCRIPTDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+
+if [ ! -e docker ]
+then
+    echo "You need to install Docker in order to boot the JourneyMonitor docker infrastructure"
+    exit 1
+fi
+
+if [ ! -d ${SCRIPTDIR}/../../control ] || [ ! -d ${SCRIPTDIR}/../../monitor ]
+then
+    echo "This script expects checkouts of the 'control' and 'monitor' modules at the same level as this 'infra' checkout."
+    exit 1
+fi
+
+if [ -z $(docker images -q journeymonitor/control:latest 2> /dev/null) ]
+then
+    echo "Image 'journeymonitor/control:latest' doesn't exist, creating..."
+    /usr/bin/env bash ${SCRIPTDIR}/../../control/docker/build-app-container.sh
+fi
+
+if [ -z $(docker images -q journeymonitor/monitor:latest 2> /dev/null) ]
+then
+    echo "Image 'journeymonitor/monitor:latest' doesn't exist, creating..."
+    /usr/bin/env bash ${SCRIPTDIR}/../../monitor/docker/build-app-container.sh
+fi
+
+docker network create journeymonitor 2> /dev/null || true
+
+echo "Starting image 'journeymonitor/control:latest' as container 'journeymonitor-control'"
+/usr/bin/env bash ${SCRIPTDIR}/../../control/docker/run-app-container.sh
+
+echo "Starting image 'journeymonitor/monitor:latest' as container 'journeymonitor-monitor'"
+/usr/bin/env bash ${SCRIPTDIR}/../../monitor/docker/run-app-container.sh
+
+echo "All done."
+echo "If you need a shell on one of the containers, go to the project folder (e.g. '../control') and issue 'make docker-shell'"

puppet/hieradata/common.yaml

@@ -1,6 +1,5 @@
 ---
 classes:
-  - base
   - unattended-upgrades
   - simplecd
   - user-root

puppet/hieradata/env/dev.yaml

@@ -12,10 +12,12 @@ cassandra::rpc_address: 0.0.0.0
 cassandra::broadcast_rpc_address: 127.0.0.1
 app-analyze::cassandra_cqlsh_host: &cassandra_cqlsh_host 127.0.0.1
 app-analyze::spark_master_address: *spark_master_address
-app-analyze::endpoint_monitor_api: &endpoint_monitor_api http://127.0.0.1:8081/
+app-analyze::endpoint_monitor_api: &endpoint_monitor_api http://journeymonitor-monitor:8081/
+app-analyze::endpoint_control_api: &endpoint_control_api http://journeymonitor-control:8082/
 app-monitor::api:allowed_ips:
   - 127.0.0.1
 cronjobs::endpoint_monitor_api: *endpoint_monitor_api
+cronjobs::endpoint_control_api: *endpoint_control_api
 cronjobs::endpoint_analyze_api: "http://127.0.0.1:%{hiera('env::app-analyze:api:listen_port')}/"
 
 # The following are used in app environments (see env-mgmt module), but never in Puppet manifests

puppet/hieradata/env/prod.yaml

@@ -13,9 +13,11 @@ cassandra::broadcast_rpc_address: 127.0.0.1
 app-analyze::cassandra_cqlsh_host: &cassandra_cqlsh_host 10.31.32.82
 app-analyze::spark_master_address: *spark_master_address
 app-analyze::endpoint_monitor_api: &endpoint_monitor_api http://journeymonitor.com:8081/
+app-analyze::endpoint_control_api: &endpoint_control_api http://127.0.0.1:8082/
 app-monitor::api:allowed_ips:
   - 80.69.33.177 # net-lab experiments systems come from this address
 cronjobs::endpoint_monitor_api: *endpoint_monitor_api
+cronjobs::endpoint_control_api: *endpoint_control_api
 cronjobs::endpoint_analyze_api: "http://80.69.45.171:%{hiera('env::app-analyze:api:listen_port')}/"
 
 # The following are used in app environments (see env-mgmt module), but never in Puppet manifests

puppet/hieradata/host/control.container.yaml

@@ -0,0 +1,3 @@
+---
+env: dev
+role: control-app-container

puppet/hieradata/host/monitor.container.yaml

@@ -1,3 +1,3 @@
 ---
 env: dev
-role: monitor-app-server
+role: monitor-app-container

puppet/hieradata/role/analyze-app-master.yaml

@@ -1,5 +1,6 @@
 ---
 classes:
+  - base
   - postfix
   - spark-master
   - app-analyze

puppet/hieradata/role/analyze-app-slave.yaml

@@ -1,5 +1,6 @@
 ---
 classes:
+  - base
   - postfix
   - spark-slave
   - cassandra

puppet/hieradata/role/control-and-monitor-app-server.yaml

@@ -1,5 +1,6 @@
 ---
 classes:
+  - base
   - selenium
   - browsermob-proxy
   - php5

puppet/hieradata/role/control-app-container.yaml

@@ -0,0 +1,15 @@
+---
+classes:
+  - base-minimal
+  - base
+  - php5
+  - composer
+  - user-journeymonitor
+  - nodejs
+  - bower
+  - cronjobs
+  - postfix
+  - nginx
+
+applications:
+  - control

puppet/hieradata/role/control-app-server.yaml

@@ -1,5 +1,6 @@
 ---
 classes:
+  - base
   - php5
   - composer
   - user-journeymonitor

puppet/hieradata/role/monitor-app-container.yaml

@@ -0,0 +1,17 @@
+---
+classes:
+  - base-minimal
+  - selenium
+  - browsermob-proxy
+  - php5
+  - composer
+  - user-journeymonitor
+  - cronjobs
+  - postfix
+  - nginx
+  - jre7
+  - jre8
+  - app-monitor
+
+applications:
+  - monitor

puppet/hieradata/role/monitor-app-server.yaml

@@ -1,5 +1,6 @@
 ---
 classes:
+  - base
   - selenium
   - browsermob-proxy
   - php5

puppet/modules/base-minimal/manifests/init.pp

@@ -0,0 +1,27 @@
+class base-minimal {
+
+  package {
+    [ "vim",
+      "unzip",
+      "git",
+      "sqlite",
+      "curl",
+      "make",
+      "software-properties-common" # for add-apt-repository
+    ]:
+      ensure => present,
+      require => [ Exec["apt-get update"] ],
+  }
+
+  file { ["/opt/puppet", "/opt/puppet/install", "/opt/puppet/flags"]:
+    ensure => "directory",
+    owner  => "root",
+    group  => "root",
+    mode   => 0755,
+  }
+
+  exec { "apt-get update":
+    command => "/usr/bin/apt-get update",
+  }
+
+}

puppet/modules/base/manifests/init.pp

@@ -1,36 +1,20 @@
 class base {
 
+  include base-minimal
+
   package {
     [ "openssh-server",
       "ethtool",
       "tcpdump",
-      "vim",
       "htop",
       "iotop",
       "sysstat",
       "acpid",
-      "unzip",
       "bwm-ng",
-      "make",
-      "g++",
-      "curl",
-      "sqlite",
-      "git",
-      "software-properties-common"  # for add-apt-repository
+      "g++"
     ]:
       ensure => present,
-      require => Exec["apt-get update"],
-  }
-
-  file { ["/opt/puppet", "/opt/puppet/install", "/opt/puppet/flags"]:
-    ensure => "directory",
-    owner  => "root",
-    group  => "root",
-    mode   => 0755,
-  }
-
-  exec { "apt-get update":
-    command => "/usr/bin/apt-get update",
+      require => [ Class["base-minimal"], Exec["apt-get update"] ]
   }
 
 }

puppet/modules/cronjobs/templates/etc/cron.d/journeymonitor-monitor.erb

@@ -5,7 +5,7 @@ MAILTO=""
 * * * * * root /opt/simplecd/simplecd.sh tag travisci-build-master-* https://github.com/journeymonitor/monitor.git https://github.com/journeymonitor/monitor/commit/ 2>&1 | while IFS= read -r line;do echo "$(date) $line";done >> /var/tmp/simplecd/journeymonitor-monitor.log
 <% end %>
 
-* * * * * root sleep 30 && sudo -u journeymonitor PHP_ENV=<%= @env %> /usr/bin/php /opt/journeymonitor/monitor/bin/import.php http://127.0.0.1:8082/api/internal/testcases.json 2>&1 | while IFS= read -r line;do echo "$(date) $line";done >> /var/tmp/journeymonitor-monitor-import.log && rm -f /etc/cron.d/journeymonitor-run-* && sudo -u journeymonitor PHP_ENV=<%= @env %> /usr/bin/php /opt/journeymonitor/monitor/bin/create.php 2>&1 | while IFS= read -r line;do echo "$(date) $line";done >> /var/tmp/journeymonitor-monitor-create.log && chown root:root /etc/cron.d/journeymonitor-run-* && chmod 0644 /etc/cron.d/journeymonitor-run-*
+* * * * * root sleep 30 && sudo -u journeymonitor PHP_ENV=<%= @env %> /usr/bin/php /opt/journeymonitor/monitor/bin/import.php <%= scope.function_hiera(['cronjobs::endpoint_control_api']) %>api/internal/testcases.json 2>&1 | while IFS= read -r line;do echo "$(date) $line";done >> /var/tmp/journeymonitor-monitor-import.log && rm -f /etc/cron.d/journeymonitor-run-* && sudo -u journeymonitor PHP_ENV=<%= @env %> /usr/bin/php /opt/journeymonitor/monitor/bin/create.php 2>&1 | while IFS= read -r line;do echo "$(date) $line";done >> /var/tmp/journeymonitor-monitor-create.log && chown root:root /etc/cron.d/journeymonitor-run-* && chmod 0644 /etc/cron.d/journeymonitor-run-*
 
 # Selenium runs create a lot of cruft in /tmp that needs to be cleaned up
 */10 * * * * root /usr/bin/find /tmp -name "seleniumSslSupport*" -type d -mmin +60 | xargs rm -rf

puppet/modules/jre8/manifests/init.pp

@@ -5,9 +5,14 @@
     creates => "/etc/apt/sources.list.d/openjdk-r-ppa-trusty.list"
   }
 
+  exec { "apt-get update after adding openjdk-8-jdk ppa":
+    command => "/usr/bin/apt-get update",
+    require => [ Exec["add openjdk-8-jdk ppa for jre8"] ]
+  }
+
   package { ["openjdk-8-jre-headless"]:
     ensure  => "installed",
-    require => [ Exec["add openjdk-8-jdk ppa for jre8"], Exec["apt-get update"] ]
+    require => [ Exec["add openjdk-8-jdk ppa for jre8"], Exec["apt-get update after adding openjdk-8-jdk ppa"] ]
   }
 
   exec { "make openjdk-8-jre the default Java":

puppet/modules/nginx/templates/etc/nginx/sites-available/journeymonitor.com.erb

@@ -2,7 +2,7 @@
 
 server {
 
-	server_name journeymonitor.com www.journeymonitor.com <% if @app_env == "dev" %>192.168.59.99<% end %>;
+	server_name journeymonitor.com www.journeymonitor.com <% if @app_env == "dev" %>127.0.0.1<% end %>;
 	listen 0.0.0.0:80;
 	access_log /var/log/nginx/journeymonitor.com.access.log;
 	error_log /var/log/nginx/journeymonitor.com.error.log;
@@ -39,6 +39,7 @@ server {
 
 	server_name control-api;
 	listen 127.0.0.1:8082;
+	<% if @app_env == "dev" %>listen 0.0.0.0:8082;<% end %>
 	access_log /var/log/nginx/journeymonitor-control-api.access.log;
 	error_log /var/log/nginx/journeymonitor-control-api.error.log;
 	charset utf-8;
@@ -99,6 +100,7 @@ server {
 
 	server_name monitor-media;
 	listen 127.0.0.1:8083;
+	<% if @app_env == "dev" %>listen 0.0.0.0:8083;<% end %>
 	access_log /var/log/nginx/journeymonitor-monitor-media.access.log;
 	error_log /var/log/nginx/journeymonitor-monitor-media.error.log;
 	charset utf-8;