- Adding separation between ELK stack server and developer website in…

… which we are instrumenting the user's interaction

- Change vagrant file setup to allow two boxes ("elk" and "developer")
- Added installation scripts into Vagrant file provisioning

Vagrantfile

@@ -0,0 +1,65 @@
+Vagrant.configure(2) do |config|
+
+    # Setting proxy configurations for the host box. This also sets common proxy settings
+    # and files for other applications, such as apt-get/yum
+    # if Vagrant.has_plugin?("vagrant-proxyconf")
+    #     config.proxy.http = "http://127.0.0.1:3128"
+    #     config.proxy.https = "http://127.0.0.1:3128"
+    #     config.proxy.no_proxy = "localhost,127.0.0.1"
+    # end
+
+    config.vm.define "elk" do |elk|
+        elk.vm.box = "ubuntu/trusty64"
+
+        # Change the default elk vagrant box folder to point to the 
+        # elk directory within the project. This will allow separation between
+        # elk and developer folders.
+        elk.vm.synced_folder "dashboard/", "/vagrant"
+
+        # Network configuration:
+        # - Setups a static IP address to allow the client Vagrant box 
+        # to know where to connect within the local network of Vagrant 
+        # boxes.
+        # - Expose the following ports to be used within this box to
+        # host data being sent between the web server and the ELK server.
+        elk.vm.network "private_network", ip: "192.168.86.100"
+
+        # Provisioner: Runs the provisioning script that will provision
+        # the vagrant box for the first time, or forced. 
+        elk.vm.provision "shell" do |s| 
+            s.path = "dashboard/scripts/install.sh"
+            s.privileged = false
+        end
+
+        # Provisioner: Run this script always. This allows the box to setup
+        # the elk server and web service all the times. This could be done
+        # in an initrc file, but this will do.
+        elk.vm.provision "shell", path: "dashboard/scripts/restart.sh", run: "always"
+
+        # Host configuration: Set specific requirements for the host to
+        # provide the Guest Box to use.
+        elk.vm.provider :virtualbox do |vb|
+            vb.customize ["modifyvm", :id, "--cpus", "2", "--memory", "2048"]
+        end
+    end
+
+    config.vm.define "developer" do |dev|
+        # Specify a base virtual machine that is based on Ubuntu Trusty Tahr
+        dev.vm.box = "ubuntu/trusty64"
+
+        # Setup a static IP to allow both vagrant boxes to know where 
+        # to contact each other. This will allow communication between the
+        # web developer and the logging server.
+        dev.vm.network "private_network", ip: "192.168.86.10"
+
+        # Specify the provisioning script that will be used in order to 
+        # install the necessary files needed for this vagrant box
+        dev.vm.provision "shell", inline: "twistd -y /vagrant/twisted_client.py &", 
+            run: "always"
+
+        # Change the default client vagrant box folder to point to the 
+        # client directory within the project. This will allow separation between
+        # client and server folders.
+        dev.vm.synced_folder "client/", "/vagrant"
+    end
+end

client/twisted_client.py

@@ -0,0 +1,17 @@
+from twisted.web.server import Site
+from twisted.web.resource import Resource
+from twisted.internet import reactor
+from twisted.web.static import File
+
+# Create a /test web resource which points to the /www directory
+# stored within the client folder. This directory contains the example
+# webpage that the user will interact with.
+root = Resource()
+root.putChild("test", File("/vagrant/www"))
+factory = Site(root)
+
+# Enable the webserver and listen on port 9000 
+# so we can distinguish this web server to regular 
+# port 80 web servers.
+reactor.listenTCP(80, factory)
+reactor.run()

dashboard/test/index.html → client/www/index.html

@@ -61,7 +61,7 @@
 	.mute(['SYS']); // don't log SYSTEM actions
 
 	ac.registerActivityLogger(
-		"http://localhost:9000",
+		"http://192.168.86.100",
 		"my-component",
 		"v0.1"
 		);
@@ -73,4 +73,4 @@
 	);
 	</script>
 </body>
-</html>
+</html>

dashboard/scripts/install.sh

@@ -1,46 +1,35 @@
-#!/usr/bin/env bash
-
-#note: apt-get proxy settings require http:// prefix
-sudo -E apt-get update
-
-#if there are errors about prereqs force install them with: sudo -E apt-get -y -f install
-sudo -E apt-get -y install openjdk-7-jdk
-
-wget http://repo.continuum.io/miniconda/Miniconda-3.7.0-Linux-x86_64.sh
-
-bash Miniconda-3.7.0-Linux-x86_64.sh -b
+#!/bin/bash
+MINICONDA_SCRIPT="http://repo.continuum.io/miniconda/Miniconda-3.7.0-Linux-x86_64.sh"
+ELASTIC_DPKG_SRC="https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.4.2.deb"
+LOGSTASH_DPKG_SRC="https://download.elasticsearch.org/logstash/logstash/packages/debian/logstash_1.4.2-1-2c0f5a1_all.deb"
+KIBANA_SRC="https://download.elasticsearch.org/kibana/kibana/kibana-3.1.2.tar.gz"
+
+sudo -E apt-get update                             || exit $?
+sudo -E apt-get -y install openjdk-7-jdk           || exit $?
+wget -q $MINICONDA_SCRIPT                          || exit $?
+chmod +x ./Miniconda-3.7.0-Linux-x86_64.sh         || exit $?
+./Miniconda-3.7.0-Linux-x86_64.sh -b               || exit $?
 
 #wget http://09c8d0b2229f813c1b93-c95ac804525aac4b6dba79b00b39d1d3.r79.cf1.rackcdn.com/Anaconda-2.1.0-Linux-x86_64.sh
-
-
-echo export PATH="/home/vagrant/miniconda/bin:$PATH" >> $HOME/.bashrc
-
+echo export PATH="$HOME/miniconda/bin:$PATH" >> $HOME/.bashrc
 source $HOME/.bashrc
+$HOME/miniconda/bin/conda update --yes conda     || exit $?
+
+wget -q $ELASTIC_DPKG_SRC $LOGSTASH_DPKG_SRC     || exit $?
+sudo dpkg -i elasticsearch-1.4.2.deb             || exit $?
+sudo dpkg -i logstash_1.4.2-1-2c0f5a1_all.deb    || exit $?
+
+# Download and install Kibana to the vagrant box. This involves downloading
+# Kibana 3.1.2, extracting the contents of the tar ball, and copying the
+# kibanan files to /etc/elasticsearch and /etc/logstash
+wget -q $KIBANA_SRC                                           || exit $?
+tar -xvf kibana-3.1.2.tar.gz                                  || exit $?
+sudo cp /vagrant/files/elasticsearch.yml /etc/elasticsearch/  || exit $?
+sudo cp /vagrant/files/xdata.conf /etc/logstash/conf.d/       || exit $?
+
+# Restart all the services to ensure the configurations are being used properly
+# and Run the kibana twisted web server so the developer has access to the
+# dashboad provided by Kibana.
+sudo mkdir /var/log/xdata                          || exit $?
+sudo touch  /var/log/xdata/xdata.log               || exit $?
 
-conda update --yes conda
-
-wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.4.2.deb
-sudo dpkg -i elasticsearch-1.4.2.deb 
-sudo service elasticsearch start
-
-wget https://download.elasticsearch.org/logstash/logstash/packages/debian/logstash_1.4.2-1-2c0f5a1_all.deb
-sudo dpkg -i logstash_1.4.2-1-2c0f5a1_all.deb
-sudo service logstash start
-
-wget https://download.elasticsearch.org/kibana/kibana/kibana-3.1.2.tar.gz
-tar -xvf kibana-3.1.2.tar.gz
-
-sudo cp /vagrant/files/elasticsearch.yml /etc/elasticsearch/
-sudo cp /vagrant/files/xdata.conf /etc/logstash/conf.d/
-
-#config
-#/etc/elasticsearch/elasticsearch.yml
-
-#logstash conf
-# /etc/logstash/conf.d/test.conf
-
-# create required log dir
-sudo mkdir /var/log/xdata
-
-# start twisted
-sudo twistd -y /vagrant/twisted_app.py &

dashboard/scripts/restart.sh

@@ -1,3 +1,20 @@
+sudo service elasticsearch restart
+sudo service logstash restart
+
+# For Logstash and ElasticSearch, it takes a while before the
+# network port is established by the process. Here we wait until
+# the port is open.
+# TODO: Exit counter.
+while true;
+do
+	nc -z localhost 9200
+	if [ "$?" == "0" ]; then
+		break
+	fi
+	sleep 1;
+done
+
+
 curl -XDELETE 'http://127.0.0.1:9200/xdata/'
 
 curl -XPUT 'http://127.0.0.1:9200/xdata/'
@@ -78,5 +95,6 @@ curl -XPUT 'http://127.0.0.1:9200/xdata/_mapping/USERACTION' -d '
 }
 '
 
-# sudo service elasticsearch restart
-# sudo service logstash restart
+# Start the webservice that allows us to ping ELK and
+# dump data to the ELK service
+sudo -E twistd -y /vagrant/twisted_app.py &

dashboard/twisted_app.py

@@ -64,7 +64,7 @@
 logger_err = logging.getLogger('error')
 
 kibana = File('/home/vagrant/kibana-3.1.2')
-test = File('/vagrant/test')
+#test = File('/vagrant/test')
 
 wf_dict = {
     0: "WF_OTHER",
@@ -86,9 +86,21 @@ def render_GET(self, request):
         return "I am request #" + str(self.numberRequests) + "\n"
 
 class Logger(Resource):
+    def render_OPTIONS(self, request):
+        request.setHeader('Access-Control-Allow-Origin', 'http://192.168.86.10')
+        request.setHeader('Access-Control-Allow-Methods', 'POST')
+        request.setHeader('Access-Control-Allow-Headers', 'x-prototype-version,x-requested-with,Content-Type')
+        request.setHeader('Access-Control-Max-Age', 2520) # 42 hours
+        return ''
+
     def render_POST(self, request):
         newdata = request.content.getvalue()
         newdata = simplejson.loads(newdata)
+        request.setHeader('Access-Control-Allow-Origin', 'http://192.168.86.10')
+        request.setHeader('Access-Control-Allow-Methods', 'POST')
+        request.setHeader('Access-Control-Allow-Headers', 'x-prototype-version,x-requested-with,Content-Type')
+        request.setHeader('Access-Control-Max-Age', 2520) # 42 hours
+
         try:
             for a in newdata:
                 if 'wf_state' in a['parms']:
@@ -102,10 +114,8 @@ def render_POST(self, request):
 
 root = Resource()
 root.putChild("kibana", kibana)
-root.putChild("test", test)
-
 root.putChild("counter", Counter())
 root.putChild("send_log", Logger())
 
-reactor.listenTCP(9000, Site(root))
-reactor.run()
+reactor.listenTCP(80, Site(root))
+reactor.run()