Skip to content
This repository has been archived by the owner. It is now read-only.
Permalink
Browse files
- Adding separation between ELK stack server and developer website in…
… which we are instrumenting the user's interaction

- Change vagrant file setup to allow two boxes ("elk" and "developer")
- Added installation scripts into Vagrant file provisioning
  • Loading branch information
Felipe V committed Mar 9, 2015
1 parent 8bb3246 commit 1c213517df635094e5d7bbd0c8054aabb7bc552d
Showing 9 changed files with 150 additions and 66 deletions.
@@ -0,0 +1,65 @@
Vagrant.configure(2) do |config|

# Setting proxy configurations for the host box. This also sets common proxy settings
# and files for other applications, such as apt-get/yum
# if Vagrant.has_plugin?("vagrant-proxyconf")
# config.proxy.http = "http://127.0.0.1:3128"
# config.proxy.https = "http://127.0.0.1:3128"
# config.proxy.no_proxy = "localhost,127.0.0.1"
# end

config.vm.define "elk" do |elk|
elk.vm.box = "ubuntu/trusty64"

# Change the default elk vagrant box folder to point to the
# elk directory within the project. This will allow separation between
# elk and developer folders.
elk.vm.synced_folder "dashboard/", "/vagrant"

# Network configuration:
# - Setups a static IP address to allow the client Vagrant box
# to know where to connect within the local network of Vagrant
# boxes.
# - Expose the following ports to be used within this box to
# host data being sent between the web server and the ELK server.
elk.vm.network "private_network", ip: "192.168.86.100"

# Provisioner: Runs the provisioning script that will provision
# the vagrant box for the first time, or forced.
elk.vm.provision "shell" do |s|
s.path = "dashboard/scripts/install.sh"
s.privileged = false
end

# Provisioner: Run this script always. This allows the box to setup
# the elk server and web service all the times. This could be done
# in an initrc file, but this will do.
elk.vm.provision "shell", path: "dashboard/scripts/restart.sh", run: "always"

# Host configuration: Set specific requirements for the host to
# provide the Guest Box to use.
elk.vm.provider :virtualbox do |vb|
vb.customize ["modifyvm", :id, "--cpus", "2", "--memory", "2048"]
end
end

config.vm.define "developer" do |dev|
# Specify a base virtual machine that is based on Ubuntu Trusty Tahr
dev.vm.box = "ubuntu/trusty64"

# Setup a static IP to allow both vagrant boxes to know where
# to contact each other. This will allow communication between the
# web developer and the logging server.
dev.vm.network "private_network", ip: "192.168.86.10"

# Specify the provisioning script that will be used in order to
# install the necessary files needed for this vagrant box
dev.vm.provision "shell", inline: "twistd -y /vagrant/twisted_client.py &",
run: "always"

# Change the default client vagrant box folder to point to the
# client directory within the project. This will allow separation between
# client and server folders.
dev.vm.synced_folder "client/", "/vagrant"
end
end
@@ -0,0 +1,17 @@
from twisted.web.server import Site
from twisted.web.resource import Resource
from twisted.internet import reactor
from twisted.web.static import File

# Create a /test web resource which points to the /www directory
# stored within the client folder. This directory contains the example
# webpage that the user will interact with.
root = Resource()
root.putChild("test", File("/vagrant/www"))
factory = Site(root)

# Enable the webserver and listen on port 9000
# so we can distinguish this web server to regular
# port 80 web servers.
reactor.listenTCP(80, factory)
reactor.run()
@@ -61,7 +61,7 @@
.mute(['SYS']); // don't log SYSTEM actions

ac.registerActivityLogger(
"http://localhost:9000",
"http://192.168.86.100",
"my-component",
"v0.1"
);
@@ -73,4 +73,4 @@
);
</script>
</body>
</html>
</html>

This file was deleted.

@@ -1,46 +1,35 @@
#!/usr/bin/env bash

#note: apt-get proxy settings require http:// prefix
sudo -E apt-get update

#if there are errors about prereqs force install them with: sudo -E apt-get -y -f install
sudo -E apt-get -y install openjdk-7-jdk

wget http://repo.continuum.io/miniconda/Miniconda-3.7.0-Linux-x86_64.sh

bash Miniconda-3.7.0-Linux-x86_64.sh -b
#!/bin/bash
MINICONDA_SCRIPT="http://repo.continuum.io/miniconda/Miniconda-3.7.0-Linux-x86_64.sh"
ELASTIC_DPKG_SRC="https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.4.2.deb"
LOGSTASH_DPKG_SRC="https://download.elasticsearch.org/logstash/logstash/packages/debian/logstash_1.4.2-1-2c0f5a1_all.deb"
KIBANA_SRC="https://download.elasticsearch.org/kibana/kibana/kibana-3.1.2.tar.gz"

sudo -E apt-get update || exit $?
sudo -E apt-get -y install openjdk-7-jdk || exit $?
wget -q $MINICONDA_SCRIPT || exit $?
chmod +x ./Miniconda-3.7.0-Linux-x86_64.sh || exit $?
./Miniconda-3.7.0-Linux-x86_64.sh -b || exit $?

#wget http://09c8d0b2229f813c1b93-c95ac804525aac4b6dba79b00b39d1d3.r79.cf1.rackcdn.com/Anaconda-2.1.0-Linux-x86_64.sh


echo export PATH="/home/vagrant/miniconda/bin:$PATH" >> $HOME/.bashrc

echo export PATH="$HOME/miniconda/bin:$PATH" >> $HOME/.bashrc
source $HOME/.bashrc
$HOME/miniconda/bin/conda update --yes conda || exit $?

wget -q $ELASTIC_DPKG_SRC $LOGSTASH_DPKG_SRC || exit $?
sudo dpkg -i elasticsearch-1.4.2.deb || exit $?
sudo dpkg -i logstash_1.4.2-1-2c0f5a1_all.deb || exit $?

# Download and install Kibana to the vagrant box. This involves downloading
# Kibana 3.1.2, extracting the contents of the tar ball, and copying the
# kibanan files to /etc/elasticsearch and /etc/logstash
wget -q $KIBANA_SRC || exit $?
tar -xvf kibana-3.1.2.tar.gz || exit $?
sudo cp /vagrant/files/elasticsearch.yml /etc/elasticsearch/ || exit $?
sudo cp /vagrant/files/xdata.conf /etc/logstash/conf.d/ || exit $?

# Restart all the services to ensure the configurations are being used properly
# and Run the kibana twisted web server so the developer has access to the
# dashboad provided by Kibana.
sudo mkdir /var/log/xdata || exit $?
sudo touch /var/log/xdata/xdata.log || exit $?

conda update --yes conda

wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.4.2.deb
sudo dpkg -i elasticsearch-1.4.2.deb
sudo service elasticsearch start

wget https://download.elasticsearch.org/logstash/logstash/packages/debian/logstash_1.4.2-1-2c0f5a1_all.deb
sudo dpkg -i logstash_1.4.2-1-2c0f5a1_all.deb
sudo service logstash start

wget https://download.elasticsearch.org/kibana/kibana/kibana-3.1.2.tar.gz
tar -xvf kibana-3.1.2.tar.gz

sudo cp /vagrant/files/elasticsearch.yml /etc/elasticsearch/
sudo cp /vagrant/files/xdata.conf /etc/logstash/conf.d/

#config
#/etc/elasticsearch/elasticsearch.yml

#logstash conf
# /etc/logstash/conf.d/test.conf

# create required log dir
sudo mkdir /var/log/xdata

# start twisted
sudo twistd -y /vagrant/twisted_app.py &
@@ -1,3 +1,20 @@
sudo service elasticsearch restart
sudo service logstash restart

# For Logstash and ElasticSearch, it takes a while before the
# network port is established by the process. Here we wait until
# the port is open.
# TODO: Exit counter.
while true;
do
nc -z localhost 9200
if [ "$?" == "0" ]; then
break
fi
sleep 1;
done


curl -XDELETE 'http://127.0.0.1:9200/xdata/'

curl -XPUT 'http://127.0.0.1:9200/xdata/'
@@ -78,5 +95,6 @@ curl -XPUT 'http://127.0.0.1:9200/xdata/_mapping/USERACTION' -d '
}
'

# sudo service elasticsearch restart
# sudo service logstash restart
# Start the webservice that allows us to ping ELK and
# dump data to the ELK service
sudo -E twistd -y /vagrant/twisted_app.py &
@@ -64,7 +64,7 @@
logger_err = logging.getLogger('error')

kibana = File('/home/vagrant/kibana-3.1.2')
test = File('/vagrant/test')
#test = File('/vagrant/test')

wf_dict = {
0: "WF_OTHER",
@@ -86,9 +86,21 @@ def render_GET(self, request):
return "I am request #" + str(self.numberRequests) + "\n"

class Logger(Resource):
def render_OPTIONS(self, request):
request.setHeader('Access-Control-Allow-Origin', 'http://192.168.86.10')
request.setHeader('Access-Control-Allow-Methods', 'POST')
request.setHeader('Access-Control-Allow-Headers', 'x-prototype-version,x-requested-with,Content-Type')
request.setHeader('Access-Control-Max-Age', 2520) # 42 hours
return ''

def render_POST(self, request):
newdata = request.content.getvalue()
newdata = simplejson.loads(newdata)
request.setHeader('Access-Control-Allow-Origin', 'http://192.168.86.10')
request.setHeader('Access-Control-Allow-Methods', 'POST')
request.setHeader('Access-Control-Allow-Headers', 'x-prototype-version,x-requested-with,Content-Type')
request.setHeader('Access-Control-Max-Age', 2520) # 42 hours

try:
for a in newdata:
if 'wf_state' in a['parms']:
@@ -102,10 +114,8 @@ def render_POST(self, request):

root = Resource()
root.putChild("kibana", kibana)
root.putChild("test", test)

root.putChild("counter", Counter())
root.putChild("send_log", Logger())

reactor.listenTCP(9000, Site(root))
reactor.run()
reactor.listenTCP(80, Site(root))
reactor.run()

0 comments on commit 1c21351

Please sign in to comment.