Logging

omarqureshi · Sep 19, 2012 · 95d533d · 95d533d
1 parent 9fe508e
commit 95d533d
Show file tree

Hide file tree

Showing 23 changed files with 387 additions and 38 deletions.
diff --git a/manifests/nodes.pp b/manifests/nodes.pp
@@ -21,10 +21,12 @@
       { hostname => "app.production.translator.edisonnation.com", ip => "10.183.37.2"},
       { hostname => "app1.production.edisonnation.com", ip => "10.176.42.95" },
       { hostname => "app2.production.edisonnation.com", ip => "10.176.42.155" },
+      { hostname => "app3.production.edisonnation.com", ip => "10.183.162.189" },
       { hostname => "db.production.edisonnation.com", ip => "10.176.42.86" },
       { hostname => "cache.production.edisonnation.com", ip => "10.183.173.51" },
       { hostname => "jobs.production.edisonnation.com", ip => "10.183.170.51" },
       { hostname => "assets.production.edisonnation.com", ip => "10.183.173.1" },
+      { hostname => "logs.edisonnation.com", ip => "10.183.170.37" },
     ],
   }
 }
@@ -81,10 +83,10 @@
     group => "www",
     mode => 750,
   }
+  package {"openjdk-6-jre": ensure => installed }
 }
 
 node 'en-logs' inherits 'ruby-193' {
-  package {"openjdk-6-jre": ensure => installed }
   class {"mongodb": auth => true }
   wget::fetch {"elasticsearch":
     source => "https://github.com/downloads/elasticsearch/elasticsearch/elasticsearch-0.19.8.deb",
@@ -100,7 +102,8 @@
     ensure => running,
     require => Package["elasticsearch"]
   }
-  include 'graylog'
+
+  class {'graylog': dirname => 'logs.edisonnation.com', www_user => "www" }
   iptables::role { "graylog": }
   nginx::unicorn_app { 'logs.edisonnation.com':
     require => Class["graylog"],
@@ -119,7 +122,9 @@
       ensure => latest,
       require => [Rvm_system_ruby['1.9.3-p194'], Rvm_gemset["ruby-1.9.3-p194@graylog"]]
   }
-
+  class {"graylog_god_wrapper": role => "app", env => "production" }
+  env_setup::rails_env { 'production': }
+  env_setup::role { 'app': }
 }
 
 node 'ruby-187' inherits basenode {
@@ -206,14 +211,17 @@
     log => "/var/www/edisonnation.com/current/log/*.log",
     options => ["daily", "size 100M", "missingok", "rotate 15", "compress", "delaycompress", "notifempty", "copytruncate"]
   }
+  include 'rsyslog'
+  package {"wkhtmltopdf": ensure => installed}
 }
 
 node 'en-tesla-ci' inherits 'en-tesla' {
-  nginx::jenkins_site { 'edisonnation.com': }
+  nginx::jenkins_site { 'edisonnation.com':
+    passwdloc => 'en-staging', 
+  }
   include mysql::server
   include jenkins
-  package {"imagemagick": ensure => installed }
-  package {"libmagick9-dev": ensure => installed }
+  iptables::role { "web-server": }
 }
 
 node 'en-db' inherits 'en-tesla' {
@@ -233,6 +241,7 @@
 }
 
 node 'en-jobs' inherits 'en-tesla' {
+  iptables::role { "web-server": }
   env_setup::role { "jobs": }
 }
 
@@ -255,6 +264,8 @@
       ensure => latest,
       require => [Rvm_system_ruby['1.8.7-p358'], Rvm_gemset["ruby-1.8.7-p358@tesla"]]
   }
+  include 'logstashd'
+  logstashd::nginx {"nginx": }
 }
 
 node 'en-staging-app' inherits 'en-app' { 
@@ -345,6 +356,14 @@
     nginx::add_redirect { 'edisonnation.com': redirect => 'www.edisonnation.com' }
 }
 
+node 'en-production-app3' inherits 'en-production-app' {
+    nginx::unicorn_site { 'www.edisonnation.com': 
+    assethost => 'assets.production.edisonnation.com', 
+    domain => 'www.edisonnation.com',
+    sslloc => 'en.com' }
+    nginx::add_redirect { 'edisonnation.com': redirect => 'www.edisonnation.com' }
+}
+
 node 'en-production-app2' inherits 'en-production-app' {
     nginx::unicorn_site { 'edisonnationmedical.com': 
     assethost => 'assets.production.edisonnation.com',

diff --git a/modules/cron/manifests/init.pp b/modules/cron/manifests/init.pp
@@ -0,0 +1,23 @@
+# manage cron jobs in separate files - call with enable => "false" to delete the job
+class cron {
+  define create( $enable = "true", $interval = "daily", $script = "", $package = "" ) {
+    file { "/etc/cron.$interval/$name":
+      content         => $script,
+      ensure          => $enable ? {
+        "false" => absent,
+        default => file,
+      },
+      force           => true,
+      owner           => root,
+      group           => root,
+      mode            => $interval ? {
+        "d"     => 644,
+        default => 755,
+      },
+      require         => $package ? {
+        ""      => undef,
+        default => Package[$package],
+      },
+    }
+  }
+}
diff --git a/modules/dnsmasq/templates/dnsmasq.conf.erb b/modules/dnsmasq/templates/dnsmasq.conf.erb
@@ -553,4 +553,5 @@
 
 <% hostnames.each do |hostname| %>
 address=/<%= hostname["hostname"] %>/<%= hostname["ip"] %>
+ptr-record=<%= hostname["ip"].split(".").reverse.join(".") %>.in-addr.arpa,"<%= hostname["hostname"] %>"
 <% end %>
diff --git a/modules/graylog/files/application.god b/modules/graylog/files/application.god
@@ -1,6 +1,6 @@
-PID_DIR       = '/var/www/translator.edisonnation.com/shared/pids'
+PID_DIR       = '/var/www/logs.edisonnation.com/shared/pids'
 RAILS_ENV     = 'production'
-RAILS_ROOT    = '/var/www/translator.edisonnation.com/current'
+RAILS_ROOT    = '/var/www/logs.edisonnation.com/current'
 BIN_PATH      = "/usr/local/rvm/rubies/ruby-1.9.3-p194/bin/ruby"
 
 God.log_file  = "/var/log/god.log"
@@ -11,4 +11,4 @@ God.log_level = :info
   God.load "#{RAILS_ROOT}/config/god/configs/#{config}.god"
 end
 
-require '/var/www/translator.edisonnation.com/current/config/god/configs/contacts.rb'
+require '/var/www/logs.edisonnation.com/current/config/god/configs/contacts.rb'
diff --git a/modules/graylog/files/mongoid.yml b/modules/graylog/files/mongoid.yml
@@ -0,0 +1,6 @@
+production:
+  host: localhost
+  port: 27017
+  username: grayloguser
+  password: 123
+  database: graylog2
diff --git a/modules/graylog/manifests/init.pp b/modules/graylog/manifests/init.pp
@@ -1,5 +1,4 @@
-class graylog($_dirname, $_graylog_ver="0.9.6p1") {
-  $graylog_ver = $_graylog_ver
+class graylog($dirname, $www_user, $graylog_ver="0.9.6p1") {
 
   wget::fetch {"graylog2":
     source => "https://github.com/downloads/Graylog2/graylog2-server/graylog2-server-${graylog_ver}.tar.gz",
@@ -29,61 +28,149 @@
     require => Wget::Fetch["graylog2"],
   }
 
-  file {"/var/www/${_dirname}":
+  file {"/var/www/${dirname}":
     ensure => directory,
     require => [File["/var/www"], Exec["untar_graylog_${graylog_ver}"]],
+    owner => $www_user,
+    group => $www_user,
   }
 
-  file {"/var/www/${_dirname}/current":
+  file {"/var/www/${dirname}/current":
     ensure => link,
     target => "/root/graylog2-web-interface-${graylog_ver}",
-    require => File["/var/www/${_dirname}"],
+    require => File["/var/www/${dirname}"],
   }
 
-  file {"/var/www/${_dirname}/logs":
+  file {"/var/www/${dirname}/shared":
     ensure => directory,
-    require => File["/var/www/${_dirname}/current"],
+    require => File["/var/www/${dirname}"],
+    owner => $www_user,
+    group => $www_user,
   }
 
-  file {"/var/www/${_dirname}/current/config/god":
+  file {"/var/www/${dirname}/shared/log":
     ensure => directory,
-    require => File["/var/www/${_dirname}/current"],
+    require => File["/var/www/${dirname}/shared"],
+    owner => $www_user,
+    group => $www_user,    
   }
 
-  file {"/var/www/${_dirname}/config/god/configs":
+  file {"/var/www/${dirname}/current/log":
+    ensure => link,
+    target => "/var/www/${dirname}/shared/log",
+  }
+
+  file {"/var/www/${dirname}/shared/pids":
+    ensure => directory,
+    require => File["/var/www/${dirname}/shared"],
+    owner => $www_user,
+    group => $www_user,    
+  }
+
+  file {"/var/www/${dirname}/shared/tmp":
+    ensure => directory,
+    require => File["/var/www/${dirname}/shared"],
+    owner => $www_user,
+    group => $www_user,    
+  }
+
+  file {"/var/www/${dirname}/shared/tmp/sockets":
     ensure => directory,
-    require => File["/var/www/${_dirname}/current/config/god"],
+    require => File["/var/www/${dirname}/shared/tmp"],
+    owner => $www_user,
+    group => $www_user,    
   }
 
-  file {"/var/www/${_dirname}/config/god/production":
+
+
+  file {"/var/www/${dirname}/current/config/god":
     ensure => directory,
-    require => File["/var/www/${_dirname}/current/config/god"],
+    require => File["/var/www/${dirname}/current"],
+    owner => $www_user,
+    group => $www_user,    
   }
 
-  file {"/var/www/${_dirname}/config/god/production/app-server":
+  file {"/var/www/${dirname}/current/config/god/configs":
     ensure => directory,
-    require => File["/var/www/${_dirname}/current/config/god/production"],
+    require => File["/var/www/${dirname}/current/config/god"],
+    owner => $www_user,
+    group => $www_user,    
   }
 
+  file {"/var/www/${dirname}/current/config/god/production":
+    ensure => directory,
+    require => File["/var/www/${dirname}/current/config/god"],
+    owner => $www_user,
+    group => $www_user,    
+  }
 
-  file {"/var/www/${_dirname}/config/god/configs/unicorn.god":
-    require => File["/var/www/${_dirname}/current/config/god/configs"],
+  file {"/var/www/${dirname}/current/config/god/production/app-server":
+    ensure => directory,
+    require => File["/var/www/${dirname}/current/config/god/production"],
+    owner => $www_user,
+    group => $www_user,    
+  }
+
+
+  file {"/var/www/${dirname}/current/config/god/configs/unicorn.god":
+    require => File["/var/www/${dirname}/current/config/god/configs"],
     source => "puppet:///modules/graylog/unicorn.god",
+    owner => $www_user,
+    group => $www_user,    
   }
 
-  file {"/var/www/${_dirname}/config/god/configs/nginx.god":
-    require => File["/var/www/${_dirname}/current/config/god/configs"],
+  file {"/var/www/${dirname}/current/config/unicorn.rb":
+    require => File["/var/www/${dirname}/current"],
+    source => "puppet:///modules/graylog/unicorn.rb",
+    owner => $www_user,
+    group => $www_user,    
+  }
+
+
+  file {"/var/www/${dirname}/current/config/god/configs/nginx.god":
+    require => File["/var/www/${dirname}/current/config/god/configs"],
     source => "puppet:///modules/graylog/nginx.god",
+    owner => $www_user,
+    group => $www_user,    
   }
 
-  file {"/var/www/${_dirname}/config/god/configs/contacts.rb":
-    require => File["/var/www/${_dirname}/current/config/god/configs"],
+  file {"/var/www/${dirname}/current/config/god/configs/contacts.rb":
+    require => File["/var/www/${dirname}/current/config/god/configs"],
     source => "puppet:///modules/graylog/contacts.rb",
+    owner => $www_user,
+    group => $www_user,    
+  }
+
+  file {"/var/www/${dirname}/current/config/god/production/app-server/all.god":
+    require => File["/var/www/${dirname}/current/config/god/production/app-server"],
+    source => "puppet:///modules/graylog/application.god",
+    owner => $www_user,
+    group => $www_user,    
+  }
+
+  file {"/var/www/${dirname}/current/config/mongoid.yml":
+    require => File["/var/www/${dirname}/current"],
+    source => "puppet:///modules/graylog/mongoid.yml",
+    owner => $www_user,
+    group => $www_user,    
+  }
+
+
+  file {"/var/www/${dirname}/current/.rvmrc":
+    require => File["/var/www/${dirname}/current"],
+    content => "rvm 1.9.3@graylog",
+    owner => $www_user,
+    group => $www_user,
   }
 
-  file {"/var/www/${_dirname}/config/god/production/app-server/all.god":
-    require => File["/var/www/${_dirname}/current/config/god/production"],
-    source => "puppet:///modules/graylog/all.god",
+  cron::create {"send-subscriptions":
+    interval => "daily",
+    script => "#!/bin/bash
+source /etc/profile.d/rails_env
+source \"/usr/local/rvm/scripts/rvm\"
+rvm 1.9.3@graylog
+cd /var/www/${dirname}/current
+bundle exec rake RAILS_ENV=production subscriptions:send"
   }
 
 }
diff --git a/modules/iptables/files/common.role b/modules/iptables/files/common.role
@@ -6,6 +6,8 @@ iptables -A OUTPUT -p tcp --dport ${SSH} -j ACCEPT
 iptables -A OUTPUT -p tcp --sport ${SSH} -j ACCEPT
 iptables -A OUTPUT -p tcp --dport ${SMTP} -j ACCEPT
 iptables -A OUTPUT -p udp --dport ${NTP} -j ACCEPT
+iptables -A OUTPUT -p udp --dport ${RSYSLOG} -j ACCEPT
+iptables -A OUTPUT -p udp --sport ${RSYSLOG} -j ACCEPT
 iptables -A OUTPUT -p tcp --dport ${NTP} -j ACCEPT
 iptables -A OUTPUT -p udp --dport ${DNS} -j ACCEPT
 iptables -A OUTPUT -p tcp --dport ${WEB} -j ACCEPT
@@ -14,6 +16,8 @@ iptables -A OUTPUT -p tcp --dport ${WEB_SSL} -j ACCEPT
 iptables -A OUTPUT -p tcp --dport ${PUPPET} -j ACCEPT
 iptables -A OUTPUT -p tcp --dport ${MYSQL} -j ACCEPT
 iptables -A OUTPUT -p tcp --dport ${POSTGRES} -j ACCEPT
+iptables -A OUTPUT -p tcp --dport 465 -j ACCEPT
+iptables -A OUTPUT -p udp --dport 12201 -j ACCEPT
 
 # Drop some commonly probed ports
 iptables -A INPUT -p tcp --dport 23 -j DROP # telnet

diff --git a/modules/iptables/files/graylog.role b/modules/iptables/files/graylog.role
@@ -4,3 +4,6 @@ iptables -A INPUT -p tcp -m tcp --dport ${WEB} --tcp-flags SYN,RST,ACK SYN -j AC
 iptables -A INPUT -p tcp -m tcp --dport ${WEB_SSL} --tcp-flags SYN,RST,ACK SYN -j ACCEPT
 iptables -A OUTPUT -p tcp --sport ${WEB} -j ACCEPT
 iptables -A OUTPUT -p tcp --sport ${WEB_SSL} -j ACCEPT
+iptables -A INPUT -p udp --dport ${RSYSLOG} -j ACCEPT
+iptables -A INPUT -p udp --sport ${RSYSLOG} -j ACCEPT
+iptables -A INPUT -p udp --dport 12201 -j ACCEPT
diff --git a/modules/iptables/files/names b/modules/iptables/files/names
@@ -14,6 +14,7 @@ export NRPE=5666
 export NTP=123
 export POSTGRES=5432
 export PUPPET=8140
+export RSYSLOG=514
 export RSYNCD=873
 export SMTP=25
 export SPHINX=3312