add pagerduty notification support / fix minor issues

.env.example

@@ -1,11 +1,10 @@
 SSL_KEY_PATH="/opt/your-cert.com.key"
 SSL_CERT_PATH="/opt/your-cert.com.crt"
 
-
 NETHEALER_SERVER="nethealer.yourdomain.com"
 NETHEALER_INFLUXDB="influxdb.yourdomain.com"
-NETHEALER_USERNAME=""
-NETHEALER_PASSWORD=""
+NETHEALER_USERNAME="influxdb username (r/w)"
+NETHEALER_PASSWORD="influxdb password"
 
 THRESHOLD_EXPIRE="600"
 THRESHOLD_WARNING="3"
@@ -14,3 +13,6 @@ THREDHOLD_CRITICAL="5"
 NOTIFICATION_EMAIL_SMTP="mail.yourdomain.com"
 NOTIFICATION_EMAIL_FROM="healer@yourdomain.com"
 NOTIFICATION_EMAIL_TO="your@email.com"
+
+#optional
+#NOTIFICATION_PAGERDUTY_KEY=""

Gemfile

@@ -12,6 +12,7 @@ gem 'rufus-scheduler', '~> 3.1.3'
 gem 'redis'
 gem 'redis-namespace'
 gem 'influxdb'
+gem 'pagerduty'
 group :test, :development do
   gem 'dotenv', '~> 2.0.2'
   gem 'rack-test', '~> 0.6.3'

app_config.rb

@@ -42,17 +42,14 @@ class << self
     end
   end
 
-
-
-  class JIRA
-    @host = ENV['JIRA_HOSTNAME']
-    @user = ENV['JIRA_USER']
-    @password = ENV['JIRA_PASSWORD']
-
+  class PAGERDUTY
+    @key = ENV['PAGERDUTY_KEY']
+
     class << self
-      attr_reader :host, :user, :password
+      attr_reader :key
     end
   end
+end
 
   class FLOWDOCK
     @ops_flow = ENV['FLOWDOCK_OPS']

script/pooler.rb

@@ -7,6 +7,7 @@
 require 'rest-client'
 require 'influxdb'
 require 'net/smtp'
+require 'pagerduty'
 require 'yaml'
 
 Dotenv.load
@@ -88,28 +89,26 @@ def feed_nethealer(payloads)
 end
 
 def gc_fastnetmon_redis
-  $count += 1
-  if $count > 30
+  if $count > 5
     puts "#{Time.now} - [INFO] - Running garbage collection..." if $debug == 2
     $notifications_warning = []
     $notifications_critical = []
     gc = []
-    pattern = '*_information'
+    pattern = '*_packets_dump'
     $redis_connection.scan_each(:match => pattern) {|key| gc << key.rpartition('_')[0] }
     gc.each do |junk|
       puts "removing null key for #{junk}" if $debug == 2
-      $redis_connection.del("#{junk}_information")
-      $redis_connection.del("#{junk}_flow_dump")
+      #$redis_connection.del("#{junk}_information")
+      #$redis_connection.del("#{junk}_flow_dump")
       $redis_connection.del("#{junk}_packets_dump")
     end
     $count = 0
   end
+  $count += 1
   return true
 end
 
 
-
-
 #
 # Schedulers
 #
@@ -118,7 +117,7 @@ def gc_fastnetmon_redis
 
 scheduler.every '5s' do
   current = []
-  pattern = '*_packets_dump'
+  pattern = '*_information'
   begin
     $redis_connection.scan_each(:match => pattern) {|key| current << key.rpartition('_')[0].rpartition('_')[0] }
   rescue
@@ -212,6 +211,9 @@ def gc_fastnetmon_redis
 # Notification schedulers
 #
 
+pagerduty_enabled = true unless AppConfig::PAGERDUTY.key = "" || AppConfig::PAGERDUTY.key.nil? 
+pagerduty = Pagerduty.new(AppConfig::PAGERDUTY.key) if pagerduty_enabled
+
 $notifications_warning = []
 $notifications_critical = []
 
@@ -231,7 +233,6 @@ def gc_fastnetmon_redis
     reports = reports['reports']
     capture = {}
     reports.each { |k,v| capture["#{k}"] = v.delete('capture') }
-    #top = top_talkers(10)
 
     message = <<MESSAGE_END
 From: DDoS Detection <#{AppConfig::NOTIFICATIONS.smtp_from}>
@@ -252,7 +253,9 @@ def gc_fastnetmon_redis
       Net::SMTP.start(AppConfig::NOTIFICATIONS.smtp) do |smtp|
         smtp.send_message message, AppConfig::NOTIFICATIONS.smtp_from,AppConfig::NOTIFICATIONS.smtp_to
       end
+      incident = pagerduty.trigger("DDoS WARNING: #{reports.to_yaml}") if pagerduty_enabled
       puts "|Notifications_Warning_Sent| - #{Time.now}"
+
     else
       puts "|Notifications_Warning_Skip| - #{Time.now}"
     end
@@ -265,8 +268,7 @@ def gc_fastnetmon_redis
     reports = reports['reports']
     capture = {}
     reports.each { |k,v| capture["#{k}"] = v.delete('capture') }
-    #top = top_talkers(10)
-
+
     message = <<MESSAGE_END
 From: DDoS Detection <#{AppConfig::NOTIFICATIONS.smtp_from}>
 To: Network Operations <#{AppConfig::NOTIFICATIONS.smtp_to}>
@@ -289,6 +291,7 @@ def gc_fastnetmon_redis
       Net::SMTP.start(AppConfig::NOTIFICATIONS.smtp) do |smtp|
         smtp.send_message message, AppConfig::NOTIFICATIONS.smtp_from,AppConfig::NOTIFICATIONS.smtp_to
       end
+      incident = pagerduty.trigger("DDoS CRITICAL: #{reports.to_yaml}") if pagerduty_enabled
       puts "|Notifications_Critical_Sent| - #{Time.now}"
 
     else