Merge pull request #248 from ali-graham/247_split_executive

Split executive into two parts

etc/flapjack_config.yaml.example

@@ -8,6 +8,26 @@ development:
     host: 127.0.0.1
     port: 6379
     db: 13
+  processor:
+    enabled: no
+    queue: events
+    notifier_queue: notifications
+    archive_events: true
+    events_archive_maxage: 10800
+    new_check_scheduled_maintenance_duration: 1 month
+    logger:
+      level: INFO
+  notifier:
+    enabled: no
+    queue: notifications
+    email_queue: email_notifications
+    sms_queue: sms_notifications
+    jabber_queue: jabber_notifications
+    pagerduty_queue: pagerduty_notifications
+    notification_log_file: log/notification.log
+    default_contact_timezone: Australia/Broken_Hill
+    logger:
+      level: INFO
   executive:
     enabled: yes
     email_queue: email_notifications

features/events.feature

@@ -1,4 +1,4 @@
-@events
+@events @processor
 Feature: events
   So people can be notified when things break and recover
   flapjack-executive must process events correctly

features/events_check_names.feature

@@ -1,4 +1,4 @@
-@events
+@events @processor
 Feature: events and check names
   Flapjack must handle weird characters in check names in events
 

features/notification_rules.feature

@@ -1,4 +1,4 @@
-@notification_rules @resque
+@notification_rules @resque @processor @notifier
 Feature: Notification rules on a per contact basis
 
   Background:

features/notifications.feature

@@ -1,4 +1,4 @@
-@notifications
+@notifications @processor @notifier
 Feature: notifications
   So people can be notified when things break and recover
   flapjack-notifier must send notifications correctly

features/steps/events_steps.rb

@@ -2,9 +2,19 @@
 
 def drain_events
   loop do
-    event = Flapjack::Data::Event.next(:block => false, :redis => @redis)
+    event = Flapjack::Data::Event.next('events', :block => false, :redis => @redis)
     break unless event
-    @app.send(:process_event, event)
+    @processor.send(:process_event, event)
+  end
+  drain_notifications
+end
+
+def drain_notifications
+  return unless @notifier_redis
+  loop do
+    notification = Flapjack::Data::Notification.next('notifications', :block => false, :redis => @notifier_redis)
+    break unless notification
+    @notifier.send(:process_notification, notification)
   end
 end
 
@@ -255,16 +265,16 @@ def icecube_schedule_to_time_restriction(sched, time_zone)
 Then /^a notification should not be generated(?: for check '([\w\.\-]+)' on entity '([\w\.\-]+)')?$/ do |check, entity|
   check  ||= @check
   entity ||= @entity
-  message = @logger.messages.find_all {|m| m =~ /enerating notifications for event #{entity}:#{check}/ }.last
-  found = message ? message.match(/Not generating notifications/) : false
+  message = @logger.messages.find_all {|m| m =~ /enerating notification for event #{entity}:#{check}/ }.last
+  found = message ? message.match(/Not generating notification/) : false
   found.should be_true
 end
 
 Then /^a notification should be generated(?: for check '([\w\.\-]+)' on entity '([\w\.\-]+)')?$/ do |check, entity|
   check  ||= @check
   entity ||= @entity
-  message = @logger.messages.find_all {|m| m =~ /enerating notifications for event #{entity}:#{check}/ }.last
-  found = message ? message.match(/Generating notifications/) : false
+  message = @logger.messages.find_all {|m| m =~ /enerating notification for event #{entity}:#{check}/ }.last
+  found = message ? message.match(/Generating notification/) : false
   found.should be_true
 end
 
@@ -280,14 +290,12 @@ def icecube_schedule_to_time_restriction(sched, time_zone)
 end
 
 # added for notification rules:
-
 Given /^the following entities exist:$/ do |entities|
   entities.hashes.each do |entity|
     contacts = entity['contacts'].split(',')
     contacts.map! do |contact|
       contact.strip
     end
-    #puts "adding entity #{entity['name']} (#{entity['id']}) with contacts: [#{contacts.join(', ')}]"
     Flapjack::Data::Entity.add({'id'       => entity['id'],
                                 'name'     => entity['name'],
                                 'contacts' => contacts},
@@ -358,13 +366,6 @@ def icecube_schedule_to_time_restriction(sched, time_zone)
   @redis.keys("drop_alerts_for_contact:#{contact_id}*").should be_empty
 end
 
-# When /^the (\w*) alert block for user (\d*) for (?:the check|check '([\w\.\-]+)' for entity '([\w\.\-]+)') for state (.*) expires$/ do |media, contact, check, entity, state|
-#   check  = check  ? check  : @check
-#   entity = entity ? entity : @entity
-#   num_deleted = @redis.del("drop_alerts_for_contact:#{contact}:#{media}:#{entity}:#{check}:#{state}")
-#   puts "Warning: no keys expired" unless num_deleted > 0
-# end
-
 Then /^(.*) email alert(?:s)? should be queued for (.*)$/ do |num_queued, address|
   check  = check  ? check  : @check
   entity = entity ? entity : @entity

features/steps/notifications_steps.rb

@@ -59,14 +59,27 @@ def add_contact(contact = {})
                              :redis => @redis )
 end
 
+# TODO create the notification object in redis, flag the relevant operation as
+# only needing that part running, split up the before block that covers these
 When /^an event notification is generated for entity '([\w\.\-]+)'$/ do |entity|
   event = Flapjack::Data::Event.new('type'    => 'service',
                                     'state'   => 'critical',
                                     'summary' => '100% packet loss',
                                     'entity'  => entity,
                                     'check'   => 'ping')
+
+  notification_type = Flapjack::Data::Notification.type_for_event(event)
+
   entity_check = Flapjack::Data::EntityCheck.for_entity_name(entity, 'ping', :redis => @redis)
-  @app.send(:generate_notification_messages, event, entity_check, Time.now.to_i)
+  max_notified_severity = entity_check.max_notified_severity_of_current_failure
+
+  severity = Flapjack::Data::Notification.severity_for_event(event, max_notified_severity)
+  last_state = entity_check.historical_state_before(event.time)
+
+  Flapjack::Data::Notification.add('notifications', event,
+    :type => notification_type, :severity => severity, :last_state => last_state,
+    :redis => @redis)
+  drain_notifications
 end
 
 Then /^an SMS notification for entity '([\w\.\-]+)' should be queued for the user$/ do |entity|

features/support/env.rb

@@ -24,7 +24,8 @@
 require 'webmock/cucumber'
 WebMock.disable_net_connect!
 
-require 'flapjack/executive'
+require 'flapjack/notifier'
+require 'flapjack/processor'
 require 'flapjack/patches'
 
 require 'resque_spec'
@@ -124,20 +125,36 @@ def self.time_travel_to(dest_time)
 
 Before do
   @logger = MockLogger.new
-  # Use a separate database whilst testing
-  @app = Flapjack::Executive.new(:logger => @logger,
-    :config => {'email_queue' => 'email_notifications',
-                'sms_queue' => 'sms_notifications',
-                'default_contact_timezone' => 'America/New_York'},
-    :redis_config => redis_opts)
-  @redis = @app.instance_variable_get('@redis')
 end
 
 After do
+  @logger.messages = []
+end
+
+
+Before('@processor') do
+  @processor = Flapjack::Processor.new(:logger => @logger,
+    :redis_config => redis_opts, :config => {})
+  @redis = @processor.instance_variable_get('@redis')
+end
+
+After('@processor') do
   @redis.flushdb
   @redis.quit
-  # Reset the logged messages
-  @logger.messages = []
+end
+
+Before('@notifier') do
+  @notifier  = Flapjack::Notifier.new(:logger => @logger,
+    :redis_config => redis_opts,
+    :config => {'email_queue' => 'email_notifications',
+                'sms_queue' => 'sms_notifications',
+                'default_contact_timezone' => 'America/New_York'})
+  @notifier_redis = @notifier.instance_variable_get('@redis')
+end
+
+After('@notifier') do
+  @notifier_redis.flushdb
+  @notifier_redis.quit
 end
 
 Before('@resque') do

lib/flapjack/coordinator.rb

@@ -7,12 +7,10 @@
 
 require 'flapjack/configuration'
 require 'flapjack/patches'
-require 'flapjack/executive'
 require 'flapjack/redis_pool'
 
 require 'flapjack/logger'
 require 'flapjack/pikelet'
-require 'flapjack/executive'
 
 module Flapjack
 
@@ -166,13 +164,32 @@ def remove_pikelets(piks, opts = {})
     end
 
     def pikelets(config_env)
-      return {} unless config_env
-      exec_cfg = config_env.has_key?('executive') && config_env['executive']['enabled'] ?
-        {'executive' => config_env['executive']} :
-        {}
-      return exec_cfg unless config_env && config_env['gateways'] &&
+      config = {}
+      return config unless config_env
+
+      # backwards-compatible with config file for previous 'executive' pikelet
+      exec_cfg = nil
+      if config_env.has_key?('executive') && config_env['executive']['enabled']
+        exec_cfg = config_env['executive']
+      end
+      ['processor', 'notifier'].each do |k|
+        if exec_cfg
+          if config_env.has_key?(k)
+            # need to allow for new config fields to override old settings if both present
+            merged = exec_cfg.merge(config_env[k])
+            config.update(k => merged) if merged['enabled']
+          else
+            config.update(k => exec_cfg)
+          end
+        else
+          next unless (config_env.has_key?(k) && config_env[k]['enabled'])
+          config.update(k => config_env[k])
+        end
+      end
+
+      return config unless config_env && config_env['gateways'] &&
         !config_env['gateways'].nil?
-      exec_cfg.merge( config_env['gateways'].select {|k, v|
+      config.merge( config_env['gateways'].select {|k, v|
         Flapjack::Pikelet.is_pikelet?(k) && v['enabled']
       } )
     end

lib/flapjack/data/contact.rb

@@ -235,10 +235,11 @@ def remove_media(media)
       end
 
       # drop notifications for
-      def drop_notifications?(opts)
+      def drop_notifications?(opts = {})
         media    = opts[:media]
         check    = opts[:check]
         state    = opts[:state]
+
         # build it and they will come
         @redis.exists("drop_alerts_for_contact:#{self.id}") ||
           (media && @redis.exists("drop_alerts_for_contact:#{self.id}:#{media}")) ||
@@ -248,7 +249,7 @@ def drop_notifications?(opts)
             @redis.exists("drop_alerts_for_contact:#{self.id}:#{media}:#{check}:#{state}"))
       end
 
-      def update_sent_alert_keys(opts)
+      def update_sent_alert_keys(opts = {})
         media  = opts[:media]
         check  = opts[:check]
         state  = opts[:state]

lib/flapjack/data/event.rb

@@ -6,7 +6,7 @@ module Flapjack
   module Data
     class Event
 
-      attr_accessor :previous_state, :previous_state_duration
+      attr_accessor :counter, :previous_state, :previous_state_duration
 
       attr_reader :check, :summary, :details, :acknowledgement_id
 
@@ -20,7 +20,7 @@ class Event
       # Calling next with :block => false, will return a nil if there are no
       # events on the queue.
       #
-      def self.next(opts={})
+      def self.next(queue, opts = {})
         raise "Redis connection not set" unless redis = opts[:redis]
 
         defaults = { :block => true,
@@ -31,23 +31,23 @@ def self.next(opts={})
         if options[:archive_events]
           dest = "events_archive:#{Time.now.utc.strftime "%Y%m%d%H"}"
           if options[:block]
-            raw = redis.brpoplpush('events', dest, 0)
+            raw = redis.brpoplpush(queue, dest, 0)
           else
-            raw = redis.rpoplpush('events', dest)
+            raw = redis.rpoplpush(queue, dest)
             return unless raw
           end
           redis.expire(dest, options[:events_archive_maxage])
         else
           if options[:block]
-            raw = redis.brpop('events', 0)[1]
+            raw = redis.brpop(queue, 0)[1]
           else
-            raw = redis.rpop('events')
+            raw = redis.rpop(queue)
             return unless raw
           end
         end
         begin
           parsed = ::Oj.load( raw )
-        rescue => e
+        rescue Oj::Error => e
           if options[:logger]
             options[:logger].warn("Error deserialising event json: #{e}, raw json: #{raw.inspect}")
           end

lib/flapjack/data/message.rb

@@ -5,7 +5,6 @@
 # contact+media recipient.
 
 require 'flapjack/data/contact'
-require 'flapjack/data/notification'
 
 module Flapjack
   module Data
@@ -15,8 +14,8 @@ class Message
 
       def self.for_contact(contact, opts = {})
         self.new(:contact => contact,
-                 :notification_contents => opts[:notification_contents],
-                 :medium => opts[:medium], :address => opts[:address],
+                 :medium => opts[:medium],
+                 :address => opts[:address],
                  :duration => opts[:duration])
       end
 
@@ -36,15 +35,13 @@ def contents
              'contact_first_name' => contact.first_name,
              'contact_last_name'  => contact.last_name}
         c['duration'] = duration if duration
-        return c if @notification_contents.nil?
-        c.merge(@notification_contents)
+        c
       end
 
     private
 
       def initialize(opts = {})
         @contact = opts[:contact]
-        @notification_contents = opts[:notification_contents]
         @medium = opts[:medium]
         @address = opts[:address]
         @duration = opts[:duration]