Allow registering events to timelines using regexes

Up until now, in order to register event type to the timeline, we
needed to list all of the event types in settings yaml.

This commit adds another option: specify event type to group/level
mapping using regular expressions, which should make mapping
definitions much shorter for providers with many different event types
that follow a certain pattern.

Changes were made in to areas: in database querying and in event to
category assignment.

When creating database condition for events, we now use three sources:

 1. a set of event types that should be included in result,
 2. a set of event types that should not be present in result and
 3. a set of regular expressions that event types can match.

An event is part of the result if its type is in include set or
matches any of the regular expressions and is not part of the exclude
set. For example, take this fragment of settings file:

    :ems:
      :ems_dummy:
        :event_handling:
          :event_groups:
            :addition:
              :critical_regex:
              - ^dummy_add_.+$
            :update:
              :critical:
              - dummy_123_update
              :critical_regex:
              - ^dummy_.+_update$
              :detail:
              - dummy_abc_update
              - dummy_def_update
              :detail_regex:
              - ^dummy_detail_.+_update$

This translates into following three sources for :update category,
assuming user does not want to display details:

  * include_set: dummy_123_update
  * exclude_set: dummy_abc_update, dummy_def_update
  * regexes: ^dummy_.+_updates$

In this case, events of type dummy_abc_update will not be displayed,
since they match details and are thus put in exclude set. On the
other hand, if user does want to see the details, things are a bit
different:

  * include_set: dummy_123_update, dummy_abc_update, dummy_def_update
  * exclude_set:
  * regexes: ^dummy_.+_updates$, ^dummy_detail_.+_update$

All this is done in order to preserve the existing behavior.

Category assignment is done in two phases. First, we try to assign
category only looking at the full name of event. If this yields no
category, we try to assign one by matching against regular
expressions. If we continue with categories from example above, we
would get those mappings:

  * dummy_123_update is mapped to :update. It also matches :addition
    category, but since explicit names have higher priority, that
    regular expression is not checked at all.
  * dummy_nil_update is mapped to :update because it matches one of
    the :critical_regex expressions.
  * dummy_add_event is mapped to :addition.

app/controllers/application_controller/timelines.rb

@@ -105,42 +105,50 @@ def tl_build_timeline_report_options
       end
 
       temp_clause = @tl_record.event_where_clause(@tl_options.evt_type)
+      conditions = [temp_clause[0], "timestamp >= ?", "timestamp <= ?"]
+      parameters = temp_clause[1..-1] + [from_dt, to_dt]
 
-      cond = "( "
-      cond = cond << temp_clause[0]
-      params = temp_clause.slice(1, temp_clause.length)
-
-      event_set = @tl_options.event_set
-      if !event_set.empty?
-        if @tl_options.policy_events? && @tl_options.policy.result != "both"
-          where_clause = [") and (timestamp >= ? and timestamp <= ?) and (event_type in (?)) and (result = ?)",
-                          from_dt,
-                          to_dt,
-                          event_set.flatten,
-                          @tl_options.policy.result]
-        else
-          where_clause = [") and (timestamp >= ? and timestamp <= ?) and (event_type in (?))",
-                          from_dt,
-                          to_dt,
-                          event_set.flatten]
-        end
-      else
-        where_clause = [") and (timestamp >= ? and timestamp <= ?)",
-                        from_dt,
-                        to_dt]
-      end
-      cond << where_clause[0]
+      tl_add_event_type_conditions(conditions, parameters)
+      tl_add_policy_conditions(conditions, parameters) if @tl_options.policy_events?
 
-      params2 = where_clause.slice(1, where_clause.length - 1)
-      params = params.concat(params2)
-      @report.where_clause = [cond, *params]
+      condition = "(" + conditions.reduce { |a, c| a + ") and (" + c } + ")"
+      @report.where_clause = [condition] + parameters
       @report.rpt_options ||= {}
-      @report.rpt_options[:categories] =
-        @tl_options.management_events? ? @tl_options.management.categories : @tl_options.policy.categories
+      @report.rpt_options[:categories] = @tl_options.categories
       @title = @report.title
     end
   end
 
+  def tl_add_event_type_conditions(conditions, parameters)
+    or_subexpressions = []
+    @tl_options.get_set(:regexes).each do |regex|
+      or_subexpressions << "event_type ~ ?"
+      parameters << regex.source
+    end
+
+    includes = @tl_options.get_set(:include_set)
+    unless includes.empty?
+      or_subexpressions << "event_type in (?)"
+      parameters << includes
+    end
+
+    or_condition = or_subexpressions.reduce { |a, e| a + ") or (" + e }
+    conditions << "(" + or_condition + ")" unless or_condition.nil?
+
+    excludes = @tl_options.get_set(:exclude_set)
+    unless excludes.empty?
+      conditions << "event_type not in (?)"
+      parameters << excludes
+    end
+  end
+
+  def tl_add_policy_conditions(conditions, parameters)
+    if @tl_options.policy.result != "both"
+      conditions << "result = ?"
+      parameters << @tl_options.policy.result
+    end
+  end
+
   def tl_build_timeline(refresh = nil)
     tl_build_init_options(refresh)                # Intialize options(refresh) if !@report
     @ajax_action = "tl_chooser"

app/controllers/application_controller/timelines/options.rb

@@ -36,13 +36,25 @@ def update_start_end(sdate, edate)
     def update_from_params(params)
       self.level = params[:tl_fl_typ] == "critical" ? :critical : :detail
       self.categories = {}
-      if params[:tl_categories]
-        params[:tl_categories].each do |category|
-          categories[events[category]] = {:display_name => category}
-          categories[events[category]][:event_groups] = level == :critical ?
-            event_groups[events[category]][level] :
-            event_groups[events[category]][level] + event_groups[events[category]][:critical]
+      params.fetch(:tl_categories, []).each do |category_display_name|
+        category_data = event_groups[events[category_display_name]]
+        category = {
+          :display_name => category_display_name,
+          :include_set  => category_data[:critical] || [],
+          :exclude_set  => [],
+          :regexes      => (category_data[:critical_regex] || []).map do |r|
+            Regexp.new(r)
+          end
+        }
+        if level == :detail
+          category[:include_set] += category_data[:detail] || []
+          category[:regexes] += (category_data[:detail_regex] || []).map do |r|
+            Regexp.new(r)
+          end
+        else
+          category[:exclude_set] += category_data[:detail] || []
         end
+        categories[events[category_display_name]] = category
       end
     end
 
@@ -53,14 +65,6 @@ def events
       end
     end
 
-    def event_set
-      event_set = []
-      categories.each do |category|
-        event_set.push(category.last[:event_groups])
-      end
-      event_set
-    end
-
     def drop_cache
       @events = @event_groups = nil
     end
@@ -77,10 +81,13 @@ def event_groups
     def update_from_params(params)
       self.result = params[:tl_result] || "success"
       self.categories = {}
-      if params[:tl_categories]
-        params[:tl_categories].each do |category|
-          categories[category] = {:display_name => category, :event_groups => events[category]}
-        end
+      params.fetch(:tl_categories, []).each do |category|
+        categories[category] = {
+          :display_name => category,
+          :include_set  => events[category],
+          :exclude_set  => [],
+          :regexes      => []
+        }
       end
     end
 
@@ -93,11 +100,6 @@ def events
     def drop_cache
       @events = @fltr_cache = nil
     end
-
-    def event_set
-      categories.blank? ? [] : categories.collect { |_, e| e[:event_groups] }
-    end
-
   end
 
   Options = Struct.new(
@@ -126,8 +128,12 @@ def evt_type
       management_events? ? :event_streams : :policy_events
     end
 
-    def event_set
-      (policy_events? ? policy : management).event_set
+    def categories
+      (policy_events? ? policy : management).categories
+    end
+
+    def get_set(name)
+      categories.values.reduce([]) { |a, v| a.push(*v[name]) }
     end
 
     def drop_cache

lib/report_formatter/timeline.rb

@@ -34,14 +34,23 @@ def build_document_body
       # some of the OOTB reports have db as EventStream or PolicyEvent,
       # those do not have event categories, so need to go thru else block for such reports.
       if (mri.db == "EventStream" || mri.db == "PolicyEvent") && mri.rpt_options.try(:[], :categories)
-        mri.rpt_options[:categories].each do |_, options|
+        event_map = mri.table.data.each_with_object({}) do |event, buckets|
+          bucket_name = mri.rpt_options[:categories].find do |_, options|
+            options[:include_set].include?(event.event_type)
+          end&.last.try(:[], :display_name)
+
+          bucket_name ||= mri.rpt_options[:categories].find do |_, options|
+            options[:regexes].find { |regex| regex.match(event.event_type) }
+          end.last[:display_name]
+
+          buckets[bucket_name] ||= []
+          buckets[bucket_name] << event
+        end
+
+        event_map.each do |name, events|
           @events_data = []
-          all_events = mri.table.data.select { |e| options[:event_groups].include?(e.event_type) }
-          all_events.each do |row|
-            tl_event(row, col) # Add this row to the tl event xml
-          end
-          @events.push(:name => options[:display_name],
-                       :data => [@events_data])
+          events.each { |row| tl_event(row, col) }
+          @events.push(:name => name, :data => [@events_data])
         end
       else
         mri.table.data.each_with_index do |row, _d_idx|

spec/controllers/application_controller/timelines_spec.rb

@@ -47,8 +47,8 @@
         expect(controller).to receive(:render)
         controller.send(:tl_chooser)
         options = assigns(:tl_options)
-        expect(options.management[:categories][:power][:event_groups]).to include('AUTO_FAILED_SUSPEND_VM')
-        expect(options.management[:categories][:power][:event_groups]).to_not include('PowerOffVM_Task')
+        expect(options.management[:categories][:power][:include_set]).to include('AUTO_FAILED_SUSPEND_VM')
+        expect(options.management[:categories][:power][:include_set]).to_not include('PowerOffVM_Task')
       end
 
       it "checking Detailed events checkbox of Timelines options should append them to list of events" do
@@ -60,8 +60,8 @@
         expect(controller).to receive(:render)
         controller.send(:tl_chooser)
         options = assigns(:tl_options)
-        expect(options.management[:categories][:power][:event_groups]).to include('PowerOffVM_Task')
-        expect(options.management[:categories][:power][:event_groups]).to include('AUTO_FAILED_SUSPEND_VM')
+        expect(options.management[:categories][:power][:include_set]).to include('PowerOffVM_Task')
+        expect(options.management[:categories][:power][:include_set]).to include('AUTO_FAILED_SUSPEND_VM')
       end
     end
   end

spec/lib/report_formater/timeline_spec.rb

@@ -126,9 +126,11 @@ def stub_ems_event(event_type)
                                    :headers   => %w(id name event_type timestamp),
                                    :timeline  => {:field => "EmsEvent-timestamp", :position => "Last"})
       @report.rpt_options = {:categories => {:power    => {:display_name => "Power Activity",
-                                                           :event_groups => %w(VmPoweredOffEvent VmPoweredOnEvent)},
+                                                           :include_set  => %w(VmPoweredOffEvent VmPoweredOnEvent),
+                                                           :regexes      => []},
                                              :snapshot => {:display_name => "Snapshot Activity",
-                                                           :event_groups => %w(AlarmCreatedEvent AlarmRemovedEvent)}}}
+                                                           :include_set  => %w(AlarmCreatedEvent AlarmRemovedEvent),
+                                                           :regexes      => []}}}
 
       data = []
       30.times do
@@ -172,6 +174,87 @@ def stub_ems_event(event_type)
       expect(JSON.parse(events)[0]["data"][0].length).to eq(45)
     end
   end
+
+  describe '#events count for regex categories' do
+    def stub_ems_event(event_type)
+      ems = FactoryGirl.create(:ems_redhat)
+      EventStream.create!(:event_type => event_type, :ems_id => ems.id)
+    end
+
+    before do
+      @report = FactoryGirl.create(
+        :miq_report,
+        :db        => "EventStream",
+        :col_order => %w(id name event_type timestamp),
+        :headers   => %w(id name event_type timestamp),
+        :timeline  => {:field => "EmsEvent-timestamp", :position => "Last"}
+      )
+      @report.rpt_options = {
+        :categories => {
+          :power    => {
+            :display_name => "Power Activity",
+            :include_set  => [],
+            :regexes      => [/Event$/]
+          },
+          :snapshot => {
+            :display_name => "Snapshot Activity",
+            :include_set  => %w(AlarmCreatedEvent AlarmRemovedEvent),
+            :regexes      => []
+          }
+        }
+      }
+
+      data = []
+      (1..5).each do |n|
+        event_type = "VmPower#{n}Event"
+        data.push(
+          Ruport::Data::Record.new(
+            "id"         => stub_ems_event(event_type).id,
+            "name"       => "Baz",
+            "event_type" => event_type,
+            "timestamp"  => Time.zone.now
+          )
+        )
+      end
+
+      7.times do
+        data.push(
+          Ruport::Data::Record.new(
+            "id"         => stub_ems_event("AlarmRemovedEvent").id,
+            "name"       => "Baz",
+            "event_type" => "AlarmRemovedEvent",
+            "timestamp"  => Time.zone.now
+          )
+        )
+      end
+
+      @report.table = Ruport::Data::Table.new(
+        :column_names => %w(id name event_type timestamp),
+        :data         => data
+      )
+    end
+
+    it 'shows correct count of timeline events based on categories' do
+      allow_any_instance_of(Ruport::Controller::Options).to receive(:mri).and_return(@report)
+      events = ReportFormatter::ReportTimeline.new.build_document_body
+      expect(JSON.parse(events)[0]["data"][0].length).to eq(5)
+      expect(JSON.parse(events)[1]["data"][0].length).to eq(7)
+    end
+
+    it 'shows correct count of timeline events together for report object with no categories' do
+      @report.rpt_options = {}
+      allow_any_instance_of(Ruport::Controller::Options).to receive(:mri).and_return(@report)
+      events = ReportFormatter::ReportTimeline.new.build_document_body
+      expect(JSON.parse(events)[0]["data"][0].length).to eq(12)
+    end
+
+    it 'shows correct count of timeline events for timeline based report when rpt_options is nil' do
+      @report.rpt_options = nil
+      allow_any_instance_of(Ruport::Controller::Options).to receive(:mri).and_return(@report)
+      events = ReportFormatter::ReportTimeline.new.build_document_body
+      expect(JSON.parse(events)[0]["data"][0].length).to eq(12)
+    end
+  end
 end
 
 describe '#set data for headers that exist in col headers' do
@@ -188,9 +271,11 @@ def stub_ems_event(event_type)
                                  :headers   => %w(id name event_type timestamp vm_location),
                                  :timeline  => {:field => "EmsEvent-timestamp", :position => "Last"})
     @report.rpt_options = {:categories => {:power    => {:display_name => "Power Activity",
-                                                         :event_groups => %w(VmPoweredOffEvent VmPoweredOnEvent)},
+                                                         :include_set  => %w(VmPoweredOffEvent VmPoweredOnEvent),
+                                                         :regexes      => []},
                                            :snapshot => {:display_name => "Snapshot Activity",
-                                                         :event_groups => %w(AlarmCreatedEvent AlarmRemovedEvent)}}}
+                                                         :include_set  => %w(AlarmCreatedEvent AlarmRemovedEvent),
+                                                         :regexes      => []}}}
 
     data = [Ruport::Data::Record.new("id"          => stub_ems_event("VmPoweredOffEvent").id,
                                      "name"        => "Baz",