Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Performance fixes for the exception show page

  • Loading branch information...
commit 2c1c7e7fd0229fa622ccf3f020b39dbac25c4dcc 1 parent 69d21f4
@justinweiss authored
View
8 lib/exceptionl/exception_group.rb
@@ -10,8 +10,14 @@ class Exceptionl::ExceptionGroup
# generate the same digest.
attr_accessor :digest
+ # The list of machines that have seen this exception
+ attr_accessor :machines
+
+ # The first time this exception occurred
+ attr_accessor :first_timestamp
+
# The most recent time this exception occurred
- attr_accessor :timestamp
+ attr_accessor :most_recent_timestamp
# The most recent ExceptionReport instance belonging to this group
attr_accessor :most_recent_report
View
2  lib/exceptionl/server.rb
@@ -110,7 +110,7 @@ def initialize
end
get '/similar/:digest.html' do
- @group = store.group(params[:digest]).paginate(:page => params[:page], :per_page => PER_PAGE)
+ @group = store.reports_in_group(params[:digest]).paginate(:page => params[:page], :per_page => PER_PAGE)
if @group
erb :similar
else
View
6 lib/exceptionl/server/views/show.erb
@@ -4,9 +4,9 @@
<dl class="group_details">
<dt>Count</dt><dd><%= h(@group.count) %></dd>
- <dt>First occurred</dt><dd><%= h(@group.last.timestamp) %></dd>
- <dt>Most recently occurred</dt><dd><%= h(@group.first.timestamp) %></dd>
- <dt>On machines</dt><dd><%= h(@group.map(&:machine).uniq.join(', ')) %></dd>
+ <dt>First occurred</dt><dd><%= h(@group.first_timestamp) %></dd>
+ <dt>Most recently occurred</dt><dd><%= h(@group.most_recent_timestamp) %></dd>
+ <dt>On machines</dt><dd><%= h(@group.machines.join(', ')) %></dd>
</dl>
<p><a href="/similar/<%= @report.digest%>.html">See all</a></p>
<% plugins.each do |plugin| %>
View
7 lib/exceptionl/store/base.rb
@@ -28,11 +28,16 @@ def find(id)
raise NotImplementedError, "Must be implemented by child class"
end
- # Returns a list of exceptions whose digest is +digest+.
+ # Returns the ExceptionGroup matching +digest+
def group(digest)
raise NotImplementedError, "Must be implemented by child class"
end
+ # Returns a list of exceptions whose digest is +digest+.
+ def reports_in_group(digest)
+ raise NotImplementedError, "Must be implemented by child class"
+ end
+
# Searches for exception reports maching +params+. Search should
# support searching by application name, machine name, exception
# name, and exception type. The keys in +params+ should match
View
30 lib/exceptionl/store/in_memory.rb
@@ -29,12 +29,16 @@ def store(exception_report)
exception_report.id
end
- # returns the list of exceptions in the group matching +digest+, ordered by
- # timestamp.
- def group(digest)
+ # Returns a list of exceptions whose digest is +digest+.
+ def reports_in_group(digest)
exception_groups[digest]
end
+ # returns the exception group matching +digest+
+ def group(digest)
+ build_group_for_exceptions(reports_in_group(digest))
+ end
+
# Empty this exception store. Useful for tests!
def clear
@exceptions = []
@@ -55,14 +59,22 @@ def empty?
def recent
data = []
exception_groups.map do |digest, group|
- data << Exceptionl::ExceptionGroup.new.tap do |g|
- g.count = group.length
- g.digest = digest
- g.timestamp = group.last.timestamp
- g.most_recent_report = group.last
- end
+ data << build_group_for_exceptions(group)
end
data.reverse
end
+
+ protected
+
+ def build_group_for_exceptions(group)
+ Exceptionl::ExceptionGroup.new.tap do |g|
+ g.count = group.length
+ g.digest = group.first.digest
+ g.machines = group.map(&:machine).uniq
+ g.first_timestamp = group.first.timestamp
+ g.most_recent_timestamp = group.last.timestamp
+ g.most_recent_report = group.last
+ end
+ end
end
View
112 lib/exceptionl/store/mongoid.rb
@@ -21,37 +21,19 @@ def initialize(config_file)
Mongoid.configure do |config|
config.from_hash(settings[ENV['RACK_ENV']])
end
+ Thread.new { migrate_data }
end
# Store +exception_report+ in the database.
def store(exception_report)
report = ExceptionReport.create_from_exception_report(exception_report)
-
- ExceptionGroup.collection.update(
- {:digest => report.digest},
- {
- '$inc' => {:count => 1},
- '$set' => {:most_recent_report_id => report.id, :timestamp => report.timestamp},
- },
- :upsert => true)
-
- # Update indexes to pre-populate the search dropdowns.
- Machine.collection.update(
- {:name => report.machine},
- {:name => report.machine},
- :upsert => true)
-
- Application.collection.update(
- {:name => report.application},
- {:name => report.application},
- :upsert => true)
-
+ update_caches(report)
report.id
end
# Have we logged any exceptions?
def empty?
- ExceptionGroup.where(:timestamp.gt => 7.days.ago).count == 0
+ ExceptionGroup.where(:most_recent_timestamp.gt => 7.days.ago).count == 0
end
# Return the last 7 days worth of unique exception reports grouped
@@ -64,7 +46,7 @@ def recent
# an array here because we don't want to load all the mongid
# models in memory. This is also made trickier because of my
# hacked-up :include stuff I built into ExceptionGroup.
- ExceptionGroup::PaginationHelper.new(ExceptionGroup.where(:timestamp.gt => 7.days.ago).order_by(:timestamp.desc))
+ ExceptionGroup::PaginationHelper.new(ExceptionGroup.where(:most_recent_timestamp.gt => 7.days.ago).order_by(:most_recent_timestamp.desc))
end
# Find an exception report with the given id.
@@ -74,19 +56,25 @@ def find(id)
# All applications that have been seen by this store
def applications
- Application.all.order_by(:name).map(&:name)
+ Application.all.order_by(:name.asc).map(&:name)
end
# All machines that have been seen by this store
def machines
- Machine.all.order_by(:name).map(&:name)
+ Machine.all.order_by(:name.asc).map(&:name)
end
- # returns the group this exception is a part of, ordered by
- # timestamp
- def group(digest)
+ # Returns all the exceptions in a group, ordered by
+ # most_recent_timestamp
+ def reports_in_group(digest)
ExceptionReport.where(:digest => digest).order_by(:timestamp.desc)
end
+
+ # returns the ExceptionGroup object corresponding to a particular
+ # digest
+ def group(digest)
+ ExceptionGroup.where(:digest => digest).first
+ end
# Searches for exception reports maching +params+. Supports querying
# by arbitrary data in the +data+ hash associated with the exception, with the format:
@@ -114,7 +102,7 @@ def search(params)
end
end
- scope.order_by(:timestamp.desc)
+ scope.order_by(:most_recent_timestamp.desc)
end
# Creates the MongoDB indexes used by this driver. Should be called
@@ -124,6 +112,68 @@ def create_indexes
Exceptionl::Store::Mongoid::ExceptionReport.create_indexes
Exceptionl::Store::Mongoid::ExceptionGroup.create_indexes
end
+
+ # Migrate the data in the mongoid database to a newer format. This
+ # should eventually be more robust, like the Rails version, but for
+ # now this should be fine.
+ def migrate_data
+ if SchemaMigrations.where(:version => 1).empty?
+ ExceptionGroup.all.order_by(:timestamp).desc.each do |group|
+ exceptions = ExceptionReport.where(:digest => group.digest).order_by(:timestamp)
+ unless exceptions.empty?
+ exceptions = exceptions.to_a
+ group.attributes[:timestamp] = nil
+ group.first_timestamp = exceptions[0].timestamp
+ group.most_recent_timestamp = exceptions[-1].timestamp
+ group.machines = exceptions.map(&:machine).uniq
+ group.save
+ end
+ end
+ SchemaMigrations.create(:version => 1)
+ end
+ end
+
+ protected
+
+ # In order to make Exceptionl super-fast, we keep a bunch of cached
+ # data (like exception report groups, machines, and
+ # applications). +update_caches+ updates all of this cached data
+ # when an exception report comes in.
+ def update_caches(report)
+ ExceptionGroup.collection.update(
+ {:digest => report.digest},
+ {
+ '$inc' => {:count => 1},
+ '$set' => {:most_recent_report_id => report.id, :most_recent_timestamp => report.timestamp},
+ '$addToSet' => {:machines => report.machine}
+ },
+ :upsert => true)
+
+ # Make sure the first_timestamp parameter is set. Unfortunately
+ # mongoid doesn't have an $add modifier yet, so we have to do another query.
+ ExceptionGroup.collection.update(
+ {:digest => report.digest, :first_timestamp => nil},
+ {:first_timestamp => report.timestamp})
+
+ # Update indexes to pre-populate the search dropdowns.
+ Machine.collection.update(
+ {:name => report.machine},
+ {:name => report.machine},
+ :upsert => true)
+
+ Application.collection.update(
+ {:name => report.application},
+ {:name => report.application},
+ :upsert => true)
+
+ report.id
+ end
+end
+
+# Keeps track of the migrations we've run so far.
+class Exceptionl::Store::Mongoid::SchemaMigrations
+ include Mongoid::Document
+ field :version
end
# A cache of all the applications that have had exception reports seen
@@ -149,10 +199,12 @@ class Exceptionl::Store::Mongoid::ExceptionGroup < Exceptionl::ExceptionGroup
include Mongoid::Document
field :count, :type => Integer
field :digest
- field :timestamp, :type => Time
+ field :machines, :type => Array
+ field :first_timestamp, :type => Time
+ field :most_recent_timestamp, :type => Time
field :most_recent_report_id, :type => Integer
index :digest
- index :timestamp
+ index :most_recent_timestamp
# Cache most recent report so we can preload a bunch at once
attr_accessor :most_recent_report
View
6 test/unit/stores/in_memory_test.rb
@@ -18,8 +18,10 @@ def test_group
store_exception(@store, "test2")
assert_equal 2, @store.recent.length
- assert_equal 1, @store.group(@store.find(2).digest).length
- assert_equal 2, @store.group(@store.find(0).digest).length
+ assert_equal 1, @store.reports_in_group(@store.find(2).digest).length
+ assert_equal 1, @store.group(@store.find(2).digest).count
+ assert_equal 2, @store.reports_in_group(@store.find(0).digest).length
+ assert_equal 2, @store.group(@store.find(0).digest).count
end
def store_exception(store, message = "failed", count = 1, data = {})
Please sign in to comment.
Something went wrong with that request. Please try again.