From ee995157786dd59632c73596a8a2cd8953b399de Mon Sep 17 00:00:00 2001 From: Chris Heald Date: Tue, 24 Mar 2009 02:31:08 -0700 Subject: [PATCH] Initial commit. See readme for info. --- LICENSE | 19 +++++ README | 19 +++++ app/metal/scrap.rb | 184 +++++++++++++++++++++++++++++++++++++++++++++ init.rb | 5 ++ sample.html | 12 +++ scrap.yml.example | 9 +++ 6 files changed, 248 insertions(+) create mode 100755 LICENSE create mode 100755 README create mode 100755 app/metal/scrap.rb create mode 100755 init.rb create mode 100755 sample.html create mode 100755 scrap.yml.example diff --git a/LICENSE b/LICENSE new file mode 100755 index 0000000..dc589b7 --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2008 Chris Heald + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/README b/README new file mode 100755 index 0000000..3b9fd5e --- /dev/null +++ b/README @@ -0,0 +1,19 @@ +Scrap is a Rails Metal endpoint designed to expose various garbage and memory-related metrics about your app. It may be particularly useful in tracking down memory leaks. + +To use it, simply install the plugin. This will provide a new url, /stats/scrap, which will report a number of metrics about your app. + +h2. Config + +If present, Scrap will use a config/scrap.yml file. See the provided example file for a list of the configuration options accepted. + +* max requests: How many requests to keep a record of. Older requests will be pushed out of the queue when the limit has been reached. Default is 150. +* max_objects: How many objects/deltas to show. Default is 50. +* classes: A hash of class names to do object counting on. Values may be "true" which prints the object count with a default set of options, or it may be a hash consisting of the following: +** print_objects: boolean - toggles the output of a representation of each instance of the type. +** show_fields: array - list of fields to show per instance. This actually invokes the "attributes" method of the object, so it's really only useful for ActiveRecord objects. +** small: boolean - if false, will not print counts in h3 tags. Default is true. +** min: integer - minimum count, if set, that an object must have to appear in the delta or top objects list. Default is nil. + +h2. Other considerations + +Scrap will take advantage of many of the wonderful metrics provided by recent versions of Ruby Enterprise Edition. It will work with other versions of Ruby, but you'll get a lot more info out of it if you're running REE 1.8.6-20090201 or later. \ No newline at end of file diff --git a/app/metal/scrap.rb b/app/metal/scrap.rb new file mode 100755 index 0000000..f130f53 --- /dev/null +++ b/app/metal/scrap.rb @@ -0,0 +1,184 @@ +class Scrap < Rails::Rack::Metal + PATH = "/stats/scrap".freeze + COMMIFY_REGEX = /(\d)(?=(\d\d\d)+(?!\d))/ + + @@gc_stats = {} + @@last_gc_run = nil + @@last_gc_mem = nil + @@requests_processed = 0 + @@request_list = [] + @@alive_at = nil + @@gc_stats_enabled = nil + @@config = nil + + def self.config + @@config ||= YAML::load open(File.join(Rails.root, "config", "scrap.yml")).read + rescue Errno::ENOENT + @@config = {} + rescue + puts "[scrap] scrap.yml: #{$!.message}" + @@config = {} + end + + def self.call(env) + if !@@gc_stats_enabled then + GC.enable_stats if GC.respond_to? :enable_stats + @@gc_stats_enabled = true + end + @@requests_processed += 1 + @@last_gc_run ||= @@alive_at ||= Time.now.to_f + @@last_gc_mem ||= get_usage + + req = sprintf("[%-10.2fMB] %s %s", get_usage, env["REQUEST_METHOD"], env["PATH_INFO"]) + req << "
#{ObjectSpace.statistics}
" if ObjectSpace.respond_to? :statistics + @@request_list.unshift req + @@request_list.pop if @@request_list.length > (config["max_requests"] || 150) + + if env["PATH_INFO"] == PATH + gc_stats + else + NotFoundResponse + end + end + + def self.gc_stats + collected = nil + puts "Respond to? #{ObjectSpace.respond_to? :live_objects}" + if ObjectSpace.respond_to? :live_objects then + live = ObjectSpace.live_objects + GC.start + collected = live - ObjectSpace.live_objects + else + GC.start + end + GC.start + usage = get_usage + + mem_delta = usage - @@last_gc_mem + time_delta = Time.now.to_f - @@last_gc_run + + s = "[#{$$}] Garbage Report" + s << "" + + s << "

Scrap - PID #{$$}

" + + s << "" + s << sprintf("", usage) + s << sprintf("", mem_delta) + s << sprintf("", time_delta) + s << sprintf("", @@requests_processed) + s << sprintf("", Time.now.to_f - @@alive_at) + if GC.respond_to? :time then + s << sprintf("", GC.time / 1000000.0) + end + if collected + s << sprintf("", collected) + s << sprintf("", ObjectSpace.live_objects) + end + s << "
Memory usage:%2.2fMB
Delta:%2.2fMB
Last Scrap req:%2.2f seconds ago
Requests processed:%s
Alive for:%2.2f seconds
Time spent in GC:%2.2f seconds
Collected objects:%2d
Live objects:%2d
" + + s << "

Top #{config["max_objects"]} deltas since last request

" + s << "" + memcheck(config["max_objects"], Object, :deltas).each do |v| + next if v.last == 0 + s << "" + end + s << "
#{v.first}#{sprintf("%s%s", v.last >= 0 ? "+" : "-", commify(v.last))}
" + + s << "

Top #{config["max_objects"]} objects

" + s << "" + memcheck(config["max_objects"]).each do |v| + s << "" + end + s << "
#{v.first}#{commify v.last}
" + + (config["classes"] || {}).each do |klass, val| + puts val.inspect + opts = val === true ? {"print_objects" => true} : val + add_os(klass.constantize, s, opts) + end + + s << "

Request history

" + @@request_list.each do |req| + s << req + s << "
" + end + + @@last_gc_run = Time.now.to_f + @@last_gc_mem = usage + @@requests_processed = 0 + [200, {"Content-Type" => "text/html"}, s] + end + + def self.get_usage + usage = 0 + begin + usage = `cat /proc/#{$$}/stat`.split(" ")[22].to_i / (1024 * 1024).to_f + rescue + # pass + end + return usage + end + + def self.add_os(c, s, options = {}) + print_objects = options["print_objects"] + small = options["small"] + min = options["min"] + show_fields = options["show_fields"] + + ct = ObjectSpace.each_object(c) {} + return if min and ct < min + + if small + s << "#{c} (#{ct})
" + else + s << "

#{c} (#{ct})

" + end + + return if !print_objects + s << "" + val = ObjectSpace.each_object(c) do |m| + s << "" + if show_fields then + show_fields.each do |field| + v = m.attributes[field.to_s] + if v.blank? + s << "" + else + s << "" + end + end + end + s << "" + end + s << "
" << "<#{m.class.to_s}:#{sprintf("0x%.8x", m.object_id)}> #{field}: #{v}
" + end + + def self.memcheck(top, klass = Object, mode = :normal) + top ||= 50 + os = Hash.new(0) + ObjectSpace.each_object(klass) do |o| + begin; + # If this is true, it's an association proxy, and we don't want to invoke method_missing on it, + # as it will result in a load of the association proxy from the DB, doing extra DB work and + # potentially creating a lot of AR objects. Hackalicious. + next if o.respond_to? :proxy_respond_to? + os[o.class.to_s] += 1 if o.respond_to? :class + rescue; end + end + if mode == :deltas then + os2 = Hash.new(0) + os.each do |k, v| + os2[k] = v - (@@gc_stats[k] || 0) + @@gc_stats[k] = v + end + sorted = os2.sort_by{|k,v| -v }.first(top) + else + sorted = os.sort_by{|k,v| -v }.first(top) + end + end + + def self.commify(i) + i.to_s.gsub(COMMIFY_REGEX, "\\1,") + end +end \ No newline at end of file diff --git a/init.rb b/init.rb new file mode 100755 index 0000000..bb87cc7 --- /dev/null +++ b/init.rb @@ -0,0 +1,5 @@ +begin + Rails::Rack::Metal::metal_paths << File.join(File.dirname(__FILE__), "app", "metal") +rescue NameError + puts "[WARNING] Scrap requires Rails 2.3 or better. Not booting." +end \ No newline at end of file diff --git a/sample.html b/sample.html new file mode 100755 index 0000000..d2efe69 --- /dev/null +++ b/sample.html @@ -0,0 +1,12 @@ + +[9752] Garbage Report

Scrap - PID 9752

Memory usage:59.01MB
Delta:0.00MB
Last Scrap req:0.39 seconds ago
Requests processed:1
Alive for:0.39 seconds
Time spent in GC:0.18 seconds
Collected objects:309826
Live objects:696773

Top deltas since last request

String+153,220
Array+5,690
Proc+3,268
Hash+3,090
Regexp+1,978
Class+1,362
ActionController::Routing::DividerSegment+946
ActiveSupport::Callbacks::Callback+754
ActionController::Routing::StaticSegment+604
Module+563
Gem::Version+425
Gem::Requirement+389
ActionController::Routing::Route+305
ActionController::Routing::DynamicSegment+258
ActionController::Routing::OptionalFormatSegment+223
ActiveSupport::Callbacks::CallbackChain+218
Range+190
ActiveRecord::Reflection::AssociationReflection+142
Time+132
Gem::Specification+128
Gem::Dependency+120
Float+103
UnboundMethod+98
Magick::CompositeOperator+56
Set+34
Magick::PreviewType+30
HashWithIndifferentAccess+24
Magick::ColorspaceType+23
Magick::FilterTypes+22
Mime::Type+21
ActiveRecord::Reflection::ThroughReflection+19
Magick::ChannelType+18
Magick::ImageLayerMethod+16
Rails::Plugin+15
Rails::OrderedOptions+14
Magick::VirtualPixelMethod+13
Magick::GravityType+12
Magick::QuantumExpressionOperator+12
ActionController::MiddlewareStack::Middleware+12
Magick::ImageType+12
Mutex+12
Rails::GemDependency+11
Rails::GemPlugin+11
Magick::CompressionType+11
Magick::StretchType+10
Bignum+9
Magick::OrientationType+9
ThinkingSphinx::Index::FauxColumn+9
Magick::InterlaceType+8
Magick::DistortImageMethod+8

Top objects

String111,167
Array5,860
Proc3,268
Hash3,091
Regexp1,978
Class1,362
ActionController::Routing::DividerSegment946
ActiveSupport::Callbacks::Callback754
ActionController::Routing::StaticSegment604
Module563
Gem::Version425
Gem::Requirement389
ActionController::Routing::Route305
ActionController::Routing::DynamicSegment258
ActionController::Routing::OptionalFormatSegment223
ActiveSupport::Callbacks::CallbackChain218
Range190
ActiveRecord::Reflection::AssociationReflection142
Time130
Gem::Specification128
Gem::Dependency120
UnboundMethod98
Float94
Magick::CompositeOperator56
Set34
Magick::PreviewType30
HashWithIndifferentAccess24
Magick::ColorspaceType23
Magick::FilterTypes22
Mime::Type21
ActiveRecord::Reflection::ThroughReflection19
Magick::ChannelType18
Magick::ImageLayerMethod16
Rails::Plugin15
Rails::OrderedOptions14
Magick::VirtualPixelMethod13
Magick::QuantumExpressionOperator12
Mutex12
ActionController::MiddlewareStack::Middleware12
Magick::ImageType12
Magick::GravityType12
Rails::GemDependency11
Magick::CompressionType11
Rails::GemPlugin11
Magick::StretchType10
Magick::OrientationType9
Bignum9
ThinkingSphinx::Index::FauxColumn9
Magick::MetricType8
Magick::StorageType8

Request history

[59.01 MB] GET /stats/scrap
Number of objects    : 824154 (612748 AST nodes, 74.35%)
+Heap slot size       : 20
+GC cycles so far     : 26
+Number of heaps      : 7
+Total size of objects: 16096.76 KB
+Total size of heaps  : 18036.66 KB (1939.91 KB = 10.76% unused)
+Leading free slots   : 13649 (266.58 KB = 1.48%)
+Trailing free slots  : 0 (0.00 KB = 0.00%)
+Number of contiguous groups of 16 slots: 2113 (3.66%)
+Number of terminal objects: 3890 (0.42%)
+

\ No newline at end of file diff --git a/scrap.yml.example b/scrap.yml.example new file mode 100755 index 0000000..06c8eac --- /dev/null +++ b/scrap.yml.example @@ -0,0 +1,9 @@ +max_requests: 150 +max_objects: 10 +classes: + "ActionController::Base": true + "ActionController::Session::AbstractStore::SessionHash": true + "ActiveRecord::Base": + print_objects: true + foo: bar + show_fields: [id, updated_at] \ No newline at end of file