Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

synced with mainline

  • Loading branch information...
commit fcccd0648bb41f68ff7bcf77791f72f489045429 2 parents 14ded38 + 771fc34
Philip (flip) Kromer authored
View
29 .gitignore
@@ -1,21 +1,22 @@
-## MAC OS
-.DS_Store
-## TEXTMATE
-*.tmproj
-tmtags
-## EMACS
-*~
-\#*
-.\#*
-## VIM
-*.swp
+
+## EMACS
+## MAC OS
## PROJECT::GENERAL
+## PROJECT::SPECIFIC
+## TEXTMATE
+## VIM
+*.swp
+*.tmproj
+*~
+.DS_Store
+.\#*
+/TAGS
+\#*
coverage
-rdoc
pkg
-
-## PROJECT::SPECIFIC
+rdoc
+tmtags
View
1  Rakefile
@@ -10,6 +10,7 @@ begin
gem.email = "info@infochimps.org"
gem.homepage = "http://github.com/infochimps/graphiterb"
gem.authors = ["Philip (flip) Kromer (@mrflip)"]
+ gem.add_dependency "configliere", ">= 0"
gem.add_development_dependency "rspec", ">= 1.2.9"
gem.add_development_dependency "yard", ">= 0"
# gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
View
2  VERSION
@@ -1 +1 @@
-0.1.5
+0.2.13
View
51 examples/api_call_monitor.rb
@@ -1,51 +0,0 @@
-#!/usr/bin/env ruby
-$: << File.dirname(__FILE__)+'/../lib/'
-require 'graphiterb'
-require 'graphiterb/graphite_script'
-
-WC_EXEC = '/usr/bin/wc'
-
-class ApiCallMonitor < Graphiterb::GraphiteLogger
- API_CALLS_TO_MONITOR = %w[trstrank wordbag influence conversation]
- ERROR_CODES_TO_MONITOR = %w[4.. 5.. 200]
-
- def initialize *args
- super *args
- @current_total = Hash.new
- @prev_total = Hash.new
- end
-
- def calls api
- total_calls = `cat /var/www/apeyeye/shared/log/apeyeye-access.log | egrep 'GET /soc/net/tw/#{api}' | #{WC_EXEC} -l` rescue 0
- @current_total[api] = total_calls.to_i
- end
-
- def errors error_code
- log_cat = `cat /var/www/apeyeye/shared/log/apeyeye-access.log | egrep 'GET /soc/net/tw/.*HTTP/1\.[0-1]..#{error_code}' | #{WC_EXEC} -l` rescue 0
- @current_total[error_code] = log_cat.to_i
- end
-
- def rate item
- @prev_total[item] ||= @current_total[item]
- rate = @current_total[item].to_i - @prev_total[item].to_i
- @prev_total[item] = @current_total[item]
- [0, rate].max
- end
-
- def get_metrics metrics, iter, since
- API_CALLS_TO_MONITOR.each do |api|
- metrics << [scope_name(hostname, api, 'total_accesses'), calls(api)]
- metrics << [scope_name(hostname, api, 'accesses'), rate(api)]
- end
- ERROR_CODES_TO_MONITOR.each do |code|
- metrics << [scope_name(hostname, code.gsub('.','x'), 'total_errors'), errors(code)]
- metrics << [scope_name(hostname, code.gsub('.','x'), 'errors'), rate(code)]
- end
- end
-end
-
-
-warn "Update delay is #{Settings.update_delay} seconds. You probably want something larger: some of these checks are data-intensive" if Settings.update_delay < 60
-Settings.die "Update delay is #{Settings.update_delay} seconds. You need to radio in at least as often as /usr/local/share/graphite/conf/storage-schemas says -- this is typically 5 minutes." if Settings.update_delay >= 300
-
-ApiCallMonitor.new('apeyeye', :iters => nil, :time => Settings.update_delay).run!
View
113 examples/file_monitor.rb
@@ -1,113 +0,0 @@
-#!/usr/bin/env ruby
-$: << File.dirname(__FILE__)+'/../lib/'
-require 'graphiterb'
-Settings.define :work_dir, :description => "Base directory where scrapers store files. (Ex: /data/ripd/com.tw)", :required => true
-require 'graphiterb/graphite_script'
-
-#
-# Usage:
-#
-# nohup ~/ics/backend/graphiterb/bin/file_monitor.rb --work_dir=/data/ripd/com.tw --carbon_server=whatever --update_delay=120 > /data/log/file_monitor.log 2>&1 &
-#
-
-WC_EXEC = '/usr/bin/wc'
-
-class FilePool
- # Path to sample for files
- attr_accessor :path
- # wildcard sequence for files under the current directory
- attr_accessor :filter_re
- # A recent file was modified within this window
- attr_accessor :recent_window
- # Only consider the last this-many files
- MAX_FILES = 30
-
- def initialize path, filter_re=/.*/, options={}
- self.path = path
- self.filter_re = filter_re
- end
-
- # Name for this pool, suitable for inclusion in a metrics handle
- def name
- path.gsub(/\./,'_').gsub(%r{/}, '.').gsub(%r{(^\.|\.$)},'')
- end
-
- #
- # Lists all files in the pool
- # @param filter_block files only keeps filenames that pass this filter
- #
- def files &filter_block
- Dir[File.join(path, '**/*')].
- reject{|f| File.directory?(f) }.
- select{|f| f =~ filter_re }.
- sort.reverse[0..MAX_FILES].
- select(&filter_block)
- end
-
- def num_files &filter_block
- files(&filter_block).count
- end
-
- def sizes &filter_block
- files(&filter_block).map{|f| File.size(f) rescue nil }.compact
- end
- def size &filter_block
- sizes(&filter_block).sum
- end
- def avg_size &filter_block
- sizes(&filter_block).sum.to_f / num_files(&filter_block).to_f
- end
-
- def lines_in_result_of command, *args
- begin
- escaped_args = args.map{|f| "'#{f}'" }
- result = `#{command} #{escaped_args.join(" ")}`.chomp
- result.split(/[\r\n]+/)
- rescue StandardError => e ; warn(e.backtrace, e) ; return nil ; end
- end
-
- def line_counts &filter_block
- files = files(&filter_block) ; return 0 if files.blank?
- result = lines_in_result_of(WC_EXEC, '-l', *files) or return 0
- counts = result.map{|wc| wc =~ /^\s*(\d+)\s+/ and $1 }.compact
- counts.map(&:to_i).sum
- end
-
- def self.recent? file
- (Time.now - File.mtime(file)) < 3600
- end
- def self.recency_filter
- Proc.new{|file| recent?(file) }
- end
-end
-
-class FileMonitor < Graphiterb::GraphiteSystemLogger
- attr_accessor :path
- attr_accessor :pools
-
- def initialize *args
- super *args
- self.path = Settings.work_dir
- self.pools = {}
- populate_pools!
- end
-
- def populate_pools!
- Dir[File.join(path, '*')].select{|d| File.directory?(d) }.each do |dir|
- self.pools[dir] ||= FilePool.new(dir, %r{20\d*/.*\.(?:tsv|json|xml)})
- end
- end
-
- def get_metrics metrics, iter, since
- recent = FilePool.recency_filter
- pools.each do |pool_path, pool|
- metrics << [scope_name(pool.name, hostname, 'active_files'), pool.num_files(&recent) ]
- metrics << [scope_name(pool.name, hostname, 'active_file_size'), pool.size(&recent) ]
- metrics << [scope_name(pool.name, hostname, 'line_counts'), pool.line_counts(&recent) ]
- end
- end
-end
-
-warn "Update delay is #{Settings.update_delay} seconds. You probably want something larger: some of the metrics are expensive." if Settings.update_delay < 60
-warn "Update delay is #{Settings.update_delay} seconds. You probably want something smaller: need to report in faster than the value in the graphite/conf/storage-schemas." if Settings.update_delay >= 300
-FileMonitor.new('scraper', :iters => nil, :time => Settings.update_delay).run!
View
36 examples/loadavg_graphite_sender.rb
@@ -1,36 +0,0 @@
-#!/usr/bin/env ruby
-require 'rubygems'
-$: << File.dirname(__FILE__)+'/../lib'
-require 'graphiterb'
-Configliere.use :commandline, :config_file
-
-Settings.read 'graphite.yaml'
-Settings.resolve!
-Log = Logger.new($stderr) unless defined?(Log)
-
-class LoadavgGraphiteSender < Graphiterb::GraphiteSender
- def hostname
- @hostname ||= `hostname`.chomp
- end
-
- def loadavgs
- # File.open('/proc/loadavg').read.strip.split[0..2]
- `uptime`.chomp.gsub(/.*:\s+/, '').split(/[,\s]+/)
- end
-
- def loadavgs_metrics
- %w[1min 5min 15min].zip(loadavgs).map do |duration, avg|
- ["system.#{hostname}.loadavg_#{duration}", avg]
- end
- end
-
- def send_loop
- loop do
- send *loadavgs_metrics
- Log.info "Sleeping #{Settings.update_delay}"
- sleep Settings.update_delay.to_i
- end
- end
-end
-
-LoadavgGraphiteSender.new.send_loop
View
2  examples/run_servers.sh
@@ -1,2 +0,0 @@
-PYTHONPATH=$pwd/whisper nohup ./bin/run-graphite-devel-server.py --libs=$pwd/webapp/ /usr/local/share/graphite/ >> ./storage/log/webapp/server.log 2>&1 &
-PYTHONPATH=$pwd/whisper nohup ./carbon/bin/carbon-cache.py --debug start >> ./storage/log/carbon-cache/console.log 2>&1 &
View
23 examples/storage_monitor.rb
@@ -1,23 +0,0 @@
-#!/usr/bin/env ruby
-$: << File.dirname(__FILE__)+'/../lib/'
-require 'rubygems'
-require 'graphiterb/graphite_script'
-
-class AvailSpaceMonitor < Graphiterb::GraphiteLogger
- def diskfree
- `/bin/df`.chomp.split("\n").
- grep(%r{^/dev/}).
- map{|line| line.split(/\s+/) } rescue []
- end
-
- def get_metrics metrics, iter, since
- diskfree.each do |handle, size, spaceused, spacefree, percentfree, location|
- metrics << ["system.#{hostname}#{handle.gsub(/\//,'.')}.available", spacefree.to_i]
- end
- end
-end
-
-warn "Update delay is #{Settings.update_delay} seconds. You probably want something larger: some of the metrics are expensive." if Settings.update_delay < 30
-warn "Update delay is #{Settings.update_delay} seconds. You probably want something smaller: need to report in faster than the value in the graphite/conf/storage-schemas." if Settings.update_delay >= 60
-
-AvailSpaceMonitor.new('system').run!
View
6 examples/system.rb
@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'graphiterb/script'
+Configliere.use :commandline, :config_file
+Settings.resolve!
+Graphiterb::Monitors::System.new('system', :iters => 5, :time => 5).run! if $0 == __FILE__
View
20 examples/toy.rb
@@ -3,24 +3,12 @@
$: << File.dirname(__FILE__)+'/../lib'
require 'graphiterb'
Configliere.use :commandline, :config_file
-
-Settings.read 'graphite.yaml'
Settings.resolve!
-Log = Logger.new($stderr) unless defined?(Log)
-
-monitor = Graphiterb::GraphiteLogger.new(:iters => nil, :time => 5)
-
-handle = 'simple_toy'
-loop do
- monitor.periodically do |metrics, iter, since|
- metrics << ["scraper.toy.#{handle}.iter", iter]
- metrics << ["scraper.toy.#{handle}.iter", iter]
- metrics << ["scraper.toy.#{handle}.iter", iter]
- metrics << ["scraper.toy.#{handle}.iter", iter]
+class ToyMonitor < Graphiterb::Monitors::PeriodicMonitor
+ def get_metrics metrics, since
+ metrics << [scope('random', graphite_identifier), rand]
end
- delay = 2
- sleep delay
- print delay.to_s+"\t"
end
+ToyMonitor.new('toy', :iters => 5, :time => 5).run! if $0 == __FILE__
View
33 graphiterb.gemspec
@@ -5,11 +5,11 @@
Gem::Specification.new do |s|
s.name = %q{graphiterb}
- s.version = "0.1.5"
+ s.version = "0.2.12"
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
s.authors = ["Philip (flip) Kromer (@mrflip)"]
- s.date = %q{2010-08-16}
+ s.date = %q{2010-09-28}
s.description = %q{Uses http://github.com/mrflip/configliere and http://graphite.wikidot.com}
s.email = %q{info@infochimps.org}
s.extra_rdoc_files = [
@@ -24,18 +24,20 @@ Gem::Specification.new do |s|
"README.textile",
"Rakefile",
"VERSION",
- "examples/api_call_monitor.rb",
- "examples/file_monitor.rb",
- "examples/loadavg_graphite_sender.rb",
- "examples/run_servers.sh",
- "examples/storage_monitor.rb",
"examples/toy.rb",
"graphiterb.gemspec",
"lib/graphiterb.rb",
- "lib/graphiterb/graphite_logger.rb",
- "lib/graphiterb/graphite_script.rb",
- "lib/graphiterb/graphite_sender.rb",
- "lib/graphiterb/graphite_system_logger.rb",
+ "lib/graphiterb/accumulator.rb",
+ "lib/graphiterb/monitors.rb",
+ "lib/graphiterb/monitors/accumulations_consumer.rb",
+ "lib/graphiterb/monitors/directory_tree.rb",
+ "lib/graphiterb/monitors/disk_space.rb",
+ "lib/graphiterb/monitors/system.rb",
+ "lib/graphiterb/script.rb",
+ "lib/graphiterb/sender.rb",
+ "lib/graphiterb/utils.rb",
+ "lib/graphiterb/utils/log.rb",
+ "lib/graphiterb/utils/system.rb",
"spec/graphiterb_spec.rb",
"spec/spec.opts",
"spec/spec_helper.rb"
@@ -48,11 +50,7 @@ Gem::Specification.new do |s|
s.test_files = [
"spec/graphiterb_spec.rb",
"spec/spec_helper.rb",
- "examples/storage_monitor.rb",
- "examples/api_call_monitor.rb",
- "examples/loadavg_graphite_sender.rb",
- "examples/toy.rb",
- "examples/file_monitor.rb"
+ "examples/toy.rb"
]
if s.respond_to? :specification_version then
@@ -60,13 +58,16 @@ Gem::Specification.new do |s|
s.specification_version = 3
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+ s.add_runtime_dependency(%q<configliere>, [">= 0"])
s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
s.add_development_dependency(%q<yard>, [">= 0"])
else
+ s.add_dependency(%q<configliere>, [">= 0"])
s.add_dependency(%q<rspec>, [">= 1.2.9"])
s.add_dependency(%q<yard>, [">= 0"])
end
else
+ s.add_dependency(%q<configliere>, [">= 0"])
s.add_dependency(%q<rspec>, [">= 1.2.9"])
s.add_dependency(%q<yard>, [">= 0"])
end
View
23 lib/graphiterb.rb
@@ -1,15 +1,18 @@
-require 'socket'
-require 'logger'
+require 'graphiterb/utils'
-require 'configliere'
-Configliere.use :define
+module Graphiterb
+ autoload :Monitors, 'graphiterb/monitors'
+ autoload :Sender, 'graphiterb/sender'
+ autoload :Accumulator, 'graphiterb/accumulator'
+end
+
+Settings.use :define, :config_file
+
+Settings.define :log, :description => "Log output for Graphiterb"
Settings.define :carbon_server, :default => 'localhost', :description => "Host address for carbon database server", :required => true
Settings.define :carbon_port, :default => '2003', :description => "Port for carbon database server", :required => true
Settings.define :update_delay, :default => 30, :description => "How long to wait between updates. Must be faster than the value in the graphite/conf/storage-schemas", :required => true, :type => Integer
Settings.define :on_error_delay, :default => 0.1, :description => "How long to wait on connect errors", :required => true, :type => Float
-
-require 'graphiterb/graphite_sender'
-module Graphiterb
- autoload :GraphiteLogger, 'graphiterb/graphite_logger'
- autoload :GraphiteSystemLogger, 'graphiterb/graphite_system_logger'
-end
+Settings.define :node_name_file, :default => '/etc/node_name', :description => "Path to a file which contains the name of this node", :type => String
+Settings.read '/etc/graphiterb/graphiterb.yaml' if File.exist? '/etc/graphiterb/graphiterb.yaml'
+Settings.resolve!
View
95 lib/graphiterb/accumulator.rb
@@ -0,0 +1,95 @@
+module Graphiterb
+
+ # An accumulator that uses a Redis database as a fast store.
+ #
+ # a = Accumulator.new
+ # a.increment('my_value')
+ #
+ # It's assumed that the Redis database is local and on the default
+ # port, but pass in :host or :port (or any other options Redis.new
+ # understands) to change this.
+ #
+ # By default incrementing 'my_value' which actually increment a
+ # counter stored at the key
+ # 'graphiterb_accumulator:my_value:GRAPHITE_IDENTIFIER'.
+ #
+ # See Graphiterb::Monitors::AccumulationsConsumer for the periodic
+ # monitor that will consume the accumulated counts.
+ class Accumulator
+
+ # The Redis database.
+ attr_accessor :redis
+
+ # The name of the Redis namespace (a string) in which
+ # accumulations are stored (defaults to 'graphiterb')
+ attr_accessor :namespace
+
+ # The Redis namespace (an object) used for the accumulators.
+ attr_accessor :accumulators
+
+ # The top-level Graphite scope inserted for each record.
+ attr_accessor :main_scope
+
+ # Provides methods for finding out about the node this code is
+ # running on.
+ include Graphiterb::Utils::SystemInfo
+
+ # Initialize a new Accumulator.
+ #
+ # Takes the same options as Redis.new.
+ #
+ # Also takes the :namespace option to change where the
+ # accumulations are stored. Different applications can use
+ # different accumulators with different namespaces in a single,
+ # shared Redis.
+ #
+ # @param [String] main_scope
+ # @param [Hash] options
+ def initialize main_scope, options={}
+ require 'redis'
+ begin
+ require 'redis-namespace'
+ rescue LoadError
+ require 'redis/namespace'
+ end
+ @main_scope = main_scope
+ @redis = Redis.new(options)
+ @namespace = options[:namespace] || 'graphiterb'
+ @accumulators = Redis::Namespace.new(namespace, :redis => redis)
+ end
+
+ # Increment the Graphite target +args+ by the given +amount+.
+ #
+ # The target will be automatically scoped, see Accumulator#scope.
+ #
+ # @param [Integer] amount
+ # @param [Array<String>, String] args
+ def increment_by amount, *args
+ accumulators.incrby(scope(*args), amount)
+ end
+
+ # Increment the Graphite target +args+.
+ #
+ # @param [Array<String>, String] args
+ def increment *args
+ accumulators.incr(scope(*args))
+ end
+
+ # Return the scoped accumulator name.
+ #
+ # This will be a valid string target that can be passed directly
+ # to Graphite.
+ #
+ # a = Accumulator.new('scrapers')
+ # a.scope('foo.bar', 'baz')
+ # #=> 'scrapers.foo.bar.baz.ip-120.112.4.383'
+ #
+ # @param [Array<String>, String] args
+ # @return [String]
+ def scope *args
+ [main_scope, args, graphite_identifier].flatten.compact.map(&:to_s).join('.')
+ end
+
+ end
+end
+
View
42 lib/graphiterb/graphite_logger.rb
@@ -1,42 +0,0 @@
-require 'monkeyshines/monitor'
-
-module Graphiterb
- class GraphiteLogger < Monkeyshines::Monitor::PeriodicMonitor
- # Connection to graphite server
- attr_reader :sender
- # the leading segment for sent metrics -- eg 'scrapers' or 'api_calls'
- attr_reader :main_scope
-
- def initialize main_scope, *args
- super *args
- self.time_interval ||= Settings.update_delay
- @sender = GraphiteSender.new
- @main_scope = main_scope
- end
-
- def periodically &block
- super do |iter, since|
- metrics = []
- block.call(metrics, iter, since)
- sender.send *metrics
- end
- end
-
- def hostname
- @host ||= `hostname`.chomp.gsub(".","_")
- end
-
- def scope_name *scope
- [main_scope, scope].flatten.reject(&:blank?).join('.')
- end
-
- def run!
- loop do
- periodically do |metrics, iter, since|
- get_metrics metrics, iter, since
- end
- sleep 1
- end
- end
- end
-end
View
5 lib/graphiterb/graphite_system_logger.rb
@@ -1,5 +0,0 @@
-module Graphiterb
- class GraphiteSystemLogger < GraphiteLogger
-
- end
-end
View
162 lib/graphiterb/monitors.rb
@@ -0,0 +1,162 @@
+module Graphiterb
+ module Monitors
+
+ autoload :DiskSpace, 'graphiterb/monitors/disk_space'
+ autoload :System, 'graphiterb/monitors/system'
+ autoload :DirectoryTree, 'graphiterb/monitors/directory_tree'
+ autoload :AccumulationsConsumer, 'graphiterb/monitors/accumulations_consumer'
+
+ # Accepts a lightweight call every iteration.
+ #
+ # Once either a time or an iteration criterion is met, executes the block
+ # and resets the timer until next execution.
+ #
+ # Note that the +time_interval+ is measured *excution to execution* and not
+ # in multiples of iter_interval. Say I set a time_interval of 300s, and
+ # happen to iterate at 297s and 310s after start. Then the monitor will
+ # execute at 310s, and the next execution will happen on or after 610s.
+ #
+ # Also note that when *either* criterion is met, *both* criteria are
+ # reset. Say I set a time interval of 300s and an +iter_interval+ of 10_000;
+ # and that at 250s I reach iteration 10_000. Then the monitor will execute
+ # on or after 20_000 iteration or 550s, whichever happens first.
+ #
+ # Stolen from Monkeyshines::Monitor::PeriodicMonitor
+ class PeriodicMonitor
+
+ # The main scope under which the monitor's metrics will be
+ # written.
+ attr_accessor :main_scope
+
+ # Maximum number of seconds that should elapse between running
+ # the monitor.
+ attr_accessor :time_interval
+
+ # Maximum number of internal "iterations" that should elapse between running the monitor
+ attr_accessor :iter_interval
+
+ # The options hash the monitor was created with.
+ attr_accessor :options
+
+ # Internal metrics stored by the monitor.
+ attr_accessor :last_time, :current_iter, :iter, :started_at
+
+ # Provides methods for finding out about the node this code is
+ # running on.
+ include Graphiterb::Utils::SystemInfo
+
+ # Create a new PeriodicMonitor
+ def initialize main_scope, options={}
+ self.main_scope = main_scope
+ self.started_at = Time.now.utc.to_f
+ self.last_time = started_at
+ self.iter = 0
+ self.current_iter = 0
+ self.options = options
+ self.time_interval = options[:time] || 30
+ self.iter_interval = options[:iters] || 30
+ end
+
+ # The Graphiterb::Sender used to communicate with the Graphite
+ # server.
+ #
+ # @return [Graphiterb::Sender]
+ def sender
+ @sender ||= Graphiterb::Sender.new
+ end
+
+ # True if more than +iter_interval+ has elapsed since last
+ # execution.
+ def enough_iterations?
+ iter % iter_interval == 0 if iter_interval
+ end
+
+ # True if more than +time_interval+ has elapsed since last execution.
+ def enough_time? now
+ (now - last_time) > time_interval if time_interval
+ end
+
+ # Time since monitor was created
+ #
+ # @return [Time]
+ def since
+ Time.now.utc.to_f - started_at
+ end
+
+ # Overall iterations per second
+ #
+ # @return [Float]
+ def rate
+ iter.to_f / since.to_f
+ end
+
+ # "Instantaneous" iterations per second
+ #
+ # @return [Float]
+ def inst_rate now
+ current_iter.to_f / (now-last_time).to_f
+ end
+
+ # Return the scope built from this monitor's +main_scope+ and
+ # the given +names+.
+ #
+ # monitor.main_scope
+ # #=> 'system.parameters'
+ # monitor.scope 'disk', 'space'
+ # #=> 'system.paramters.disk.space'
+ #
+ # @param [Array<String>] names
+ # @return [String]
+ def scope *names
+ [main_scope, *names].flatten.reject(&:blank?).join('.')
+ end
+
+ # If the interval conditions are met, executes block; otherwise
+ # just does bookkeeping and returns.
+ #
+ # @yield [Array<String>, Time]
+ def periodically &block
+ self.iter += 1
+ self.current_iter += 1
+ now = Time.now.utc.to_f
+ if enough_iterations? || enough_time?(now)
+ metrics = []
+ block.call(metrics, (now-last_time))
+ sender.send(*metrics)
+ self.last_time = now
+ self.current_iter = 0
+ end
+ end
+
+ # Add metrics to the +metrics+ array.
+ #
+ # This method is meant to be overridden by a sub-class (indeed,
+ # it will raise an error if called directly).
+ #
+ # It should take an array of metrics and a time interval since
+ # last ran and insert metrics into the array.")
+ #
+ # @param [Array<String>] metrics
+ # @param [Float] since the number of seconds since the last time the monitor ran
+ def get_metrics metrics, since
+ raise Graphiterb::NotImplementedError.new("Override the get_metrics method of the #{self.class} class")
+ end
+
+ # Run this monitor.
+ #
+ # Sleep for 1 second and then wake up and check if either enough
+ # time (+time_interval+) or enough iterations (+iter_interval+)
+ # have passed run +get_metrics+ if so.
+ def run!
+ loop do
+ periodically do |metrics, since|
+ get_metrics metrics, since
+ end
+ sleep 1
+ end
+ end
+
+ end
+
+ end
+end
View
89 lib/graphiterb/monitors/accumulations_consumer.rb
@@ -0,0 +1,89 @@
+module Graphiterb
+ module Monitors
+
+ # A monitor which consumes counts accumulated in a Redis store by
+ # Graphiterb::Accumulator.
+ class AccumulationsConsumer < Graphiterb::Monitors::PeriodicMonitor
+
+ # The Redis database.
+ attr_accessor :redis
+
+ # The name of the Redis namespace (a string) in which
+ # accumulations are stored (defaults to 'graphiterb')
+ attr_accessor :namespace
+
+ # The key used to store the last report time. Will live inside
+ # the namespace.
+ attr_accessor :report_timestamp_key
+
+ # The Redis namespace used for the accumulators.
+ attr_accessor :accumulators
+
+ # A regular expression that must match the Graphite target
+ # (defaults to always matching).
+ attr_accessor :regexp
+
+ # Instantiate a new AccumulationsConsumer.
+ #
+ # Options are passed to Redis.new as well as
+ # Graphiterb::Monitors::PeriodicMonitor.
+ #
+ # Include the :namespace option to tell this consumer which
+ # Redis namespace to consume keys from (defaults to
+ # 'graphiterb').
+ #
+ # Include the :regexp option if you want this monitor to only
+ # consume keys corresponding to Graphite targets which match the
+ # regexp. This is useful for having multiple
+ # AccumulationsConsumer monitors running with different
+ # frequencies tracking different Graphite target families.
+ def initialize options={}
+ require 'redis'
+ begin
+ require 'redis-namespace'
+ rescue LoadError
+ require 'redis/namespace'
+ end
+ @redis = Redis.new(options)
+ @namespace = options[:namespace] || 'graphiterb'
+ @accumulators = Redis::Namespace.new(namespace, :redis => @redis)
+ @regexp = options[:regexp] || /.*/
+ @report_timestamp_key = options[:report_timestamp_key] || '_last_report_timestamp_'
+ super('fake_scope', options) # FIXME shouldn't have to use a fake scope
+ end
+
+ # Uses Redis' +getset+ call to retrieve total accumulated counts
+ # from Redis and reset them to 0 atomically.
+ def get_metrics metrics, since
+ # compute the length of the interval between the last time
+ # this monitor ran and now
+ last_report_timestamp = accumulators.get(report_timestamp_key).to_i
+ now = Time.now.to_i
+ interval = now - last_report_timestamp
+
+ accumulators.keys.each do |target|
+ next if target == report_timestamp_key
+ next unless regexp && regexp =~ target
+ current_count = accumulators.getset(target, 0) rescue 0.0
+
+ # if we don't know when the last time we ran was, we bail,
+ # resetting the accumulated count to 0 in preparation for
+ # the next iteration
+ #
+ # we lost a data point this way (say the monitor went down
+ # for a while and we brought it back up) but we also don't
+ # ruin the scale of the graph...
+ next if last_report_timestamp == 0
+
+ rate = current_count.to_f / interval.to_f rescue 0.0
+ metrics << [target, rate] # no need to scope as targets are pre-scoped
+ end
+
+ # store the timestamp for this run for the next
+ accumulators.set(report_timestamp_key, now)
+ end
+
+ end
+ end
+end
+
View
123 lib/graphiterb/monitors/directory_tree.rb
@@ -0,0 +1,123 @@
+module Graphiterb
+ module Monitors
+
+ # A class for monitoring the contents of a directory.
+ #
+ # Will only monitor files modified within the last hour by
+ # default.
+ class DirectoryTree < Graphiterb::Monitors::PeriodicMonitor
+
+ # The root of the directory tree being monitored.
+ attr_accessor :root
+
+ # A regular expression filter that must be matched by files
+ # within the root directory to be counted.
+ attr_accessor :filter_re
+
+ def initialize main_scope, root, options={}
+ super(main_scope, options)
+ self.root = File.expand_path(root)
+ self.filter_re = self.options[:filter_re] || /.*/
+ end
+
+ def dirs
+ @dirs ||= Dir[File.join(root, '*')].select{|d| File.directory?(d) }.map { |path| Directory.new(path, filter_re) }
+ end
+
+ def get_metrics metrics, since
+ recent = Directory.recency_filter
+ dirs.each do |dir|
+ metrics << [scope(dir.name, 'num_files', graphite_identifier), dir.num_files(&recent) ]
+ metrics << [scope(dir.name, 'size', graphite_identifier), dir.size(&recent) ]
+ metrics << [scope(dir.name, 'lines', graphite_identifier), dir.line_counts(&recent) ]
+ end
+ end
+
+ # A class for monitoring the contents of a directory.
+ class Directory
+
+ # Path to sample for files
+ attr_accessor :path
+
+ # Wildcard sequence for files under the current directory.
+ attr_accessor :filter_re
+
+ # A recent file was modified within this window.
+ attr_accessor :recent_window
+
+ # Only consider the last this-many files.
+ MAX_FILES = 30
+
+ def initialize path, filter_re=/.*/, options={}
+ self.path = path
+ self.filter_re = filter_re
+ end
+
+ # Name for this pool, suitable for inclusion in a Graphite
+ # target.
+ #
+ # @return [String]
+ def name
+ path.gsub(/\./,'_').gsub(%r{/}, '.').gsub(%r{(^\.|\.$)},'')
+ end
+
+ #
+ # Lists all files in the pool
+ # @param filter_block files only keeps filenames that pass this filter
+ #
+ def files &filter_block
+ Dir[File.join(path, '**/*')].
+ reject{|f| File.directory?(f) }.
+ select{|f| f =~ filter_re }.
+ sort.reverse[0..MAX_FILES].
+ select(&filter_block)
+ end
+
+ def num_files &filter_block
+ files(&filter_block).count
+ end
+
+ def sizes &filter_block
+ files(&filter_block).map{|f| File.size(f) rescue nil }.compact
+ end
+
+ def size &filter_block
+ sizes(&filter_block).sum
+ end
+
+ def avg_size &filter_block
+ sizes(&filter_block).sum.to_f / num_files(&filter_block).to_f
+ end
+
+ def lines_in_result_of command, *args
+ begin
+ escaped_args = args.map{|f| "'#{f}'" }
+ result = `#{command} #{escaped_args.join(" ")}`.chomp
+ result.split(/[\r\n]+/)
+ rescue StandardError => e ; warn(e.backtrace, e) ; return nil ; end
+ end
+
+ def wc
+ @wc ||= `which wc`.chomp
+ end
+
+ def line_counts &filter_block
+ files = files(&filter_block) ; return 0 if files.blank?
+ result = lines_in_result_of(wc, '-l', *files) or return 0
+ counts = result.map{|string| string =~ /^\s*(\d+)\s+/ and $1 }.compact
+ counts.map(&:to_i).sum
+ end
+
+ def self.recent? file
+ (Time.now - File.mtime(file)) < 3600
+ end
+
+ def self.recency_filter
+ Proc.new{|file| recent?(file) }
+ end
+
+ end
+ end
+ end
+end
+
View
29 lib/graphiterb/monitors/disk_space.rb
@@ -0,0 +1,29 @@
+module Graphiterb
+ module Monitors
+
+ # A monitor for how much space is available on a node.
+ class DiskSpace < Graphiterb::Monitors::PeriodicMonitor
+
+ # Runs and parses `df'.
+ #
+ # disk_space.df
+ # #=> [["/dev/sda", "39373712", "20488716", "16884908", "55%", "/"], ["/dev/sdb", "920090332", "397413344", "475939088", "46%", "/home"]]
+ #
+ # @return [Array<Array>]
+ def df
+ `/bin/df`.chomp.split("\n").
+ grep(%r{^/dev/}).
+ map{|line| line.split(/\s+/) } rescue []
+ end
+
+ # Calls +df+ and adds the space available metric.
+ def get_metrics metrics, since
+ # "/dev/sdb1", "39373712", "20488716", "16884908", "55%", "/"
+ df.each do |handle, size, spaceused, spacefree, percentfree, location|
+ disk_name = handle.gsub(/^\//, '').split('/')
+ metrics << [scope(graphite_identifier, disk_name, 'available'), spacefree.to_i]
+ end
+ end
+ end
+ end
+end
View
82 lib/graphiterb/monitors/system.rb
@@ -0,0 +1,82 @@
+module Graphiterb
+ module Monitors
+ class System < Graphiterb::Monitors::PeriodicMonitor
+
+ def top
+ `top -b -n3`.chomp.split(/^top -/).last.split("\n") rescue []
+ end
+
+ def df
+ `/bin/df`.chomp.split("\n").
+ grep(%r{^/dev/}).
+ map{|line| line.split(/\s+/) } rescue []
+ end
+
+ def cpu lines
+ cpus = 0.0
+ total_percentage = 0.0
+ lines.each do |line|
+ next unless line =~ /^Cpu.* *: *([\d\.]+)%us/
+ cpus += 1.0
+ total_percentage += $1.to_f
+ end
+ total_percentage / cpus rescue 0.0
+ end
+
+ def processes lines
+ lines.each do |line|
+ next unless line =~ /^Tasks: *(\d+) *total, *(\d+) *running/
+ return [$1, $2].map(&:to_i)
+ end
+ [0, 0]
+ end
+
+ def memory lines
+ lines.each do |line|
+ next unless line =~ /^Mem: *(\d+)k *total, *(\d+)k *used, *(\d+)k *free/
+ total = $1.to_f
+ return [$2, $3].map do |bytes|
+ 100.0 * (bytes.to_f / total) rescue 0.0
+ end
+ end
+ [0,0]
+ end
+
+ def swap lines
+ lines.each do |line|
+ next unless line =~ /^Swap: *(\d+)k *total, *(\d+)k *used, *(\d+)k *free/
+ total = $1.to_f
+ return [$2, $3].map do |bytes|
+ 100.0 * (bytes.to_f / total) rescue 0.0
+ end
+ end
+ [0,0]
+ end
+
+ def get_metrics metrics, since
+ df.each do |handle, size, spaceused, spacefree, percentused, location|
+ disk_name = handle.gsub(/^\//, '').split('/')
+ percent_free = (100.0 * spacefree.to_f / (spaceused.to_f + spacefree.to_f)) rescue 0.0
+ metrics << [scope(graphite_identifier, disk_name, 'available'), percent_free]
+ end
+
+ lines = top
+
+ metrics << [scope(graphite_identifier, 'cpu', 'avg_usage'), cpu(lines)]
+
+ proc_total, proc_running = processes(lines)
+ metrics << [scope(graphite_identifier, 'processes', 'total'), proc_total ]
+ metrics << [scope(graphite_identifier, 'processes', 'running'), proc_running ]
+
+ mem_used, mem_free = memory(lines)
+ swap_used, swap_free = swap(lines)
+
+ metrics << [scope(graphite_identifier, 'memory', 'used'), mem_used ]
+ metrics << [scope(graphite_identifier, 'memory', 'free'), mem_free ]
+ metrics << [scope(graphite_identifier, 'swap', 'used'), swap_used ]
+ metrics << [scope(graphite_identifier, 'swap', 'free'), swap_free ]
+ end
+
+ end
+ end
+end
View
6 lib/graphiterb/graphite_script.rb → lib/graphiterb/script.rb
@@ -1,9 +1,5 @@
require 'rubygems'
require 'graphiterb'
-require 'wukong/extensions'
-Configliere.use :commandline, :config_file, :define
-
-Log = ::Logger.new($stderr) unless defined?(Log)
-Settings.read 'graphite.yaml'
+Configliere.use :commandline, :config_file, :define
Settings.resolve!
View
19 lib/graphiterb/graphite_sender.rb → lib/graphiterb/sender.rb
@@ -1,23 +1,15 @@
module Graphiterb
- #
- # @example:
- # # in initialize, perhaps
- # self.sender = GraphiteSender
- #
- # # ... somewhere else ...
- # sender.send( ['webserver.pages', pages_this_hour], ['webserver.errors', errors_this_hour] )
- #
- class GraphiteSender
+ class Sender
def initialize
open!
end
def open!
begin
- Log.warn "Connecting to server #{Settings.carbon_server} port #{Settings.carbon_port}"
+ Graphiterb.log.warn "Connecting to server #{Settings.carbon_server} port #{Settings.carbon_port}"
@socket = TCPSocket.new(Settings.carbon_server, Settings.carbon_port)
rescue StandardError => e
- Log.warn "Couldn't connect to server #{Settings.carbon_server} port #{Settings.carbon_port}: #{e.class} #{e}"
+ Graphiterb.log.warn "Couldn't connect to server #{Settings.carbon_server} port #{Settings.carbon_port}: #{e.class} #{e}"
$stderr
end
end
@@ -30,7 +22,7 @@ def safely &block
begin
block.call
rescue StandardError => e
- Log.warn "Sleeping #{Settings.on_error_delay}: #{e.class} #{e}"
+ Graphiterb.log.warn "Sleeping #{Settings.on_error_delay}: #{e.class} #{e}"
sleep Settings.on_error_delay
@socket = nil
return nil
@@ -42,11 +34,12 @@ def timestamp
end
def send *metrics
+ return if metrics.blank?
now = timestamp
safely do
message = metrics.map{|metric, val, ts| [metric, val, (ts||now)].join(" ") }.join("\n")+"\n"
socket.puts(message)
- Log.info message.gsub(/\n+/, "\t")
+ Graphiterb.log.info message.gsub(/\n+/, "\t")
end
end
end
View
17 lib/graphiterb/utils.rb
@@ -0,0 +1,17 @@
+require 'rubygems'
+require 'socket'
+require 'configliere'
+require 'active_support'
+
+require 'graphiterb/utils/log'
+
+module Graphiterb
+
+ Error = Class.new(StandardError)
+ NotImplementedError = Class.new(Error)
+
+ module Utils
+ autoload :SystemInfo, 'graphiterb/utils/system'
+ end
+end
+
View
14 lib/graphiterb/utils/log.rb
@@ -0,0 +1,14 @@
+require 'logger'
+
+module Graphiterb
+
+ class << self; attr_accessor :log end
+
+ def self.instantiate_logger!
+ Graphiterb.log ||= Logger.new(Settings[:log] || STDOUT)
+ Graphiterb.log.datetime_format = "%Y%m%d-%H:%M:%S "
+ Graphiterb.log.level = Logger::INFO
+ end
+
+end
+Graphiterb.instantiate_logger!
View
24 lib/graphiterb/utils/system.rb
@@ -0,0 +1,24 @@
+module Graphiterb
+ module Utils
+
+ # A module which provides information about the node this code is
+ # executing on.
+ #
+ # Maybe it's worth bringing Ohai into this. I'm not sure.
+ module SystemInfo
+
+ def hostname
+ @hostname ||= `hostname`.chomp.gsub(/\./,"_")
+ end
+
+ def node_name
+ @node_name ||= Settings[:node_name_file] && File.exist?(Settings[:node_name_file]) && File.read(Settings[:node_name_file]).chomp.strip.gsub(/\./, '_')
+ end
+
+ def graphite_identifier
+ node_name || hostname
+ end
+ end
+ end
+end
+
View
2  spec/graphiterb_spec.rb
@@ -2,6 +2,6 @@
describe "Graphiterb" do
it "fails" do
- fail "hey buddy, you should probably rename this file and start specing for real"
+ fail "hey buddy, you should probably rename this file and start specing for real. seriously."
end
end
Please sign in to comment.
Something went wrong with that request. Please try again.