Permalink
Browse files

synced with mainline

  • Loading branch information...
2 parents 14ded38 + 771fc34 commit fcccd0648bb41f68ff7bcf77791f72f489045429 Philip (flip) Kromer committed Feb 15, 2012
View
@@ -1,21 +1,22 @@
-## MAC OS
-.DS_Store
-## TEXTMATE
-*.tmproj
-tmtags
-## EMACS
-*~
-\#*
-.\#*
-## VIM
-*.swp
+
+## EMACS
+## MAC OS
## PROJECT::GENERAL
+## PROJECT::SPECIFIC
+## TEXTMATE
+## VIM
+*.swp
+*.tmproj
+*~
+.DS_Store
+.\#*
+/TAGS
+\#*
coverage
-rdoc
pkg
-
-## PROJECT::SPECIFIC
+rdoc
+tmtags
View
@@ -10,6 +10,7 @@ begin
gem.email = "info@infochimps.org"
gem.homepage = "http://github.com/infochimps/graphiterb"
gem.authors = ["Philip (flip) Kromer (@mrflip)"]
+ gem.add_dependency "configliere", ">= 0"
gem.add_development_dependency "rspec", ">= 1.2.9"
gem.add_development_dependency "yard", ">= 0"
# gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
View
@@ -1 +1 @@
-0.1.5
+0.2.13
@@ -1,51 +0,0 @@
-#!/usr/bin/env ruby
-$: << File.dirname(__FILE__)+'/../lib/'
-require 'graphiterb'
-require 'graphiterb/graphite_script'
-
-WC_EXEC = '/usr/bin/wc'
-
-class ApiCallMonitor < Graphiterb::GraphiteLogger
- API_CALLS_TO_MONITOR = %w[trstrank wordbag influence conversation]
- ERROR_CODES_TO_MONITOR = %w[4.. 5.. 200]
-
- def initialize *args
- super *args
- @current_total = Hash.new
- @prev_total = Hash.new
- end
-
- def calls api
- total_calls = `cat /var/www/apeyeye/shared/log/apeyeye-access.log | egrep 'GET /soc/net/tw/#{api}' | #{WC_EXEC} -l` rescue 0
- @current_total[api] = total_calls.to_i
- end
-
- def errors error_code
- log_cat = `cat /var/www/apeyeye/shared/log/apeyeye-access.log | egrep 'GET /soc/net/tw/.*HTTP/1\.[0-1]..#{error_code}' | #{WC_EXEC} -l` rescue 0
- @current_total[error_code] = log_cat.to_i
- end
-
- def rate item
- @prev_total[item] ||= @current_total[item]
- rate = @current_total[item].to_i - @prev_total[item].to_i
- @prev_total[item] = @current_total[item]
- [0, rate].max
- end
-
- def get_metrics metrics, iter, since
- API_CALLS_TO_MONITOR.each do |api|
- metrics << [scope_name(hostname, api, 'total_accesses'), calls(api)]
- metrics << [scope_name(hostname, api, 'accesses'), rate(api)]
- end
- ERROR_CODES_TO_MONITOR.each do |code|
- metrics << [scope_name(hostname, code.gsub('.','x'), 'total_errors'), errors(code)]
- metrics << [scope_name(hostname, code.gsub('.','x'), 'errors'), rate(code)]
- end
- end
-end
-
-
-warn "Update delay is #{Settings.update_delay} seconds. You probably want something larger: some of these checks are data-intensive" if Settings.update_delay < 60
-Settings.die "Update delay is #{Settings.update_delay} seconds. You need to radio in at least as often as /usr/local/share/graphite/conf/storage-schemas says -- this is typically 5 minutes." if Settings.update_delay >= 300
-
-ApiCallMonitor.new('apeyeye', :iters => nil, :time => Settings.update_delay).run!
View
@@ -1,113 +0,0 @@
-#!/usr/bin/env ruby
-$: << File.dirname(__FILE__)+'/../lib/'
-require 'graphiterb'
-Settings.define :work_dir, :description => "Base directory where scrapers store files. (Ex: /data/ripd/com.tw)", :required => true
-require 'graphiterb/graphite_script'
-
-#
-# Usage:
-#
-# nohup ~/ics/backend/graphiterb/bin/file_monitor.rb --work_dir=/data/ripd/com.tw --carbon_server=whatever --update_delay=120 > /data/log/file_monitor.log 2>&1 &
-#
-
-WC_EXEC = '/usr/bin/wc'
-
-class FilePool
- # Path to sample for files
- attr_accessor :path
- # wildcard sequence for files under the current directory
- attr_accessor :filter_re
- # A recent file was modified within this window
- attr_accessor :recent_window
- # Only consider the last this-many files
- MAX_FILES = 30
-
- def initialize path, filter_re=/.*/, options={}
- self.path = path
- self.filter_re = filter_re
- end
-
- # Name for this pool, suitable for inclusion in a metrics handle
- def name
- path.gsub(/\./,'_').gsub(%r{/}, '.').gsub(%r{(^\.|\.$)},'')
- end
-
- #
- # Lists all files in the pool
- # @param filter_block files only keeps filenames that pass this filter
- #
- def files &filter_block
- Dir[File.join(path, '**/*')].
- reject{|f| File.directory?(f) }.
- select{|f| f =~ filter_re }.
- sort.reverse[0..MAX_FILES].
- select(&filter_block)
- end
-
- def num_files &filter_block
- files(&filter_block).count
- end
-
- def sizes &filter_block
- files(&filter_block).map{|f| File.size(f) rescue nil }.compact
- end
- def size &filter_block
- sizes(&filter_block).sum
- end
- def avg_size &filter_block
- sizes(&filter_block).sum.to_f / num_files(&filter_block).to_f
- end
-
- def lines_in_result_of command, *args
- begin
- escaped_args = args.map{|f| "'#{f}'" }
- result = `#{command} #{escaped_args.join(" ")}`.chomp
- result.split(/[\r\n]+/)
- rescue StandardError => e ; warn(e.backtrace, e) ; return nil ; end
- end
-
- def line_counts &filter_block
- files = files(&filter_block) ; return 0 if files.blank?
- result = lines_in_result_of(WC_EXEC, '-l', *files) or return 0
- counts = result.map{|wc| wc =~ /^\s*(\d+)\s+/ and $1 }.compact
- counts.map(&:to_i).sum
- end
-
- def self.recent? file
- (Time.now - File.mtime(file)) < 3600
- end
- def self.recency_filter
- Proc.new{|file| recent?(file) }
- end
-end
-
-class FileMonitor < Graphiterb::GraphiteSystemLogger
- attr_accessor :path
- attr_accessor :pools
-
- def initialize *args
- super *args
- self.path = Settings.work_dir
- self.pools = {}
- populate_pools!
- end
-
- def populate_pools!
- Dir[File.join(path, '*')].select{|d| File.directory?(d) }.each do |dir|
- self.pools[dir] ||= FilePool.new(dir, %r{20\d*/.*\.(?:tsv|json|xml)})
- end
- end
-
- def get_metrics metrics, iter, since
- recent = FilePool.recency_filter
- pools.each do |pool_path, pool|
- metrics << [scope_name(pool.name, hostname, 'active_files'), pool.num_files(&recent) ]
- metrics << [scope_name(pool.name, hostname, 'active_file_size'), pool.size(&recent) ]
- metrics << [scope_name(pool.name, hostname, 'line_counts'), pool.line_counts(&recent) ]
- end
- end
-end
-
-warn "Update delay is #{Settings.update_delay} seconds. You probably want something larger: some of the metrics are expensive." if Settings.update_delay < 60
-warn "Update delay is #{Settings.update_delay} seconds. You probably want something smaller: need to report in faster than the value in the graphite/conf/storage-schemas." if Settings.update_delay >= 300
-FileMonitor.new('scraper', :iters => nil, :time => Settings.update_delay).run!
@@ -1,36 +0,0 @@
-#!/usr/bin/env ruby
-require 'rubygems'
-$: << File.dirname(__FILE__)+'/../lib'
-require 'graphiterb'
-Configliere.use :commandline, :config_file
-
-Settings.read 'graphite.yaml'
-Settings.resolve!
-Log = Logger.new($stderr) unless defined?(Log)
-
-class LoadavgGraphiteSender < Graphiterb::GraphiteSender
- def hostname
- @hostname ||= `hostname`.chomp
- end
-
- def loadavgs
- # File.open('/proc/loadavg').read.strip.split[0..2]
- `uptime`.chomp.gsub(/.*:\s+/, '').split(/[,\s]+/)
- end
-
- def loadavgs_metrics
- %w[1min 5min 15min].zip(loadavgs).map do |duration, avg|
- ["system.#{hostname}.loadavg_#{duration}", avg]
- end
- end
-
- def send_loop
- loop do
- send *loadavgs_metrics
- Log.info "Sleeping #{Settings.update_delay}"
- sleep Settings.update_delay.to_i
- end
- end
-end
-
-LoadavgGraphiteSender.new.send_loop
View
@@ -1,2 +0,0 @@
-PYTHONPATH=$pwd/whisper nohup ./bin/run-graphite-devel-server.py --libs=$pwd/webapp/ /usr/local/share/graphite/ >> ./storage/log/webapp/server.log 2>&1 &
-PYTHONPATH=$pwd/whisper nohup ./carbon/bin/carbon-cache.py --debug start >> ./storage/log/carbon-cache/console.log 2>&1 &
@@ -1,23 +0,0 @@
-#!/usr/bin/env ruby
-$: << File.dirname(__FILE__)+'/../lib/'
-require 'rubygems'
-require 'graphiterb/graphite_script'
-
-class AvailSpaceMonitor < Graphiterb::GraphiteLogger
- def diskfree
- `/bin/df`.chomp.split("\n").
- grep(%r{^/dev/}).
- map{|line| line.split(/\s+/) } rescue []
- end
-
- def get_metrics metrics, iter, since
- diskfree.each do |handle, size, spaceused, spacefree, percentfree, location|
- metrics << ["system.#{hostname}#{handle.gsub(/\//,'.')}.available", spacefree.to_i]
- end
- end
-end
-
-warn "Update delay is #{Settings.update_delay} seconds. You probably want something larger: some of the metrics are expensive." if Settings.update_delay < 30
-warn "Update delay is #{Settings.update_delay} seconds. You probably want something smaller: need to report in faster than the value in the graphite/conf/storage-schemas." if Settings.update_delay >= 60
-
-AvailSpaceMonitor.new('system').run!
View
@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'graphiterb/script'
+Configliere.use :commandline, :config_file
+Settings.resolve!
+Graphiterb::Monitors::System.new('system', :iters => 5, :time => 5).run! if $0 == __FILE__
View
@@ -3,24 +3,12 @@
$: << File.dirname(__FILE__)+'/../lib'
require 'graphiterb'
Configliere.use :commandline, :config_file
-
-Settings.read 'graphite.yaml'
Settings.resolve!
-Log = Logger.new($stderr) unless defined?(Log)
-
-monitor = Graphiterb::GraphiteLogger.new(:iters => nil, :time => 5)
-
-handle = 'simple_toy'
-loop do
- monitor.periodically do |metrics, iter, since|
- metrics << ["scraper.toy.#{handle}.iter", iter]
- metrics << ["scraper.toy.#{handle}.iter", iter]
- metrics << ["scraper.toy.#{handle}.iter", iter]
- metrics << ["scraper.toy.#{handle}.iter", iter]
+class ToyMonitor < Graphiterb::Monitors::PeriodicMonitor
+ def get_metrics metrics, since
+ metrics << [scope('random', graphite_identifier), rand]
end
- delay = 2
- sleep delay
- print delay.to_s+"\t"
end
+ToyMonitor.new('toy', :iters => 5, :time => 5).run! if $0 == __FILE__
View
@@ -5,11 +5,11 @@
Gem::Specification.new do |s|
s.name = %q{graphiterb}
- s.version = "0.1.5"
+ s.version = "0.2.12"
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
s.authors = ["Philip (flip) Kromer (@mrflip)"]
- s.date = %q{2010-08-16}
+ s.date = %q{2010-09-28}
s.description = %q{Uses http://github.com/mrflip/configliere and http://graphite.wikidot.com}
s.email = %q{info@infochimps.org}
s.extra_rdoc_files = [
@@ -24,18 +24,20 @@ Gem::Specification.new do |s|
"README.textile",
"Rakefile",
"VERSION",
- "examples/api_call_monitor.rb",
- "examples/file_monitor.rb",
- "examples/loadavg_graphite_sender.rb",
- "examples/run_servers.sh",
- "examples/storage_monitor.rb",
"examples/toy.rb",
"graphiterb.gemspec",
"lib/graphiterb.rb",
- "lib/graphiterb/graphite_logger.rb",
- "lib/graphiterb/graphite_script.rb",
- "lib/graphiterb/graphite_sender.rb",
- "lib/graphiterb/graphite_system_logger.rb",
+ "lib/graphiterb/accumulator.rb",
+ "lib/graphiterb/monitors.rb",
+ "lib/graphiterb/monitors/accumulations_consumer.rb",
+ "lib/graphiterb/monitors/directory_tree.rb",
+ "lib/graphiterb/monitors/disk_space.rb",
+ "lib/graphiterb/monitors/system.rb",
+ "lib/graphiterb/script.rb",
+ "lib/graphiterb/sender.rb",
+ "lib/graphiterb/utils.rb",
+ "lib/graphiterb/utils/log.rb",
+ "lib/graphiterb/utils/system.rb",
"spec/graphiterb_spec.rb",
"spec/spec.opts",
"spec/spec_helper.rb"
@@ -48,25 +50,24 @@ Gem::Specification.new do |s|
s.test_files = [
"spec/graphiterb_spec.rb",
"spec/spec_helper.rb",
- "examples/storage_monitor.rb",
- "examples/api_call_monitor.rb",
- "examples/loadavg_graphite_sender.rb",
- "examples/toy.rb",
- "examples/file_monitor.rb"
+ "examples/toy.rb"
]
if s.respond_to? :specification_version then
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
s.specification_version = 3
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+ s.add_runtime_dependency(%q<configliere>, [">= 0"])
s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
s.add_development_dependency(%q<yard>, [">= 0"])
else
+ s.add_dependency(%q<configliere>, [">= 0"])
s.add_dependency(%q<rspec>, [">= 1.2.9"])
s.add_dependency(%q<yard>, [">= 0"])
end
else
+ s.add_dependency(%q<configliere>, [">= 0"])
s.add_dependency(%q<rspec>, [">= 1.2.9"])
s.add_dependency(%q<yard>, [">= 0"])
end
Oops, something went wrong.

0 comments on commit fcccd06

Please sign in to comment.