Permalink
Browse files

Merge pull request #72 from obfuscurity/metrics-cache

Metrics cache
  • Loading branch information...
2 parents 31914b2 + 49648c1 commit 69fb1686f81f5a575a459bcee2ea842e8825734d @obfuscurity committed Nov 7, 2012
Showing with 162 additions and 34 deletions.
  1. +24 −1 .env.example
  2. +3 −0 Gemfile
  3. +19 −0 Gemfile.lock
  4. +2 −0 Procfile
  5. +5 −4 README.md
  6. +21 −0 Rakefile
  7. +11 −8 config.ru
  8. +1 −0 lib/descartes/models/init.rb
  9. +30 −0 lib/descartes/models/metrics.rb
  10. +28 −20 lib/descartes/public/js/list-metrics.js
  11. +17 −1 lib/descartes/routes/metrics.rb
  12. +1 −0 lib/descartes/web.rb
View
@@ -1 +1,24 @@
-#Insert environment variables you'd like to use in development mode.
+# Insert environment variables you'd like to use in development mode.
+
+# Misc
+#GRAPHITE_URL=http://your.graphite.host.com
+#METRICS_UPDATE_INTERVAL=5m
+#RACK_ENV=development
+#SESSION_SECRET=foobar
+#USE_SVG=false
+
+# Amazon AWS/S3
+#AWS_ACCESS_KEY_ID=...
+#AWS_SECRET_ACCESS_KEY=...
+#S3_BUCKET=my-sooper-secret-descartes-s3-bucket
+
+# OAuth type (either 'google' or 'github')
+#OAUTH_PROVIDER=google
+
+# Google OpenID
+#GOOGLE_OAUTH_DOMAIN=your.domain.com
+
+# GitHub OAuth
+#GITHUB_CLIENT_ID=...
+#GITHUB_CLIENT_SECRET=...
+#GITHUB_ORG_ID=your-github-org
View
@@ -11,9 +11,12 @@ gem "sinatra_auth_github", "0.9.0"
gem "openid-redis-store"
gem "rack-canonical-host"
gem "rack-ssl-enforcer"
+gem "rest-client"
gem "haml"
gem "json"
gem "rspec"
+gem "resque"
+gem "resque-scheduler"
group :development do
gem "foreman"
View
@@ -60,6 +60,17 @@ GEM
rack (>= 1.0)
rake (0.9.2.2)
redis (3.0.1)
+ redis-namespace (1.2.1)
+ redis (~> 3.0.0)
+ resque (1.23.0)
+ multi_json (~> 1.0)
+ redis-namespace (~> 1.0)
+ sinatra (>= 0.9.2)
+ vegas (~> 0.1.2)
+ resque-scheduler (2.0.0)
+ redis (>= 2.0.1)
+ resque (>= 1.20.0)
+ rufus-scheduler
rest-client (1.6.7)
mime-types (>= 1.16)
rspec (2.11.0)
@@ -73,6 +84,8 @@ GEM
ruby-openid (2.2.0)
ruby-openid-apps-discovery (1.2.0)
ruby-openid (>= 2.1.7)
+ rufus-scheduler (2.0.17)
+ tzinfo (>= 0.3.23)
sequel (3.39.0)
sinatra (1.3.1)
rack (~> 1.3, >= 1.3.4)
@@ -89,6 +102,9 @@ GEM
rack (>= 1.0.0)
thor (0.16.0)
tilt (1.3.3)
+ tzinfo (0.3.35)
+ vegas (0.1.11)
+ rack (>= 1.0.0)
warden (1.2.1)
rack (>= 1.0)
warden-github (0.9.1)
@@ -116,6 +132,9 @@ DEPENDENCIES
rack-ssl-enforcer
rack-test
rake
+ resque
+ resque-scheduler
+ rest-client
rspec
sequel
sinatra (= 1.3.1)
View
@@ -1 +1,3 @@
web: bundle exec rackup -p $PORT -s thin
+scheduler: bundle exec rake resque:scheduler QUEUE=*
+worker: bundle exec rake resque:work QUEUE=*
View
@@ -28,6 +28,8 @@ Descartes stores configuration data in PostgreSQL and Google OpenID state in Red
### Options
+* `METRICS_UPDATE_INTERVAL` - How frequently to update the list of known metrics from the remote Graphite server. The more often you add new metrics, the lower this value should be. A reasonable default for most installations would be `1h` (time strings as understood by [Rufus scheduler](https://github.com/jmettraux/rufus-scheduler#the-time-strings-understood-by-rufus-scheduler)). If users complain that they don't see new metrics, it means that it hasn't synced since a new metric has been added. You can simply restart Descartes, and optionally lower this value to suit your users' patience threshold, to manually update the metrics list. __Note__ - this feature adds `scheduler` and `worker` resque classes; if you're hosting your Descartes on Heroku, you will be charged for the extra dyno time.
+
* `USE_SVG` - When set to `true`, will cause Descartes to load SVG output from Graphite instead of the default PNG output. In the future SVG will become the default format, but there is currently a bug in stable Graphite (0.9.10 as of this writing) which causes SVG rendering to fail whenever `secondYAxis` is enabled on any target in a graph.
* `GRAPH_TEMPLATE` - Specify the [Graphite graph template](https://graphite.readthedocs.org/en/latest/render_api.html?#template) to use when rendering graphs.
@@ -82,13 +84,11 @@ See http://blog.rogeriopvl.com/archives/nginx-and-the-http-options-method/ for a
Descartes uses the Sinatra web framework under Ruby 1.9. Anyone wishing to run Descartes as a local service should be familiar with common Ruby packaging and dependency management utilities such as RVM and Bundler. If you are installing a new Ruby version with RVM, make sure that you have the appropriate OpenSSL development libraries installed before compiling Ruby.
+All environment variables can be set from the command-line, although it's suggested to use `.env` instead. This file will automatically be picked up by foreman, which is also helpful when debugging (e.g. `foreman run pry`). This file will not be committed (unless you remove or modify `.gitignore`) so you shouldn't have to worry about accidentally leaking credentials.
+
```bash
$ rvm use 1.9.2
$ bundle install
-$ export OAUTH_PROVIDER=...
-$ export <auth provider tokens>=...
-$ export GRAPHITE_URL=...
-$ export SESSION_SECRET=...
$ createdb descartes
$ bundle exec rake db:migrate:up
$ foreman start
@@ -105,6 +105,7 @@ $ heroku addons:add heroku-postgresql:dev -r $DEPLOY
$ heroku config:set -r $DEPLOY OAUTH_PROVIDER=...
$ heroku config:set -r $DEPLOY <auth provider tokens>=...
$ heroku config:set -r $DEPLOY GRAPHITE_URL=...
+$ heroku config:set -r $DEPLOY METRICS_UPDATE_INTERVAL=1h
$ heroku config:set -r $DEPLOY SESSION_SECRET...
$ heroku config:set -r $DEPLOY RAKE_ENV=production
$ git push $DEPLOY master
View
@@ -41,4 +41,25 @@ RSpec::Core::RakeTask.new(:spec) do |spec|
spec.ruby_opts = '-I .'
end
+# Resque tasks
+require 'resque/tasks'
+require 'resque_scheduler/tasks'
+
+namespace :resque do
+ task :setup do
+ require './lib/descartes/models/init'
+ require 'resque'
+ require 'resque/scheduler'
+ require 'resque_scheduler'
+ Resque.redis = ENV['REDISTOGO_URL'] || 'redis://localhost:6379/1'
+ Resque::Scheduler.dynamic = true
+ Resque.set_schedule('metrics_update', {
+ :every => ENV['METRICS_UPDATE_INTERVAL'],
+ :class => 'MetricListUpdate',
+ :queue => 'low',
+ :description => 'This job will download /metrics/index.json from Graphite and update our metrics list.'
+ })
+ end
+end
+
task :default => :spec
View
@@ -5,21 +5,24 @@ require 'descartes/github_auth'
require 'rack-canonical-host'
use Rack::CanonicalHost do
- case ENV["RACK_ENV"].to_sym
- when :production then ENV["CANONICAL_HOST"] if defined?ENV["CANONICAL_HOST"]
+ case ENV['RACK_ENV'].to_sym
+ when :production then ENV['CANONICAL_HOST'] if defined?ENV['CANONICAL_HOST']
end
end
-use Rack::Session::Cookie, :key => "rack.session",
+use Rack::Session::Cookie, :key => 'rack.session',
:expire_after => 1209600,
- :secret => (ENV["SESSION_SECRET"] || raise("missing SESSION_SECRET"))
+ :secret => (ENV['SESSION_SECRET'] || raise('missing SESSION_SECRET'))
use OmniAuth::Builder do
provider :google_apps,
- :store => OpenID::Store::Redis.new(Redis.connect(:url => ENV["REDISTOGO_URL"]) ||
- OpenID::Store::Redis.new(Redis.connect(:url => "redis://localhost:6379/1"))),
+ :store => OpenID::Store::Redis.new(Redis.connect(:url => ENV['REDISTOGO_URL']) ||
+ OpenID::Store::Redis.new(Redis.connect(:url => 'redis://localhost:6379/1'))),
:name => 'google',
- :domain => ENV["GOOGLE_OAUTH_DOMAIN"]
+ :domain => ENV['GOOGLE_OAUTH_DOMAIN']
end
-run Rack::URLMap.new("/" => Descartes::Web, "/auth/github" => Descartes::GithubAuth)
+run Rack::URLMap.new('/' => Descartes::Web, '/auth/github' => Descartes::GithubAuth)
+
+# seed our Metrics list at startup
+Metric.load
@@ -10,6 +10,7 @@
require 'tags'
require 'dashboards'
require 'graph_dashboard_relations'
+require 'metrics'
Sequel.extension :pagination
Sequel::Model.plugin :json_serializer
@@ -0,0 +1,30 @@
+class Metric
+ @@paths = []
+
+ def self.all
+ @@paths
+ end
+
+ def self.find(search)
+ results = []
+ @@paths.each do |p|
+ results << p if p.match(/#{search}/i)
+ end
+ results
+ end
+
+ def self.load
+ self.update
+ end
+
+ def self.update
+ response = RestClient.get("#{ENV['GRAPHITE_URL']}/metrics/index.json")
+ @@paths = JSON.parse(response)
+ end
+end
+
+class MetricListUpdate
+ def self.perform
+ Metric.update
+ end
+end
@@ -1,7 +1,7 @@
var myLoadedMetrics = [];
var myMatchedMetrics = [];
-var mySearchIntervalId = null;
+var mySearchTimeoutId = null;
// Apply 'selected' class to metrics in our array
// need this for when we refresh
@@ -51,7 +51,7 @@ var renderGraphs = function() {
cache: false,
dataType: 'json',
error: function(xhr, textStatus, errorThrown) { console.log(errorThrown); },
- url: graphiteUrl + '/metrics/index.json'
+ url: '/metrics/'
}).done(function(d) {
if (d.length === 0) {
console.log('No metrics found');
@@ -115,7 +115,7 @@ var renderSparklines = function() {
$('div.loading').addClass('hidden');
// Clear search timer and bind infinite-scroll pagination
- clearInterval(mySearchIntervalId);
+ clearTimeout(mySearchTimeoutId);
$(window).on('scroll', scrollNextPage);
} else {
@@ -156,20 +156,28 @@ $('div.well').on('click', 'a.metrics_graph.btn.open', function() {
// Real-time filter renders sparklines when keystroke timer exceeds 500ms
$(document).on('keyup', 'input.search-ahead', function() {
- clearInterval(mySearchIntervalId);
+ clearTimeout(mySearchTimeoutId);
clearGraphs();
+ $('#search.alert').alert('close');
var mySearchString = $(this).val();
myMatchedMetrics = [];
- for (var i in myLoadedMetrics) {
- if (myLoadedMetrics[i].match(mySearchString)) {
- myMatchedMetrics.push(myLoadedMetrics[i]);
- }
- }
- if (myMatchedMetrics.length === 0) {
- console.log("no matches found");
- } else {
- mySearchIntervalId = setInterval(renderSparklines, 500);
- }
+ mySearchTimeoutId = setTimeout(function() {
+ $.ajax({
+ accepts: {jason: 'application/json'},
+ cache: false,
+ data: {'pattern': mySearchString},
+ dataType: 'json',
+ error: function(xhr, textStatus, errorThrown) { console.log(errorThrown); },
+ url:'/metrics/search'
+ }).done(function(results) {
+ myMatchedMetrics = results;
+ if (myMatchedMetrics.length === 0) {
+ $('div.metrics').before('<div id="search" class="alert">' + 'No matches found for <strong>' + mySearchString + '</strong>.');
+ } else {
+ renderSparklines();
+ }
+ })
+ }, 500);
});
// Add metrics to array of items to graph
@@ -216,10 +224,12 @@ $('.metrics').on('click', 'div.rickshaw_graph.selected', function() {
// Create new graph on submit
$('#metrics_graph_select ul li').on('click', 'button.metrics_graph_submit', function() {
var name = $(this).parent('li').parent('ul').find('input.new-graph-name').val();
- var url = $('span.graph#metrics_graph_preview img').attr('src');
- submitGraph({node: url, name: name}, function(graph) {
- window.location.href = '/graphs/' + graph.uuid
- });
+ if (name.length > 0) {
+ var url = $('span.graph#metrics_graph_preview img').attr('src');
+ submitGraph({node: url, name: name}, function(graph) {
+ window.location.href = '/graphs/' + graph.uuid
+ });
+ }
return false;
});
@@ -255,5 +265,3 @@ var scrollNextPage = function() {
}
}
};
-
-
@@ -2,7 +2,23 @@ module Descartes
class Web < Sinatra::Base
get '/metrics/?' do
- haml :'metrics/list', :title => "Descartes - Metrics List"
+ if request.accept.include?("application/json")
+ content_type "application/json"
+ status 200
+ Metric.all.to_json
+ else
+ haml :'metrics/list', :title => "Descartes - Metrics List"
+ end
+ end
+
+ get '/metrics/search/?' do
+ if request.accept.include?("application/json")
+ content_type "application/json"
+ status 200
+ Metric.find(params[:pattern]).to_json
+ else
+ # halt
+ end
end
end
@@ -2,6 +2,7 @@
require 'rack-ssl-enforcer'
require 'omniauth-google-apps'
require 'openid_redis_store'
+require 'rest-client'
require 'redis'
require 'haml'
require 'json'

0 comments on commit 69fb168

Please sign in to comment.