Permalink
Browse files

Merge pull request #72 from obfuscurity/metrics-cache

Metrics cache
  • Loading branch information...
obfuscurity committed Nov 7, 2012
2 parents 31914b2 + 49648c1 commit 69fb1686f81f5a575a459bcee2ea842e8825734d
Showing with 162 additions and 34 deletions.
  1. +24 −1 .env.example
  2. +3 −0 Gemfile
  3. +19 −0 Gemfile.lock
  4. +2 −0 Procfile
  5. +5 −4 README.md
  6. +21 −0 Rakefile
  7. +11 −8 config.ru
  8. +1 −0 lib/descartes/models/init.rb
  9. +30 −0 lib/descartes/models/metrics.rb
  10. +28 −20 lib/descartes/public/js/list-metrics.js
  11. +17 −1 lib/descartes/routes/metrics.rb
  12. +1 −0 lib/descartes/web.rb
View
@@ -1 +1,24 @@
#Insert environment variables you'd like to use in development mode.
# Insert environment variables you'd like to use in development mode.
# Misc
#GRAPHITE_URL=http://your.graphite.host.com
#METRICS_UPDATE_INTERVAL=5m
#RACK_ENV=development
#SESSION_SECRET=foobar
#USE_SVG=false
# Amazon AWS/S3
#AWS_ACCESS_KEY_ID=...
#AWS_SECRET_ACCESS_KEY=...
#S3_BUCKET=my-sooper-secret-descartes-s3-bucket
# OAuth type (either 'google' or 'github')
#OAUTH_PROVIDER=google
# Google OpenID
#GOOGLE_OAUTH_DOMAIN=your.domain.com
# GitHub OAuth
#GITHUB_CLIENT_ID=...
#GITHUB_CLIENT_SECRET=...
#GITHUB_ORG_ID=your-github-org
View
@@ -11,9 +11,12 @@ gem "sinatra_auth_github", "0.9.0"
gem "openid-redis-store"
gem "rack-canonical-host"
gem "rack-ssl-enforcer"
gem "rest-client"
gem "haml"
gem "json"
gem "rspec"
gem "resque"
gem "resque-scheduler"
group :development do
gem "foreman"
View
@@ -60,6 +60,17 @@ GEM
rack (>= 1.0)
rake (0.9.2.2)
redis (3.0.1)
redis-namespace (1.2.1)
redis (~> 3.0.0)
resque (1.23.0)
multi_json (~> 1.0)
redis-namespace (~> 1.0)
sinatra (>= 0.9.2)
vegas (~> 0.1.2)
resque-scheduler (2.0.0)
redis (>= 2.0.1)
resque (>= 1.20.0)
rufus-scheduler
rest-client (1.6.7)
mime-types (>= 1.16)
rspec (2.11.0)
@@ -73,6 +84,8 @@ GEM
ruby-openid (2.2.0)
ruby-openid-apps-discovery (1.2.0)
ruby-openid (>= 2.1.7)
rufus-scheduler (2.0.17)
tzinfo (>= 0.3.23)
sequel (3.39.0)
sinatra (1.3.1)
rack (~> 1.3, >= 1.3.4)
@@ -89,6 +102,9 @@ GEM
rack (>= 1.0.0)
thor (0.16.0)
tilt (1.3.3)
tzinfo (0.3.35)
vegas (0.1.11)
rack (>= 1.0.0)
warden (1.2.1)
rack (>= 1.0)
warden-github (0.9.1)
@@ -116,6 +132,9 @@ DEPENDENCIES
rack-ssl-enforcer
rack-test
rake
resque
resque-scheduler
rest-client
rspec
sequel
sinatra (= 1.3.1)
View
@@ -1 +1,3 @@
web: bundle exec rackup -p $PORT -s thin
scheduler: bundle exec rake resque:scheduler QUEUE=*
worker: bundle exec rake resque:work QUEUE=*
View
@@ -28,6 +28,8 @@ Descartes stores configuration data in PostgreSQL and Google OpenID state in Red
### Options
* `METRICS_UPDATE_INTERVAL` - How frequently to update the list of known metrics from the remote Graphite server. The more often you add new metrics, the lower this value should be. A reasonable default for most installations would be `1h` (time strings as understood by [Rufus scheduler](https://github.com/jmettraux/rufus-scheduler#the-time-strings-understood-by-rufus-scheduler)). If users complain that they don't see new metrics, it means that it hasn't synced since a new metric has been added. You can simply restart Descartes, and optionally lower this value to suit your users' patience threshold, to manually update the metrics list. __Note__ - this feature adds `scheduler` and `worker` resque classes; if you're hosting your Descartes on Heroku, you will be charged for the extra dyno time.
* `USE_SVG` - When set to `true`, will cause Descartes to load SVG output from Graphite instead of the default PNG output. In the future SVG will become the default format, but there is currently a bug in stable Graphite (0.9.10 as of this writing) which causes SVG rendering to fail whenever `secondYAxis` is enabled on any target in a graph.
* `GRAPH_TEMPLATE` - Specify the [Graphite graph template](https://graphite.readthedocs.org/en/latest/render_api.html?#template) to use when rendering graphs.
@@ -82,13 +84,11 @@ See http://blog.rogeriopvl.com/archives/nginx-and-the-http-options-method/ for a
Descartes uses the Sinatra web framework under Ruby 1.9. Anyone wishing to run Descartes as a local service should be familiar with common Ruby packaging and dependency management utilities such as RVM and Bundler. If you are installing a new Ruby version with RVM, make sure that you have the appropriate OpenSSL development libraries installed before compiling Ruby.
All environment variables can be set from the command-line, although it's suggested to use `.env` instead. This file will automatically be picked up by foreman, which is also helpful when debugging (e.g. `foreman run pry`). This file will not be committed (unless you remove or modify `.gitignore`) so you shouldn't have to worry about accidentally leaking credentials.
```bash
$ rvm use 1.9.2
$ bundle install
$ export OAUTH_PROVIDER=...
$ export <auth provider tokens>=...
$ export GRAPHITE_URL=...
$ export SESSION_SECRET=...
$ createdb descartes
$ bundle exec rake db:migrate:up
$ foreman start
@@ -105,6 +105,7 @@ $ heroku addons:add heroku-postgresql:dev -r $DEPLOY
$ heroku config:set -r $DEPLOY OAUTH_PROVIDER=...
$ heroku config:set -r $DEPLOY <auth provider tokens>=...
$ heroku config:set -r $DEPLOY GRAPHITE_URL=...
$ heroku config:set -r $DEPLOY METRICS_UPDATE_INTERVAL=1h
$ heroku config:set -r $DEPLOY SESSION_SECRET...
$ heroku config:set -r $DEPLOY RAKE_ENV=production
$ git push $DEPLOY master
View
@@ -41,4 +41,25 @@ RSpec::Core::RakeTask.new(:spec) do |spec|
spec.ruby_opts = '-I .'
end
# Resque tasks
require 'resque/tasks'
require 'resque_scheduler/tasks'
namespace :resque do
task :setup do
require './lib/descartes/models/init'
require 'resque'
require 'resque/scheduler'
require 'resque_scheduler'
Resque.redis = ENV['REDISTOGO_URL'] || 'redis://localhost:6379/1'
Resque::Scheduler.dynamic = true
Resque.set_schedule('metrics_update', {
:every => ENV['METRICS_UPDATE_INTERVAL'],
:class => 'MetricListUpdate',
:queue => 'low',
:description => 'This job will download /metrics/index.json from Graphite and update our metrics list.'
})
end
end
task :default => :spec
View
@@ -5,21 +5,24 @@ require 'descartes/github_auth'
require 'rack-canonical-host'
use Rack::CanonicalHost do
case ENV["RACK_ENV"].to_sym
when :production then ENV["CANONICAL_HOST"] if defined?ENV["CANONICAL_HOST"]
case ENV['RACK_ENV'].to_sym
when :production then ENV['CANONICAL_HOST'] if defined?ENV['CANONICAL_HOST']
end
end
use Rack::Session::Cookie, :key => "rack.session",
use Rack::Session::Cookie, :key => 'rack.session',
:expire_after => 1209600,
:secret => (ENV["SESSION_SECRET"] || raise("missing SESSION_SECRET"))
:secret => (ENV['SESSION_SECRET'] || raise('missing SESSION_SECRET'))
use OmniAuth::Builder do
provider :google_apps,
:store => OpenID::Store::Redis.new(Redis.connect(:url => ENV["REDISTOGO_URL"]) ||
OpenID::Store::Redis.new(Redis.connect(:url => "redis://localhost:6379/1"))),
:store => OpenID::Store::Redis.new(Redis.connect(:url => ENV['REDISTOGO_URL']) ||
OpenID::Store::Redis.new(Redis.connect(:url => 'redis://localhost:6379/1'))),
:name => 'google',
:domain => ENV["GOOGLE_OAUTH_DOMAIN"]
:domain => ENV['GOOGLE_OAUTH_DOMAIN']
end
run Rack::URLMap.new("/" => Descartes::Web, "/auth/github" => Descartes::GithubAuth)
run Rack::URLMap.new('/' => Descartes::Web, '/auth/github' => Descartes::GithubAuth)
# seed our Metrics list at startup
Metric.load
@@ -10,6 +10,7 @@
require 'tags'
require 'dashboards'
require 'graph_dashboard_relations'
require 'metrics'
Sequel.extension :pagination
Sequel::Model.plugin :json_serializer
@@ -0,0 +1,30 @@
class Metric
@@paths = []
def self.all
@@paths
end
def self.find(search)
results = []
@@paths.each do |p|
results << p if p.match(/#{search}/i)
end
results
end
def self.load
self.update
end
def self.update
response = RestClient.get("#{ENV['GRAPHITE_URL']}/metrics/index.json")
@@paths = JSON.parse(response)
end
end
class MetricListUpdate
def self.perform
Metric.update
end
end
@@ -1,7 +1,7 @@
var myLoadedMetrics = [];
var myMatchedMetrics = [];
var mySearchIntervalId = null;
var mySearchTimeoutId = null;
// Apply 'selected' class to metrics in our array
// need this for when we refresh
@@ -51,7 +51,7 @@ var renderGraphs = function() {
cache: false,
dataType: 'json',
error: function(xhr, textStatus, errorThrown) { console.log(errorThrown); },
url: graphiteUrl + '/metrics/index.json'
url: '/metrics/'
}).done(function(d) {
if (d.length === 0) {
console.log('No metrics found');
@@ -115,7 +115,7 @@ var renderSparklines = function() {
$('div.loading').addClass('hidden');
// Clear search timer and bind infinite-scroll pagination
clearInterval(mySearchIntervalId);
clearTimeout(mySearchTimeoutId);
$(window).on('scroll', scrollNextPage);
} else {
@@ -156,20 +156,28 @@ $('div.well').on('click', 'a.metrics_graph.btn.open', function() {
// Real-time filter renders sparklines when keystroke timer exceeds 500ms
$(document).on('keyup', 'input.search-ahead', function() {
clearInterval(mySearchIntervalId);
clearTimeout(mySearchTimeoutId);
clearGraphs();
$('#search.alert').alert('close');
var mySearchString = $(this).val();
myMatchedMetrics = [];
for (var i in myLoadedMetrics) {
if (myLoadedMetrics[i].match(mySearchString)) {
myMatchedMetrics.push(myLoadedMetrics[i]);
}
}
if (myMatchedMetrics.length === 0) {
console.log("no matches found");
} else {
mySearchIntervalId = setInterval(renderSparklines, 500);
}
mySearchTimeoutId = setTimeout(function() {
$.ajax({
accepts: {jason: 'application/json'},
cache: false,
data: {'pattern': mySearchString},
dataType: 'json',
error: function(xhr, textStatus, errorThrown) { console.log(errorThrown); },
url:'/metrics/search'
}).done(function(results) {
myMatchedMetrics = results;
if (myMatchedMetrics.length === 0) {
$('div.metrics').before('<div id="search" class="alert">' + 'No matches found for <strong>' + mySearchString + '</strong>.');
} else {
renderSparklines();
}
})
}, 500);
});
// Add metrics to array of items to graph
@@ -216,10 +224,12 @@ $('.metrics').on('click', 'div.rickshaw_graph.selected', function() {
// Create new graph on submit
$('#metrics_graph_select ul li').on('click', 'button.metrics_graph_submit', function() {
var name = $(this).parent('li').parent('ul').find('input.new-graph-name').val();
var url = $('span.graph#metrics_graph_preview img').attr('src');
submitGraph({node: url, name: name}, function(graph) {
window.location.href = '/graphs/' + graph.uuid
});
if (name.length > 0) {
var url = $('span.graph#metrics_graph_preview img').attr('src');
submitGraph({node: url, name: name}, function(graph) {
window.location.href = '/graphs/' + graph.uuid
});
}
return false;
});
@@ -255,5 +265,3 @@ var scrollNextPage = function() {
}
}
};
@@ -2,7 +2,23 @@ module Descartes
class Web < Sinatra::Base
get '/metrics/?' do
haml :'metrics/list', :title => "Descartes - Metrics List"
if request.accept.include?("application/json")
content_type "application/json"
status 200
Metric.all.to_json
else
haml :'metrics/list', :title => "Descartes - Metrics List"
end
end
get '/metrics/search/?' do
if request.accept.include?("application/json")
content_type "application/json"
status 200
Metric.find(params[:pattern]).to_json
else
# halt
end
end
end
View
@@ -2,6 +2,7 @@
require 'rack-ssl-enforcer'
require 'omniauth-google-apps'
require 'openid_redis_store'
require 'rest-client'
require 'redis'
require 'haml'
require 'json'

0 comments on commit 69fb168

Please sign in to comment.