Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Refactored job running and replaced Resque with Sidekiq

  • Loading branch information...
commit c7f09aee430df3da0bff170601e6181966bf4ae8 1 parent 4619191
Joel Moss authored
8 Gemfile
View
@@ -14,7 +14,9 @@ end
gem 'jquery-rails'
gem 'twitter-bootstrap-rails'
-gem 'resque', '~> 1.20.0'
+gem 'sidekiq'
+gem 'slim'
+gem 'sinatra'
gem 'omniauth-github'
gem 'yajl-ruby'
gem 'faraday'
@@ -37,7 +39,7 @@ gem 'ansible'
# for projects to use if needed.
gem 'delayed_job', :require => nil
gem 'whenever', :require => nil
-gem 'airbrake', :require => nil
+# gem 'airbrake', :require => nil
gem 'newrelic_rpm', :require => nil
group :development, :test do
@@ -51,7 +53,7 @@ group :development do
gem 'guard'
gem 'guard-rspec'
gem 'guard-bundler'
- gem 'guard-rails'
+ gem 'guard-pow'
end
group :test do
26 Gemfile.lock
View
@@ -29,9 +29,6 @@ GEM
i18n (~> 0.6)
multi_json (~> 1.0)
addressable (2.2.7)
- airbrake (3.0.9)
- activesupport
- builder
ansible (0.2.0)
arel (3.0.2)
builder (3.0.0)
@@ -42,6 +39,7 @@ GEM
net-ssh (>= 2.0.14)
net-ssh-gateway (>= 1.1.0)
capistrano_colors (0.5.5)
+ celluloid (0.9.0)
chronic (0.6.7)
coffee-rails (3.2.2)
coffee-script (>= 2.2.0)
@@ -52,6 +50,7 @@ GEM
coffee-script-source (1.2.0)
commonjs (0.2.0)
therubyracer (~> 0.9.9)
+ connection_pool (0.9.0)
crack (0.3.1)
database_cleaner (0.7.1)
delayed_job (3.0.1)
@@ -87,8 +86,8 @@ GEM
guard-bundler (0.1.3)
bundler (>= 1.0.0)
guard (>= 0.2.2)
- guard-rails (0.1.0)
- guard (>= 0.2.2)
+ guard-pow (0.2.1)
+ guard (>= 0.3.0)
guard-rspec (0.6.0)
guard (>= 0.10.0)
has_scope (0.5.1)
@@ -206,6 +205,12 @@ GEM
sass (>= 3.1.10)
tilt (~> 1.3)
shoulda-matchers (1.0.0)
+ sidekiq (0.9.1)
+ celluloid
+ connection_pool (>= 0.9.0)
+ multi_json
+ redis
+ redis-namespace
simple_form (2.0.1)
actionpack (~> 3.0)
activemodel (~> 3.0)
@@ -213,11 +218,15 @@ GEM
rack (~> 1.3, >= 1.3.6)
rack-protection (~> 1.2)
tilt (~> 1.3, >= 1.3.3)
+ slim (1.1.1)
+ temple (~> 0.4.0)
+ tilt (~> 1.3.2)
sprockets (2.1.2)
hike (~> 1.2)
rack (~> 1.0)
tilt (~> 1.1, != 1.3.0)
sqlite3 (1.3.5)
+ temple (0.4.0)
therubyracer (0.9.10)
libv8 (~> 3.3.10)
thor (0.14.6)
@@ -248,7 +257,6 @@ PLATFORMS
ruby
DEPENDENCIES
- airbrake
ansible
capistrano (~> 2.11)
capistrano_colors
@@ -265,7 +273,7 @@ DEPENDENCIES
grit
guard
guard-bundler
- guard-rails
+ guard-pow
guard-rspec
inherited_resources
jquery-rails
@@ -277,12 +285,14 @@ DEPENDENCIES
open4
permanent_records
rails (= 3.2.2)
- resque (~> 1.20.0)
resque_spec
rspec-rails
sass-rails (~> 3.2.3)
shoulda-matchers
+ sidekiq
simple_form (~> 2)
+ sinatra
+ slim
sqlite3
twitter-bootstrap-rails
uglifier (>= 1.0.3)
19 Guardfile
View
@@ -22,12 +22,23 @@ guard 'rspec' do
watch('app/controllers/application_controller.rb') { "spec/controllers" }
# Capybara request specs
watch(%r{^app/views/(.+)/.*\.(erb|haml)$}) { |m| "spec/requests/#{m[1]}_spec.rb" }
-
+
watch('Gemfile.lock') { "spec" }
end
-guard 'rails' do
+# guard 'rails' do
+# watch('Gemfile.lock')
+# watch(%r{^(config|lib)/.*})
+# end
+
+guard 'pow' do
+ watch('.powrc')
+ watch('.powenv')
+ watch('.rvmrc')
+ watch('Gemfile')
watch('Gemfile.lock')
- watch(%r{^(config|lib)/.*})
+ watch('config/application.rb')
+ watch('config/environment.rb')
+ watch(%r{^config/environments/.*\.rb$})
+ watch(%r{^config/initializers/.*\.rb$})
end
-
26 app/assets/javascripts/app/jobs.js.coffee
View
@@ -1,5 +1,5 @@
$ ->
-
+
$("#slider").slider
value: 3
min: 1
@@ -8,32 +8,32 @@ $ ->
num = ui.value+1
verbosity = while num -= 1 then 'v'
$("#job_verbosity").val verbosity.join('')
-
-
+
+
if $('#job-results').size() > 0
-
+
div = $ '#job-results'
project_id = div.data 'project_id'
job_id = div.data 'job_id'
-
+
get_job_status = ->
$.getJSON "/projects/#{project_id}/jobs/#{job_id}", (data) ->
- if data.results == null
+ if data.completed_at == null
setTimeout get_job_status, 3000
else
- div.removeClass('hide').html data.results
-
title = div.siblings('h3')
title.find('img').remove()
text = title.text().replace /Running/, 'Task'
text = text.replace /\.\.\./, 'completed'
title.text text
-
+
$('.alert').remove()
-
+
if data.success
- $('.tabs .pull-right').html '<span class="label label-success">SUCCESS</span>'
+ $('.tabbable .nav .pull-right').html '<span class="label label-success">SUCCESS</span>'
else
- $('.tabs .pull-right').html '<span class="label label-important">FAILED</span>'
-
+ $('.tabbable .nav .pull-right').html '<span class="label label-important">FAILED</span>'
+
+ div.removeClass('hide').html data.results if data.results != null
+
setTimeout get_job_status, 3000
18 app/assets/stylesheets/custom.css.scss
View
@@ -10,7 +10,7 @@ footer {
margin-top: 65px;
font-size: 11px;
padding-top: 10px;
-
+
a {
color: #666;
}
@@ -72,10 +72,16 @@ footer {
border-bottom: 1px solid #EEE;
}
-#results pre {
- overflow-x: auto;
- white-space: pre;
- word-wrap: normal;
+#results {
+ h3 {
+ margin-bottom: 10px;
+ }
+
+ pre {
+ overflow-x: auto;
+ white-space: pre;
+ word-wrap: normal;
+ }
}
#slider {
@@ -92,7 +98,7 @@ footer {
color: #FFF;
line-height: 1.8em;
font-size: 12px;
-
+
.ansible_1 { font-weight: bold; }
.ansible_4 { text-decoration: underline; }
.ansible_30 { color: black; }
23 app/models/job.rb
View
@@ -15,28 +15,21 @@ def self.deleted
def run_task
- status, stdout_output, stderr_output = nil, "", ""
-
FileUtils.chdir project.repo.path do
- status = Open4::popen4 full_command do |pid, stdin, stdout, stderr|
- stdin.close
-
- stdout_output += stdout.read.strip
- stderr_output += stderr.read.strip
+ out = capture_stdout do
+ Strano::CLI.parse(Strano::Logger.new(self), full_command).execute!
end
- end
- if status.exitstatus != 0
- raise Exception, stderr_output
- else
- stdout_output
+ unless out.string.blank?
+ update_attribute(:results, (results || '') + "\n > #{out.string}")
+ end
end
end
def complete?
!completed_at.nil?
end
-
+
def command
"#{stage} #{task}"
end
@@ -45,11 +38,11 @@ def command
private
def full_command
- "bundle exec cap -f #{Rails.root.join('Capfile.repos')} -f Capfile -Xx#{verbosity} -l STDOUT #{command}"
+ %W(-f #{Rails.root.join('Capfile.repos')} -f Capfile -Xx#{verbosity}) + command.split(' ')
end
def execute_task
- Resque.enqueue CapExecute, id
+ CapExecute.perform_async id
end
end
41 app/models/job/cap_execute.rb
View
@@ -1,38 +1,17 @@
class Job
- module ReportFailure
- def on_failure_retry(e, job_id)
- job = Job.find(job_id)
-
- job.update_attributes :results => e.message,
- :completed_at => Time.now,
- :success => false
- end
- end
-
class CapExecute
- include Ansible
- extend ReportFailure
-
- @queue = :job
-
- def self.perform(job_id)
+ include Sidekiq::Worker, Ansible
+
+ def perform(job_id)
job = Job.find(job_id)
-
+
# Make sure the local repo is up to date.
- Project::PullRepo.perform(job.project.id) unless job.project.pull_in_progress?
-
- result = begin
- success = true
- job.run_task
- rescue
- success = false
- $!.message
- end
-
- job.update_attributes :results => new.ansi_escaped(result),
- :completed_at => Time.now,
- :success => success
+ # Project::PullRepo.perform(job.project.id) unless job.project.pull_in_progress?
+
+ job.run_task
+
+ job.update_attributes :completed_at => Time.now, :success => true
end
-
+
end
end
33 app/models/project.rb
View
@@ -1,4 +1,5 @@
require 'capistrano/cli'
+require 'capistrano_monkey/configuration/namespaces'
class Project < ActiveRecord::Base
@@ -18,9 +19,9 @@ class Project < ActiveRecord::Base
# Lets do some delegation so that we can access some common methods directly.
delegate :user_name, :repo_name, :cloned?, :capified?, :to => :repo
delegate :html_url, :description, :organization, :to => :github_data
-
+
default_scope where(:deleted_at => nil)
-
+
def self.deleted
self.unscoped.where 'deleted_at IS NOT NULL'
@@ -81,18 +82,14 @@ def repo
#
# Returns an Array of tasks.
def public_tasks
- @public_tasks ||= begin
- tasks.reject { |t| t.description.empty? || t.description =~ /^\[internal\]/ }
- end
+ @public_tasks ||= tasks.reject { |t| t.description.empty? || t.description =~ /^\[internal\]/ }
end
# The hidden and internal task list for this project's repository.
#
# Returns an Array of tasks.
def hidden_tasks
- @hidden_tasks ||= begin
- tasks.select { |t| t.description.empty? || t.description =~ /^\[internal\]/ }
- end
+ @hidden_tasks ||= tasks.select { |t| t.description.empty? || t.description =~ /^\[internal\]/ }
end
# Execute a capistrano command.
@@ -102,28 +99,30 @@ def cap(args = [])
unless Dir.exists?(repo.path)
raise Strano::RepositoryPathNotFound, "Path to local repository: #{repo.path} does not exist"
end
-
+
unless File.exists?(File.join(repo.path, 'Capfile'))
raise Strano::CapfileNotFound, "Capfile cannot be found in repository"
end
-
+
_cap = nil
FileUtils.chdir repo.path do
- _cap = Capistrano::CLI.parse(%W(-f Capfile -Xx -l STDOUT) + args).execute!
+ query = %W(-f Capfile -Xx -l STDOUT) + args
+ Rails.logger.debug " Capistrano command: #{query.join(' ')}"
+ _cap = Capistrano::CLI.parse(query).execute!
end
_cap
end
-
+
# Run git pull on the repo, as long as the last pull was more than 15 mins ago.
def pull
if !pull_in_progress? && !pulled_at.nil? && (Time.now - pulled_at) > 900
- Resque.enqueue PullRepo, id
+ PullRepo.perform_async id
end
end
-
+
# Run git pull on the repo regardless of when it was last pulled.
def pull!
- Resque.enqueue PullRepo, id unless pull_in_progress?
+ PullRepo.perform_async(id) unless pull_in_progress?
end
@@ -134,11 +133,11 @@ def tasks
end
def clone_repo
- Resque.enqueue CloneRepo, id
+ CloneRepo.perform_async id
end
def remove_repo
- Resque.enqueue RemoveRepo, id
+ RemoveRepo.perform_async id
end
def update_github_data
10 app/models/project/clone_repo.rb
View
@@ -1,13 +1,13 @@
class Project
class CloneRepo
- @queue = :project
-
- def self.perform(project_id)
+ include Sidekiq::Worker
+
+ def perform(project_id)
project = Project.find(project_id)
project.update_column :pull_in_progress, true
-
+
Strano::Repo.clone project.url
-
+
Project.update_all({:updated_at => Time.now,
:cloned_at => Time.now,
:pulled_at => Time.now,
12 app/models/project/pull_repo.rb
View
@@ -1,13 +1,17 @@
class Project
class PullRepo
- @queue = :project
-
+ include Sidekiq::Worker
+
def self.perform(project_id)
+ new.perform(project_id)
+ end
+
+ def perform(project_id)
project = Project.find(project_id)
project.update_column :pull_in_progress, true
-
+
Strano::Repo.pull project.url
-
+
Project.update_all({:updated_at => Time.now,
:pulled_at => Time.now,
:pull_in_progress => false},
6 app/models/project/remove_repo.rb
View
@@ -1,8 +1,8 @@
class Project
class RemoveRepo
- @queue = :project
-
- def self.perform(project_id)
+ include Sidekiq::Worker
+
+ def perform(project_id)
Strano::Repo.remove Project.unscoped.find(project_id).url
end
end
5 config/initializers/sidekiq.rb
View
@@ -0,0 +1,5 @@
+Sidekiq.configure_server do |config|
+ config.server_middleware do |chain|
+ chain.add Strano::SidekiqMiddleware::ExceptionHandler
+ end
+end
4 config/routes.rb
View
@@ -12,8 +12,8 @@
end
end
- require 'resque/server'
- mount Resque::Server.new, :at => "/resque"
+ require 'sidekiq/web'
+ mount Sidekiq::Web => '/sidekiq'
root :to => "dashboard#index"
5 db/migrate/20120322195922_change_verbosity_default.rb
View
@@ -0,0 +1,5 @@
+class ChangeVerbosityDefault < ActiveRecord::Migration
+ def change
+ change_column :jobs, :verbosity, :string, :default => "vvv"
+ end
+end
16 db/schema.rb
View
@@ -11,7 +11,7 @@
#
# It's strongly recommended to check this file into your version control system.
-ActiveRecord::Schema.define(:version => 20120213093204) do
+ActiveRecord::Schema.define(:version => 20120322195922) do
create_table "jobs", :force => true do |t|
t.string "task"
@@ -19,19 +19,19 @@
t.text "results"
t.integer "project_id"
t.integer "user_id"
- t.datetime "created_at", :null => false
- t.datetime "updated_at", :null => false
+ t.datetime "created_at"
+ t.datetime "updated_at"
t.datetime "completed_at"
t.string "stage"
t.datetime "deleted_at"
t.boolean "success", :default => true
- t.string "verbosity", :default => "v"
+ t.string "verbosity", :default => "vvv"
end
create_table "projects", :force => true do |t|
t.string "url"
- t.datetime "created_at", :null => false
- t.datetime "updated_at", :null => false
+ t.datetime "created_at"
+ t.datetime "updated_at"
t.text "github_data"
t.datetime "cloned_at"
t.datetime "deleted_at"
@@ -45,8 +45,8 @@
t.string "github_access_token"
t.text "github_data"
t.boolean "ssh_key_uploaded_to_github", :default => false
- t.datetime "created_at", :null => false
- t.datetime "updated_at", :null => false
+ t.datetime "created_at"
+ t.datetime "updated_at"
t.datetime "deleted_at"
t.string "token"
end
16 lib/capistrano_monkey/configuration.rb
View
@@ -0,0 +1,16 @@
+# Implementation of Capistrano::Configuration that uses a Strano::Logger as the
+# logger in order to log to the DB.
+module CapistranoMonkey
+ class Configuration < Capistrano::Configuration
+
+ # default callback to handle all output that
+ # the other callbacks not explicitly handle.
+ def self.default_io_proc
+ Proc.new do |ch, stream, out|
+ level = stream == :err ? :important : :info
+ ch[:options][:logger].send(level, out, "#{stream} :: #{ch[:server]}")
+ end
+ end
+
+ end
+end
26 lib/capistrano_monkey/configuration/namespaces.rb
View
@@ -0,0 +1,26 @@
+module Capistrano
+ class Configuration
+ module Namespaces
+ # Describe a new task. If a description is active (see #desc), it is added
+ # to the options under the <tt>:desc</tt> key. The new task is added to
+ # the namespace.
+ def task(name, options={}, &block)
+ name = name.to_sym
+ raise ArgumentError, "expected a block" unless block_given?
+
+ task_already_defined = tasks.key?(name)
+ if all_methods.any? { |m| m.to_sym == name } && !task_already_defined
+ thing = namespaces.key?(name) ? "namespace" : "method"
+
+ # Instead of raising an exception here if the task has already been
+ # defined (as Cap does by default), we ignore it, so Strano can load.
+ next_description(:reset)
+ return nil
+ end
+
+ task = TaskDefinition.new(name, self, {:desc => next_description(:reset)}.merge(options), &block)
+ define_task task
+ end
+ end
+ end
+end
2  lib/strano.rb
View
@@ -4,7 +4,7 @@
module Strano
extend Configuration
-
+
class RepositoryPathNotFound < StandardError; end
class CapfileNotFound < StandardError; end
end
41 lib/strano/cli.rb
View
@@ -0,0 +1,41 @@
+require 'capistrano/cli'
+module Strano
+ # Defines constants and methods related to configuration
+ class CLI < Capistrano::CLI
+
+ attr_accessor :logger
+
+
+ def self.parse(logger, args)
+ cli = new(args)
+ cli.parse_options!
+
+ @logger = logger
+ @logger.level = Strano::Logger::TRACE
+
+ cli.logger = logger
+ cli
+ end
+
+ # override in order to use DB logger
+ def instantiate_configuration(options={})
+ config = CapistranoMonkey::Configuration.new(options)
+ config.logger = logger
+ config
+ end
+
+ # override in order to use DB logger
+ def handle_error(error)
+ case error
+ when Net::SSH::AuthenticationFailed
+ logger.important "authentication failed for `#{error.message}'"
+ when Capistrano::Error
+ logger.important error.message
+ else
+ # we did not expect this error, so log the trace
+ logger.important error.message + "\n" + error.backtrace.join("\n")
+ end
+ end
+
+ end
+end
34 lib/strano/logger.rb
View
@@ -0,0 +1,34 @@
+# a logger for Capistrano::Configuration that logs to the database
+module Strano
+ class Logger < Capistrano::Logger
+ attr_accessor :job
+
+ def initialize(job)
+ raise ArgumentError, 'task is already completed and thus can not be logged to' if job.complete?
+ @job, @level = job, 0
+ end
+
+ def log(level, message, line_prefix=nil)
+ if level <= self.level
+ indent = "%*s" % [MAX_LEVEL, "*" * (MAX_LEVEL - level)]
+ (message.respond_to?(:lines) ? message.lines : message).each do |line|
+ if line_prefix
+ write_msg "#{indent} [#{line_prefix}] #{line.strip}\n"
+ else
+ write_msg "#{indent} #{line.strip}\n"
+ end
+ end
+ end
+ end
+
+ def close
+ # not needed here
+ end
+
+ # actual writing of a msg to the DB
+ def write_msg(msg)
+ job.update_attribute :results, (job.results || '') + msg unless msg.blank?
+ end
+
+ end
+end
18 lib/strano/sidekiq_middleware/exception_handler.rb
View
@@ -0,0 +1,18 @@
+module Strano
+ module SidekiqMiddleware
+ class ExceptionHandler
+ include Sidekiq::Util
+
+ def call(*args)
+ yield
+ rescue => ex
+ logger.warn ex
+ logger.warn args
+ logger.warn ex.backtrace.join("\n")
+
+ raise
+ end
+
+ end
+ end
+end
Please sign in to comment.
Something went wrong with that request. Please try again.