Skip to content
This repository has been archived by the owner on Dec 31, 2022. It is now read-only.

Commit

Permalink
feat(worker): "delayed_job" support added
Browse files Browse the repository at this point in the history
Resolves #42
  • Loading branch information
ajgon committed Sep 15, 2016
1 parent 0e72200 commit 7235720
Show file tree
Hide file tree
Showing 7 changed files with 278 additions and 5 deletions.
12 changes: 9 additions & 3 deletions README.md
Expand Up @@ -37,8 +37,9 @@ then [add recipes to the corresponding OpsWorks actions](#recipes).
* nginx
* Worker
* Null (no worker)
* sidekiq
* delayed_job
* resque
* sidekiq

## Requirements

Expand Down Expand Up @@ -322,12 +323,12 @@ apply here.)

### worker

Configuration for ruby workers. Currenty `Resque` and `Sidekiq`
Configuration for ruby workers. Currenty `delayed_job`, `Resque` and `Sidekiq`
are supported. Every worker is covered by `monitd` daemon out-of-the-box.

* `app['worker']['adapter']`
* **Default:** `null`
* **Supported values:** `null`, `resque`, `sidekiq`
* **Supported values:** `null`, `delayed_job`, `resque`, `sidekiq`
* Worker used to perform background tasks. `null` means no worker enabled.
* `app['worker']['process_count']`
* ** Default:** `2`
Expand All @@ -344,6 +345,11 @@ are supported. Every worker is covered by `monitd` daemon out-of-the-box.
For example, for `sidekiq` they will be serialized to
[`sidekiq.yml` config file](https://github.com/mperham/sidekiq/wiki/Advanced-Options#the-sidekiq-configuration-file).

#### delayed_job

* `app['worker']['queues']`
* Array of queues which should be processed by delayed_job

#### resque

* `app['worker']['workers']`
Expand Down
57 changes: 57 additions & 0 deletions libraries/drivers_worker_delayed_job.rb
@@ -0,0 +1,57 @@
# frozen_string_literal: true
module Drivers
module Worker
class DelayedJob < Drivers::Worker::Base
adapter :delayed_job
allowed_engines :delayed_job
output filter: [:process_count, :syslog, :queues]

def raw_out
output = node['defaults']['worker'].merge(
node['deploy'][app['shortname']]['worker'] || {}
).symbolize_keys
output[:queues] = node['deploy'][app['shortname']]['worker']['queues'] || ''
output
end

def configure(context)
add_delayed_job_monit(context)
end

def after_deploy(context)
(1..process_count).each do |process_number|
context.execute "monit restart delayed_job_#{app['shortname']}-#{process_number}" do
retries 3
end
end
end
alias after_undeploy after_deploy

private

def add_delayed_job_monit(context)
app_shortname = app['shortname']
deploy_to = deploy_dir(app)
output = out
env = environment

context.template File.join(node['monit']['basedir'], "delayed_job_#{app_shortname}.monitrc") do
mode '0640'
source 'delayed_job.monitrc.erb'
variables application: app_shortname, out: output, deploy_to: deploy_to, environment: env
end

context.execute 'monit reload'
end

def process_count
[out[:process_count].to_i, 1].max
end

def environment
framework = Drivers::Framework::Factory.build(app, node)
app['environment'].merge(framework.out[:deploy_environment] || {})
end
end
end
end
20 changes: 20 additions & 0 deletions spec/unit/libraries/drivers_worker_delayed_job_spec.rb
@@ -0,0 +1,20 @@
# frozen_string_literal: true
require 'spec_helper'

describe Drivers::Worker::DelayedJob do
it 'receives and exposes app and node' do
driver = described_class.new(aws_opsworks_app, node)

expect(driver.app).to eq aws_opsworks_app
expect(driver.node).to eq node
expect(driver.options).to eq({})
end

it 'returns proper out data' do
expect(described_class.new(aws_opsworks_app, node).out).to eq(
process_count: 2,
syslog: true,
queues: 'test_queue'
)
end
end
113 changes: 111 additions & 2 deletions spec/unit/recipes/configure_spec.rb
Expand Up @@ -579,15 +579,16 @@
end
end

context 'Sqlite3 + Thin' do
context 'Sqlite3 + Thin + delayed_job' do
let(:dummy_node) do
node(
deploy: {
dummy_project: {
database: { adapter: 'sqlite3' },
environment: 'staging',
appserver: node['deploy']['dummy_project']['appserver'].merge('adapter' => 'thin'),
webserver: node['deploy']['dummy_project']['webserver']
webserver: node['deploy']['dummy_project']['webserver'],
worker: node['deploy']['dummy_project']['worker'].merge('adapter' => 'delayed_job')
}
}
)
Expand All @@ -598,6 +599,12 @@
solo_node.set['nginx'] = node['nginx']
end.converge(described_recipe)
end
let(:chef_run_rhel) do
ChefSpec::SoloRunner.new(platform: 'amazon', version: '2015.03') do |solo_node|
solo_node.set['deploy'] = dummy_node['deploy']
solo_node.set['nginx'] = node['nginx']
end.converge(described_recipe)
end

before do
stub_search(:aws_opsworks_app, '*:*').and_return([aws_opsworks_app(data_sources: [])])
Expand Down Expand Up @@ -702,6 +709,108 @@
.with_content('extra_config_ssl {}')
expect(chef_run).to create_link("/etc/nginx/sites-enabled/#{aws_opsworks_app['shortname']}.conf")
end

it 'creates delayed_job.monitrc conf' do
expect(chef_run).to create_template("/etc/monit/conf.d/delayed_job_#{aws_opsworks_app['shortname']}.monitrc")
expect(chef_run)
.to render_file("/etc/monit/conf.d/delayed_job_#{aws_opsworks_app['shortname']}.monitrc")
.with_content('check process delayed_job_dummy_project-1')
expect(chef_run)
.to render_file("/etc/monit/conf.d/delayed_job_#{aws_opsworks_app['shortname']}.monitrc")
.with_content('with pidfile /srv/www/dummy_project/shared/pids/delayed_job.0.pid')
expect(chef_run)
.to render_file("/etc/monit/conf.d/delayed_job_#{aws_opsworks_app['shortname']}.monitrc")
.with_content(
'start program = "/bin/su - deploy -c \'cd /srv/www/dummy_project/current && ENV_VAR1="test" ' \
'ENV_VAR2="some data" RAILS_ENV="staging" bin/delayed_job start ' \
'--pid-dir=/srv/www/dummy_project/shared/pids/ -i 0 --queues=test_queue 2>&1 ' \
'| logger -t delayed_job-dummy_project-1\'" with timeout 90 seconds'
)
expect(chef_run)
.to render_file("/etc/monit/conf.d/delayed_job_#{aws_opsworks_app['shortname']}.monitrc")
.with_content(
'stop program = "/bin/su - deploy -c \'cd /srv/www/dummy_project/current && ENV_VAR1="test" ' \
'ENV_VAR2="some data" RAILS_ENV="staging" bin/delayed_job stop ' \
'--pid-dir=/srv/www/dummy_project/shared/pids/ -i 0\'" with timeout 90 seconds'
)
expect(chef_run)
.to render_file("/etc/monit/conf.d/delayed_job_#{aws_opsworks_app['shortname']}.monitrc")
.with_content('check process delayed_job_dummy_project-2')
expect(chef_run)
.to render_file("/etc/monit/conf.d/delayed_job_#{aws_opsworks_app['shortname']}.monitrc")
.with_content('with pidfile /srv/www/dummy_project/shared/pids/delayed_job.1.pid')
expect(chef_run)
.to render_file("/etc/monit/conf.d/delayed_job_#{aws_opsworks_app['shortname']}.monitrc")
.with_content(
'start program = "/bin/su - deploy -c \'cd /srv/www/dummy_project/current && ENV_VAR1="test" ' \
'ENV_VAR2="some data" RAILS_ENV="staging" bin/delayed_job start ' \
'--pid-dir=/srv/www/dummy_project/shared/pids/ -i 1 --queues=test_queue 2>&1 ' \
'| logger -t delayed_job-dummy_project-2\'" with timeout 90 seconds'
)
expect(chef_run)
.to render_file("/etc/monit/conf.d/delayed_job_#{aws_opsworks_app['shortname']}.monitrc")
.with_content(
'stop program = "/bin/su - deploy -c \'cd /srv/www/dummy_project/current && ENV_VAR1="test" ' \
'ENV_VAR2="some data" RAILS_ENV="staging" bin/delayed_job stop ' \
'--pid-dir=/srv/www/dummy_project/shared/pids/ -i 1\'" with timeout 90 seconds'
)
expect(chef_run)
.to render_file("/etc/monit/conf.d/delayed_job_#{aws_opsworks_app['shortname']}.monitrc")
.with_content('group delayed_job_dummy_project_group')
expect(chef_run).to run_execute('monit reload')
end

context 'rhel' do
it 'creates delayed_job.monitrc conf' do
expect(chef_run_rhel).to create_template("/etc/monit.d/delayed_job_#{aws_opsworks_app['shortname']}.monitrc")
expect(chef_run_rhel)
.to render_file("/etc/monit.d/delayed_job_#{aws_opsworks_app['shortname']}.monitrc")
.with_content('check process delayed_job_dummy_project-1')
expect(chef_run_rhel)
.to render_file("/etc/monit.d/delayed_job_#{aws_opsworks_app['shortname']}.monitrc")
.with_content('with pidfile /srv/www/dummy_project/shared/pids/delayed_job.0.pid')
expect(chef_run_rhel)
.to render_file("/etc/monit.d/delayed_job_#{aws_opsworks_app['shortname']}.monitrc")
.with_content(
'start program = "/bin/su - deploy -c \'cd /srv/www/dummy_project/current && ENV_VAR1="test" ' \
'ENV_VAR2="some data" RAILS_ENV="staging" bin/delayed_job start ' \
'--pid-dir=/srv/www/dummy_project/shared/pids/ -i 0 --queues=test_queue 2>&1 ' \
'| logger -t delayed_job-dummy_project-1\'" with timeout 90 seconds'
)
expect(chef_run_rhel)
.to render_file("/etc/monit.d/delayed_job_#{aws_opsworks_app['shortname']}.monitrc")
.with_content(
'stop program = "/bin/su - deploy -c \'cd /srv/www/dummy_project/current && ENV_VAR1="test" ' \
'ENV_VAR2="some data" RAILS_ENV="staging" bin/delayed_job stop ' \
'--pid-dir=/srv/www/dummy_project/shared/pids/ -i 0\'" with timeout 90 seconds'
)
expect(chef_run_rhel)
.to render_file("/etc/monit.d/delayed_job_#{aws_opsworks_app['shortname']}.monitrc")
.with_content('check process delayed_job_dummy_project-2')
expect(chef_run_rhel)
.to render_file("/etc/monit.d/delayed_job_#{aws_opsworks_app['shortname']}.monitrc")
.with_content('with pidfile /srv/www/dummy_project/shared/pids/delayed_job.1.pid')
expect(chef_run_rhel)
.to render_file("/etc/monit.d/delayed_job_#{aws_opsworks_app['shortname']}.monitrc")
.with_content(
'start program = "/bin/su - deploy -c \'cd /srv/www/dummy_project/current && ENV_VAR1="test" ' \
'ENV_VAR2="some data" RAILS_ENV="staging" bin/delayed_job start ' \
'--pid-dir=/srv/www/dummy_project/shared/pids/ -i 1 --queues=test_queue 2>&1 ' \
'| logger -t delayed_job-dummy_project-2\'" with timeout 90 seconds'
)
expect(chef_run_rhel)
.to render_file("/etc/monit.d/delayed_job_#{aws_opsworks_app['shortname']}.monitrc")
.with_content(
'stop program = "/bin/su - deploy -c \'cd /srv/www/dummy_project/current && ENV_VAR1="test" ' \
'ENV_VAR2="some data" RAILS_ENV="staging" bin/delayed_job stop ' \
'--pid-dir=/srv/www/dummy_project/shared/pids/ -i 1\'" with timeout 90 seconds'
)
expect(chef_run_rhel)
.to render_file("/etc/monit.d/delayed_job_#{aws_opsworks_app['shortname']}.monitrc")
.with_content('group delayed_job_dummy_project_group')
expect(chef_run_rhel).to run_execute('monit reload')
end
end
end

context 'No RDS (Database defined in node)' do
Expand Down
34 changes: 34 additions & 0 deletions spec/unit/recipes/deploy_spec.rb
Expand Up @@ -111,6 +111,40 @@
end
end

context 'Thin + delayed_job' do
let(:chef_run) do
ChefSpec::SoloRunner.new(platform: 'ubuntu', version: '14.04') do |solo_node|
deploy = node['deploy']
deploy['dummy_project']['appserver']['adapter'] = 'thin'
deploy['dummy_project']['worker']['adapter'] = 'delayed_job'
solo_node.set['deploy'] = deploy
end.converge(described_recipe)
end
let(:chef_run_rhel) do
ChefSpec::SoloRunner.new(platform: 'amazon', version: '2016.03') do |solo_node|
deploy = node['deploy']
deploy['dummy_project']['appserver']['adapter'] = 'thin'
deploy['dummy_project']['worker']['adapter'] = 'delayed_job'
solo_node.set['deploy'] = deploy
end.converge(described_recipe)
end

it 'performs a deploy on debian' do
expect(chef_run).to run_execute('stop thin')
expect(chef_run).to run_execute('start thin')
end

it 'performs a deploy on rhel' do
expect(chef_run_rhel).to run_execute('stop thin')
expect(chef_run_rhel).to run_execute('start thin')
end

it 'restarts delayed_jobs via monit' do
expect(chef_run).to run_execute("monit restart delayed_job_#{aws_opsworks_app['shortname']}-1")
expect(chef_run).to run_execute("monit restart delayed_job_#{aws_opsworks_app['shortname']}-2")
end
end

it 'empty node[\'deploy\']' do
chef_run = ChefSpec::SoloRunner.new(platform: 'ubuntu', version: '14.04') do |solo_node|
solo_node.set['lsb'] = node['lsb']
Expand Down
34 changes: 34 additions & 0 deletions spec/unit/recipes/undeploy_spec.rb
Expand Up @@ -81,6 +81,40 @@
end
end

context 'Thin + delayed_job' do
let(:chef_run) do
ChefSpec::SoloRunner.new(platform: 'ubuntu', version: '14.04') do |solo_node|
deploy = node['deploy']
deploy['dummy_project']['appserver']['adapter'] = 'thin'
deploy['dummy_project']['worker']['adapter'] = 'delayed_job'
solo_node.set['deploy'] = deploy
end.converge(described_recipe)
end
let(:chef_run_rhel) do
ChefSpec::SoloRunner.new(platform: 'amazon', version: '2016.03') do |solo_node|
deploy = node['deploy']
deploy['dummy_project']['appserver']['adapter'] = 'thin'
deploy['dummy_project']['worker']['adapter'] = 'delayed_job'
solo_node.set['deploy'] = deploy
end.converge(described_recipe)
end

it 'performs a rollback on debian' do
expect(chef_run).to run_execute('stop thin')
expect(chef_run).to run_execute('start thin')
end

it 'performs a rollback on rhel' do
expect(chef_run_rhel).to run_execute('stop thin')
expect(chef_run_rhel).to run_execute('start thin')
end

it 'restarts delayed_jobs via monit' do
expect(chef_run).to run_execute("monit restart delayed_job_#{aws_opsworks_app['shortname']}-1")
expect(chef_run).to run_execute("monit restart delayed_job_#{aws_opsworks_app['shortname']}-2")
end
end

it 'empty node[\'deploy\']' do
chef_run = ChefSpec::SoloRunner.new(platform: 'ubuntu', version: '14.04') do |solo_node|
solo_node.set['lsb'] = node['lsb']
Expand Down
13 changes: 13 additions & 0 deletions templates/default/delayed_job.monitrc.erb
@@ -0,0 +1,13 @@
<% ([@out[:process_count].to_i || 1].max).times do |n| %>
<% identifier = "#{@application}-#{n+1}" %>
<% pid_dir = "#{@deploy_to}/shared/pids/" %>
<% syslog = !!@out[:syslog] ? "2>&1 | logger -t delayed_job-#{identifier}" : '' %>
<% queues = @out[:queues].present? ? '--queues=' + Array.wrap(@out[:queues]).join(',') : '' %>

check process delayed_job_<%= identifier.to_s %>
with pidfile <%= pid_dir.to_s %>delayed_job.<%= n %>.pid
start program = "/bin/su - <%= node['deployer']['user'] %> -c 'cd <%= File.join(@deploy_to, 'current') %> && <%= @environment.map {|k,v| "#{k}=\"#{v}\""}.join(' ') %> bin/delayed_job start --pid-dir=<%= pid_dir.to_s %> -i <%= n %> <%= queues.to_s %> <%= syslog.to_s %>'" with timeout 90 seconds
stop program = "/bin/su - <%= node['deployer']['user'] %> -c 'cd <%= File.join(@deploy_to, 'current') %> && <%= @environment.map {|k,v| "#{k}=\"#{v}\""}.join(' ') %> bin/delayed_job stop --pid-dir=<%= pid_dir.to_s %> -i <%= n %>'" with timeout 90 seconds
group delayed_job_<%= @application.to_s %>_group

<% end %>

0 comments on commit 7235720

Please sign in to comment.