Added workers support. Resolves #18

README.md

@@ -32,7 +32,7 @@ be added soon. However, other Debian family distributions are assumed to work.
 
 Attributes format follows the guidelines of old Chef 11.x based OpsWorks stack.
 So all of them, need to be placed under `node['deploy'][<application_shortname>]`.
-Attributes (and whole logic of this cookbook) are divided to five sections.
+Attributes (and whole logic of this cookbook) are divided to six sections.
 Following convention is used: `app == node['deploy'][<application_shortname>]`
 so for example `app['framework']['adapter']` actually means
 `node['deploy'][<application_shortname>]['framework']['adapter']`.
@@ -120,6 +120,11 @@ Currently only `Rails` is supported.
 Configuration parameters for the ruby application server. Currently only
 `Unicorn` is supported.
 
+* `app['appserver']['adapter']`
+  * **Default:** `unicorn`
+  * **Supported values:** `unicorn`
+  * Server on the application side, which will receive requests from webserver
+    in front.
 * [`app['appserver']['accept_filter']`](https://unicorn.bogomips.org/Unicorn/Configurator.html#method-i-listen)
   * **Default:** `httpready`
 * [`app['appserver']['backlog']`](https://unicorn.bogomips.org/Unicorn/Configurator.html#method-i-listen)
@@ -142,11 +147,37 @@ Configuration parameters for the ruby application server. Currently only
 * [`app['appserver']['worker_processes']`](https://unicorn.bogomips.org/TUNING.html)
   * **Default:** `4`
 
+### worker
+
+Configuration for ruby workers. Currenty `Null` (no worker) and `Sidekiq`
+are supported. Every worker is covered by `monitd` daemon out-of-the-box.
+
+* `app['worker']['adapter']`
+  * **Default:** `null`
+  * **Supported values:** `null`, `sidekiq`
+  * Worker used to perform background tasks. `null` means no worker enabled.
+* `app['worker']['process_count']`
+  * ** Default:** `2`
+  * How many separate worker processes will be launched.
+* `app['worker']['syslog']`
+  * **Default:** `true`
+  * **Supported values:** `true`, `false`
+  * Log worker output to syslog?
+* `app['worker']['config']`
+  * Configuration parameters which will be directly passed to the worker.
+    For example, for `sidekiq` they will be serialized to
+    [`sidekiq.yml` config file](https://github.com/mperham/sidekiq/wiki/Advanced-Options#the-sidekiq-configuration-file).
+
 ### webserver
 
 Webserver configuration. Proxy passing to application is handled out-of-the-box.
 Currently only nginx is supported.
 
+* `app['webserver']['adapter']`
+  * **Default:** `nginx`
+  * **Supported values:** `nginx`
+  * Webserver in front of the instance. It runs on port 80,
+    and receives all requests from Load Balancer/Internet.
 * `app['webserver']['build_type']`
   * **Supported values:** `default` or `source`
   * **Default:** `default`

attributes/default.rb

@@ -69,3 +69,14 @@
   'bundle exec rake db:version > /dev/null 2>&1 && bundle exec rake db:migrate || bundle exec rake db:setup'
 default['defaults']['framework']['assets_precompile'] = true
 default['defaults']['framework']['assets_precompilation_command'] = 'bundle exec rake assets:precompile'
+
+# worker
+## common
+
+default['defaults']['worker']['adapter'] = 'null'
+default['defaults']['worker']['process_count'] = 2
+default['defaults']['worker']['syslog'] = true
+
+## sidekiq
+
+default['defaults']['worker']['config'] = { 'concurency' => 5, 'verbose' => false, 'queues' => ['default'] }

libraries/drivers_appserver_unicorn.rb

@@ -15,7 +15,8 @@ def configure(context)
       end
 
       def after_deploy(context)
-        manual_action(context, :restart)
+        manual_action(context, :stop)
+        manual_action(context, :start)
       end
       alias after_undeploy after_deploy
 

libraries/drivers_dsl_output.rb

@@ -16,7 +16,9 @@ def output
       end
 
       def handle_output(out)
-        out = out.select { |k, _v| output[:filter].include?(k.to_sym) } if output[:filter].present?
+        if output[:filter] && output[:filter].is_a?(Array)
+          out = out.select { |k, _v| output[:filter].include?(k.to_sym) }
+        end
         out
       end
     end

libraries/drivers_worker_base.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+module Drivers
+  module Worker
+    class Base < Drivers::Base
+      include Drivers::Dsl::Output
+
+      def out
+        handle_output(raw_out)
+      end
+
+      def raw_out
+        node['defaults']['worker'].merge(
+          node['deploy'][app['shortname']]['worker'] || {}
+        ).symbolize_keys
+      end
+
+      def validate_app_engine
+      end
+    end
+  end
+end

libraries/drivers_worker_factory.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+module Drivers
+  module Worker
+    class Factory
+      def self.build(app, node, options = {})
+        engine = detect_engine(app, node, options)
+        raise StandardError, 'There is no supported Worker driver for given configuration.' if engine.blank?
+        engine.new(app, node, options)
+      end
+
+      def self.detect_engine(app, node, _options)
+        Drivers::Worker::Base.descendants.detect do |worker_driver|
+          worker_driver.allowed_engines.include?(
+            node['deploy'][app['shortname']]['worker'].try(:[], 'adapter') ||
+            node['defaults']['worker']['adapter']
+          )
+        end
+      end
+    end
+  end
+end

libraries/drivers_worker_null.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+module Drivers
+  module Worker
+    class Null < Drivers::Worker::Base
+      adapter :null
+      allowed_engines :null
+      output filter: []
+    end
+  end
+end

libraries/drivers_worker_sidekiq.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+module Drivers
+  module Worker
+    class Sidekiq < Drivers::Worker::Base
+      adapter :sidekiq
+      allowed_engines :sidekiq
+      output filter: [:config, :process_count, :require, :syslog]
+
+      def configure(context)
+        add_sidekiq_config(context)
+        add_sidekiq_monit(context)
+      end
+
+      def after_deploy(context)
+        context.execute 'monit reload'
+        (1..process_count).each do |process_number|
+          context.execute "monit restart sidekiq_#{app['shortname']}-#{process_number}" do
+            retries 3
+          end
+        end
+      end
+      alias after_undeploy after_deploy
+
+      private
+
+      def add_sidekiq_config(context)
+        deploy_to = deploy_dir(app)
+        config = configuration
+
+        (1..process_count).each do |process_number|
+          context.template File.join(deploy_to, File.join('shared', 'config', "sidekiq_#{process_number}.yml")) do
+            owner node['deployer']['user']
+            group www_group
+            source 'sidekiq.conf.yml.erb'
+            variables config: config
+          end
+        end
+      end
+
+      def add_sidekiq_monit(context)
+        app_shortname = app['shortname']
+        deploy_to = deploy_dir(app)
+        output = out
+        env = environment
+
+        context.template File.join('/', 'etc', 'monit', 'conf.d', "sidekiq_#{app_shortname}.monitrc") do
+          mode '0640'
+          source 'sidekiq.monitrc.erb'
+          variables application: app_shortname, out: output, deploy_to: deploy_to, environment: env
+        end
+      end
+
+      def process_count
+        [out[:process_count].to_i, 1].max
+      end
+
+      def environment
+        framework = Drivers::Framework::Factory.build(app, node)
+        app['environment'].merge(framework.out[:deploy_environment] || {})
+      end
+
+      def configuration
+        JSON.parse(out[:config].stringify_keys.to_json)
+      end
+    end
+  end
+end

recipes/configure.rb

@@ -25,6 +25,8 @@
   framework.configure(self)
   appserver = Drivers::Appserver::Factory.build(application, node)
   appserver.configure(self)
+  worker = Drivers::Worker::Factory.build(application, node)
+  worker.configure(self)
   webserver = Drivers::Webserver::Factory.build(application, node)
   webserver.configure(self)
 end

recipes/deploy.rb

@@ -13,11 +13,13 @@
   scm = Drivers::Scm::Factory.build(application, node)
   framework = Drivers::Framework::Factory.build(application, node)
   appserver = Drivers::Appserver::Factory.build(application, node)
+  worker = Drivers::Worker::Factory.build(application, node)
   webserver = Drivers::Webserver::Factory.build(application, node)
 
   scm.before_deploy(self)
   framework.before_deploy(self)
   appserver.before_deploy(self)
+  worker.before_deploy(self)
   webserver.before_deploy(self)
 
   deploy application['shortname'] do
@@ -76,6 +78,7 @@
   scm.after_deploy(self)
   framework.after_deploy(self)
   appserver.after_deploy(self)
+  worker.after_deploy(self)
   webserver.after_deploy(self)
 
   every_enabled_rds do |rds|

recipes/setup.rb

@@ -27,6 +27,8 @@
   framework.setup(self)
   appserver = Drivers::Appserver::Factory.build(application, node)
   appserver.setup(self)
+  worker = Drivers::Worker::Factory.build(application, node)
+  worker.setup(self)
   webserver = Drivers::Webserver::Factory.build(application, node)
   webserver.setup(self)
 end

recipes/shutdown.rb

@@ -18,6 +18,8 @@
   framework.shutdown(self)
   appserver = Drivers::Appserver::Factory.build(application, node)
   appserver.shutdown(self)
+  worker = Drivers::Worker::Factory.build(application, node)
+  worker.shutdown(self)
   webserver = Drivers::Webserver::Factory.build(application, node)
   webserver.shutdown(self)
 end

recipes/undeploy.rb

@@ -11,11 +11,13 @@
   scm = Drivers::Scm::Factory.build(application, node)
   framework = Drivers::Framework::Factory.build(application, node)
   appserver = Drivers::Appserver::Factory.build(application, node)
+  worker = Drivers::Worker::Factory.build(application, node)
   webserver = Drivers::Webserver::Factory.build(application, node)
 
   scm.before_undeploy(self)
   framework.before_undeploy(self)
   appserver.before_undeploy(self)
+  worker.before_undeploy(self)
   webserver.before_undeploy(self)
 
   deploy application['shortname'] do
@@ -37,6 +39,7 @@
   scm.after_undeploy(self)
   framework.after_undeploy(self)
   appserver.after_undeploy(self)
+  worker.after_undeploy(self)
   webserver.after_undeploy(self)
 
   every_enabled_rds do |rds|

spec/fixtures/node.rb

@@ -43,6 +43,10 @@ def node(override = {})
           adapter: 'rails',
           migrate: false
         },
+        worker: {
+          adapter: 'sidekiq',
+          require: 'lorem_ipsum.rb'
+        },
         create_dirs_before_symlink: %(../shared/test),
         purge_before_symlink: %w(public/test),
         symlinks: { 'test' => 'public/test' }
@@ -73,6 +77,12 @@ def node(override = {})
         migration_command: 'rake db:migrate',
         assets_precompile: true,
         assets_precompilation_command: 'bundle exec rake assets:precompile'
+      },
+      worker: {
+        adapter: 'null',
+        process_count: 2,
+        syslog: true,
+        config: { 'concurency' => 5, 'verbose' => false, 'queues' => ['default'] }
       }
     }
   }.merge(override)

spec/unit/libraries/drivers_worker_factory_spec.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+require 'spec_helper'
+
+describe Drivers::Worker::Factory do
+  it 'raises error when unknown adapter is present' do
+    expect do
+      described_class.build(
+        aws_opsworks_app,
+        'deploy' => { aws_opsworks_app['shortname'] => { 'worker' => { 'adapter' => 'rq' } } }
+      )
+    end.to raise_error StandardError, 'There is no supported Worker driver for given configuration.'
+  end
+
+  it 'returns a Sidekiq class' do
+    worker = described_class.build(aws_opsworks_app, node)
+    expect(worker).to be_instance_of(Drivers::Worker::Sidekiq)
+  end
+end

spec/unit/libraries/drivers_worker_null_spec.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+require 'spec_helper'
+
+describe Drivers::Worker::Null do
+  it 'receives and exposes app and node' do
+    driver = described_class.new(aws_opsworks_app, node)
+
+    expect(driver.app).to eq aws_opsworks_app
+    expect(driver.node).to eq node
+    expect(driver.options).to eq({})
+  end
+
+  it 'returns proper out data' do
+    expect(described_class.new(aws_opsworks_app, node).out).to eq({})
+  end
+end

spec/unit/libraries/drivers_worker_sidekiq_spec.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+require 'spec_helper'
+
+describe Drivers::Worker::Sidekiq do
+  it 'receives and exposes app and node' do
+    driver = described_class.new(aws_opsworks_app, node)
+
+    expect(driver.app).to eq aws_opsworks_app
+    expect(driver.node).to eq node
+    expect(driver.options).to eq({})
+  end
+
+  it 'returns proper out data' do
+    expect(described_class.new(aws_opsworks_app, node).out).to eq(
+      process_count: 2,
+      syslog: true,
+      require: 'lorem_ipsum.rb',
+      config: {
+        'concurency' => 5,
+        'verbose' => false,
+        'queues' => ['default']
+      }
+    )
+  end
+end