feat(source): Added support for S3

BREAKING CHANGE: `app['scm']` has been renamed to `app['source']`. This
only affects the Custom JSON files, so if you were using this block
there, you should change it. If you were using OpsWorks git configurator
(which is probably 99.99% true), this change wouldn't affect you.

Resolves #133

.kitchen.yml

@@ -91,8 +91,8 @@ suites:
             queues:
               - default
               - mailers
-  - name: thin_nginx_padrino_delayed_job
-    data_bags_path: "test/integration/data_bags/thin_nginx_padrino_delayed_job"
+  - name: s3_thin_nginx_padrino_delayed_job
+    data_bags_path: "test/integration/data_bags/s3_thin_nginx_padrino_delayed_job"
     run_list:
       - recipe[opsworks_ruby::setup]
       - recipe[opsworks_ruby::deploy]
@@ -142,7 +142,7 @@ suites:
             - copytruncate
             - sharedscripts
           logrotate_template_group: 'www-data'
-        scm:
+        source:
           generated_ssh_wrapper: '/var/tmp/my-generated-ssh-wrapper.sh'
         webserver:
           logrotate_options:

.travis.yml

@@ -25,7 +25,7 @@ env:
     - INSTANCE=default-ubuntu-1604
     - INSTANCE=all-options-ubuntu-1604
     - INSTANCE=unicorn-apache-hanami-resque-ubuntu-1604
-    - INSTANCE=thin-nginx-padrino-delayed-job-ubuntu-1604
+    - INSTANCE=s3-thin-nginx-padrino-delayed-job-ubuntu-1604
     - INSTANCE=nullified-ubuntu-1604
     - INSTANCE=maximum-override-ubuntu-1604
 

.yamllint

@@ -0,0 +1,5 @@
+extends: default
+
+rules:
+  line-length:
+    max: 120

Gemfile.lock

@@ -38,10 +38,10 @@ GEM
     celluloid-io (0.16.2)
       celluloid (>= 0.16.0)
       nio4r (>= 1.1.0)
-    chef (12.21.31)
+    chef (12.22.1)
       addressable
       bundler (>= 1.10)
-      chef-config (= 12.21.31)
+      chef-config (= 12.22.1)
       chef-zero (>= 4.8, < 13)
       diff-lcs (~> 1.2, >= 1.2.4)
       erubis (~> 2.7)
@@ -67,7 +67,7 @@ GEM
       specinfra (~> 2.10)
       syslog-logger (~> 1.6)
       uuidtools (~> 2.1.5)
-    chef-config (12.21.31)
+    chef-config (12.22.1)
       addressable
       fuzzyurl
       mixlib-config (~> 2.0)
@@ -82,7 +82,7 @@ GEM
       chef (>= 12.0)
       fauxhai (>= 3.6, < 5)
       rspec (~> 3.0)
-    childprocess (0.8.0)
+    childprocess (0.9.0)
       ffi (~> 1.0, >= 1.0.11)
     cleanroom (1.0.0)
     colorize (0.8.1)
@@ -115,7 +115,7 @@ GEM
     ffi (1.9.23)
     ffi-yajl (2.3.1)
       libyajl2 (~> 1.2)
-    foodcritic (12.3.0)
+    foodcritic (13.0.0)
       cucumber-core (>= 1.3)
       erubis
       ffi-yajl (~> 2.0)
@@ -207,11 +207,11 @@ GEM
       plist (~> 3.1)
       systemu (~> 2.6.4)
       wmi-lite (~> 1.0)
-    overcommit (0.43.0)
+    overcommit (0.44.0)
       childprocess (~> 0.6, >= 0.6.3)
       iniparse (~> 1.4)
     parallel (1.12.1)
-    parser (2.5.0.2)
+    parser (2.5.0.3)
       ast (~> 2.4.0)
     plist (3.4.0)
     polyglot (0.3.5)
@@ -261,9 +261,9 @@ GEM
     rspec_junit_formatter (0.2.3)
       builder (< 4)
       rspec-core (>= 2, < 4, != 2.12.0)
-    rubocop (0.52.1)
+    rubocop (0.53.0)
       parallel (~> 1.10)
-      parser (>= 2.4.0.2, < 3.0)
+      parser (>= 2.5)
       powerpack (~> 0.1)
       rainbow (>= 2.2.2, < 4.0)
       ruby-progressbar (~> 1.7)
@@ -377,4 +377,4 @@ DEPENDENCIES
   travis
 
 BUNDLED WITH
-   1.15.4
+   1.16.1

TESTING.md

@@ -3,7 +3,7 @@
 ## Unit Testing and Linting
 
 ```
-docker-compose run -e SKIP="AuthorName AuthorEmail" cookbook \
+docker-compose run --rm -e SKIP="AuthorName AuthorEmail" cookbook \
 bash -c "overcommit --sign && overcommit -r && rspec"
 ```
 

attributes/default.rb

@@ -35,10 +35,11 @@
 
 default['defaults']['database']['adapter'] = 'sqlite3'
 
-# scm
+# source
 ## common
 
-default['defaults']['scm']['remove_scm_files'] = true
+default['defaults']['source']['adapter'] = 'git'
+default['defaults']['source']['remove_scm_files'] = true
 
 # appserver
 ## common

docs/source/attributes.rst

@@ -190,48 +190,49 @@ database
   -  Any other key-value pair provided here, will be passed directly to
      the ``database.yml``
 
-scm
-~~~
+source
+~~~~~~
 
 | Those parameters can also be determined from OpsWorks application, and
-  usually
-| you don’t need to provide them here. Currently only ``git`` is
-  supported.
+  usually you don’t need to provide them here.
 
--  ``app['scm']['scm_provider']``
+-  ``app['source']['adapter']``
 
-  -  **Supported values:** ``git``
+  -  **Supported values:** ``git``, ``s3``
   -  **Default:** ``git``
-  -  SCM used by the cookbook to clone the repo.
+  -  Source used by the cookbook to fetch the application codebase.
+
+-  ``app['source']['url']``
+
+  -  Source code URL (repository URL for SCMs).
 
--  ``app['scm']['remove_scm_files']``
+git
+^^^
+
+-  ``app['source']['remove_scm_files']``
 
   -  **Supported values:** ``true``, ``false``
   -  **Default:** ``true``
   -  If set to true, all SCM leftovers (like ``.git``) will be removed.
 
--  ``app['scm']['repository']``
-
-  -  Repository URL
-
--  ``app['scm']['revision']``
+-  ``app['source']['revision']``
 
   -  Branch name/SHA1 of commit which should be use as a base of the
      deployment.
 
--  ``app['scm']['ssh_key']``
+-  ``app['source']['ssh_key']``
 
   -  A private SSH deploy key (the key itself, not the file name), used
      when fetching repositories via SSH.
 
--  ``app['scm']['ssh_wrapper']``
+-  ``app['source']['ssh_wrapper']``
 
   -  A wrapper script, which will be used by git when fetching repository
      via SSH. Essentially, a value of ``GIT_SSH`` environment variable.
      This cookbook provides one of those scripts for you, so you shouldn’t
      alter this variable unless you know what you’re doing.
 
--  ``app['scm']['generated_ssh_wrapper']``
+-  ``app['source']['generated_ssh_wrapper']``
 
   -  **Default:** ``/tmp/ssh-git-wrapper.sh``
   -  If the cookbook generates an SSH wrapper for you, this is where it
@@ -240,11 +241,26 @@ scm
      attribute allows you to override that location to a partition where
      execution of the generated shell script is allowed.
 
--  ``app['scm']['enabled_submodules']``
+-  ``app['source']['enable_submodules']``
 
   -  If set to ``true``, any submodules included in the repository, will
      also be fetched.
 
+s3
+^^
+
+| This source expects a packed project in one of the following formats:
+| ``bzip2``, ``compress``, ``gzip``, ``tar``, ``xz`` or ``zip``.
+| If you are using ubuntu, ``7zip`` is also supported.
+
+-  ``app['source']['user']``
+
+  -  ``AWS_ACCESS_KEY_ID`` with read access to the bucket.
+
+-  ``app['source']['password']``
+
+  -  ``AWS_SECRET_ACCESS_KEY`` for given ``AWS_ACCESS_KEY_ID``.
+
 framework
 ~~~~~~~~~
 

docs/source/support.rst

@@ -8,8 +8,9 @@ Supported Technologies
   - `PostgreSQL`_
   - `Sqlite3`_
 
-- SCM
+- Sources
 
+  - `Amazon S3`_
   - `Git`_
 
 - Framework
@@ -43,6 +44,7 @@ Worker
 .. _MySQL: https://www.mysql.com/
 .. _PostgreSQL: https://www.postgresql.org/
 .. _Sqlite3: https://www.sqlite.org/
+.. _Amazon S3: https://aws.amazon.com/s3/
 .. _Git: https://git-scm.com/
 .. _hanami.rb: http://hanamirb.org/
 .. _Padrino: http://padrinorb.com/

libraries/all.rb

@@ -1,9 +1,15 @@
 # frozen_string_literal: true
 
-libdir = File.expand_path('..', __FILE__)
+libdir = File.expand_path(__dir__)
 require File.join(libdir, 'core_ext')
 require File.join(libdir, 'helpers')
-Dir[File.join(libdir, 'drivers_dsl_*.rb')].each { |f| require f }
+require File.join(libdir, 'archive')
 Dir[File.join(libdir, 'drivers_base.rb')].each { |f| require f }
+Dir[File.join(libdir, 'drivers_dsl_*.rb')].each { |f| require f }
 Dir[File.join(libdir, 'drivers_*_base.rb')].each { |f| require f }
-Dir[File.join(libdir, '*.rb')].each { |f| require f }
+Dir[File.join(libdir, 'drivers_appserver_*.rb')].each { |f| require f }
+Dir[File.join(libdir, 'drivers_db_*.rb')].each { |f| require f }
+Dir[File.join(libdir, 'drivers_framework_*.rb')].each { |f| require f }
+Dir[File.join(libdir, 'drivers_source_*.rb')].each { |f| require f }
+Dir[File.join(libdir, 'drivers_webserver_*.rb')].each { |f| require f }
+Dir[File.join(libdir, 'drivers_worker_*.rb')].each { |f| require f }

libraries/archive.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module OpsworksRuby
+  class Archive
+    MIME_MAGIC_TABLES ||= [
+      ['application/gzip', [[0, "\037\213".b]]],
+      ['application/x-7z-compressed', [[0, "7z\274\257'\034".b]]],
+      ['application/x-bzip', [[0, 'BZh'.b]]],
+      ['application/x-compress', [[0, "\037\235".b]]],
+      ['application/x-tar', [[257, "ustar\000".b], [257, "ustar  \000".b]]],
+      ['application/x-xz', [[0, "\3757zXZ\000".b]]],
+      ['application/zip', [[0, "PK\003\004".b]]]
+    ].freeze
+
+    TYPES ||= {
+      'application/x-bzip-compressed-tar' => [
+        %w[tar.bz tar.bz2 tb2 tbz tbz2], %w[application/x-bzip], 'Tar archive (bzip-compressed)'
+      ],
+      'application/x-compressed-tar' => [
+        %w[tar.gz tgz], %w[application/gzip], 'Tar archive (gzip-compressed)'
+      ],
+      'application/x-tarz' => [
+        %w[tar.z taz], %w[application/x-compress], 'Tar archive (compressed)'
+      ],
+      'application/x-xz-compressed-tar' => [
+        %w[tar.xz txz], %w[application/x-xz], 'Tar archive (XZ-compressed)'
+      ]
+    }.freeze
+
+    COMMANDS ||= {
+      'application/gzip' => 'gunzip -c %s > %s',
+      'application/x-7z-compressed' => '7z x -t7z %s -o%s',
+      'application/x-bzip' => 'bzip2 -ckd %s > %s',
+      'application/x-bzip-compressed-tar' => 'tar -xjf %s -C %s',
+      'application/x-compress' => 'zcat %s > %s',
+      'application/x-compressed-tar' => 'tar -xzf %s -C %s',
+      'application/x-tar' => 'tar -xf %s -C %s',
+      'application/x-tarz' => 'zcat %s | tar -xf -C %s -',
+      'application/x-xz' => 'xz --decompress --stdout %s > %s',
+      'application/x-xz-compressed-tar' => 'tar -xJf %s -C %s',
+      'application/zip' => 'unzip %s -d %s'
+    }.freeze
+
+    def initialize(source_path)
+      @source_path = source_path.to_s
+    end
+
+    def uncompress(destination)
+      cmd = Mixlib::ShellOut.new(extract_command(destination))
+      cmd.run_command
+    end
+
+    def extract_command(destination)
+      format(COMMANDS[file_mime_type], @source_path, destination)
+    end
+
+    def file_mime_type
+      mime_type = File.open(@source_path) do |io|
+        io.binmode
+        io.set_encoding(Encoding::BINARY)
+
+        find_mime_matches(io)&.first
+      end
+
+      detect_tar_mime_type(mime_type)
+    end
+
+    def find_mime_matches(io)
+      MIME_MAGIC_TABLES.detect do |item|
+        item[1].any? do |bytes|
+          io.rewind
+          io.read(bytes[0])
+          io.read(bytes[1].size) == bytes[1]
+        end
+      end
+    end
+
+    def detect_tar_mime_type(mime_type)
+      types_with_matching_extensions = TYPES.select do |_mime_name, types|
+        types[1].include?(mime_type)
+      end
+
+      types_with_matching_extensions.detect do |types|
+        types[1][0].any? { |extension| @source_path.downcase.end_with?(extension) }
+      end&.first || mime_type
+    end
+  end
+end

libraries/core_ext.rb

@@ -1,19 +1,19 @@
 # frozen_string_literal: true
 
 class Object
-  def try(*a, &b)
-    try!(*a, &b) if a.empty? || respond_to?(a.first)
+  def try(*methods, &block)
+    try!(*methods, &block) if methods.empty? || respond_to?(methods.first)
   end
 
-  def try!(*a, &b)
-    if a.empty? && block_given?
-      if b.arity.zero?
-        instance_eval(&b)
+  def try!(*methods, &block)
+    if methods.empty? && block_given?
+      if block.arity.zero?
+        instance_eval(&block)
       else
         yield self
       end
     else
-      public_send(*a, &b)
+      public_send(*methods, &block)
     end
   end
 

libraries/drivers_dsl_logrotate.rb

@@ -12,7 +12,6 @@ def log_paths(*log_paths)
           end
         end
       end
-      # rubocop:enable Metrics/MethodLength
 
       def log_paths
         self.class.log_paths.presence ||

libraries/drivers_dsl_output.rb

@@ -21,7 +21,7 @@ def out
       end
 
       def handle_output(unfiltered)
-        if output[:filter] && output[:filter].is_a?(Array)
+        if output[:filter]&.is_a?(Array)
           unfiltered.select { |k, _v| output[:filter].include?(k.to_sym) }
         else
           unfiltered