Merge branch 'release/0.2.2'

.gitignore

@@ -15,3 +15,6 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+
+# Vagrant
+.vagrant

CHANGELOG

@@ -1,3 +1,10 @@
+Version 0.2.2 (2015-04-08)
+--------------------------
+Now tolerates nil passed to trail_slash (#38)
+Added License button to README (#34)
+Added Vagrant quickstart to README (#36)
+Added dedicated Vagrant setup (#35)
+
 Version 0.2.1 (2014-05-20)
 --------------------------
 Renamed lambda and alter_filename_lambda in rename_file (#27)

README.md

@@ -3,6 +3,7 @@
 [![Build Status](https://travis-ci.org/snowplow/sluice.png)](https://travis-ci.org/snowplow/sluice)
 [![Code Climate](https://codeclimate.com/github/snowplow/sluice.png)](https://codeclimate.com/github/snowplow/sluice)
 [![Coverage Status](https://coveralls.io/repos/snowplow/sluice/badge.png?branch=master)](https://coveralls.io/r/snowplow/sluice?branch=master)
+[![License][license-image]][license]
 
 Sluice is a Ruby gem (built with [Bundler] [bundler]) to help you build cloud-friendly ETL (extract, transform, load) processes.
 
@@ -25,34 +26,34 @@ Sluice has been extracted from a pair of Ruby ETL applications built by the [Sno
 
 Or in your Gemfile:
 
-    gem 'sluice', '~> 0.2.0'
+    gem 'sluice', '~> 0.2.2'
 
 ## Usage
 
 Rubydoc and usage examples to come.
 
-## Hacking and contributing
+## Developer quickstart
 
-To hack on Sluice locally:
+Assuming git, **[Vagrant] [vagrant-install]** and **[VirtualBox] [virtualbox-install]** installed:
 
-    $ gem build sluice.gemspec
-    $ sudo gem install sluice-0.2.0.gem
+```bash
+ host$ git clone https://github.com/snowplow/sluice.git
+ host$ cd sluice
+ host$ vagrant up && vagrant ssh
+guest$ cd /vagrant
+guest$ gem install bundler
+guest$ rspec
+```
 
-To contribute:
+## Publishing
 
-1. Fork it
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Added some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create new Pull Request
-
-## Credits and thanks
-
-Sluice was developed by [Alex Dean] [alexanderdean] ([Snowplow Analytics] [snowplow-analytics]) and [Michael Tibben] [mtibben] ([99designs] [99designs]).
+```bash
+ host$ vagrant push
+```
 
 ## Copyright and license
 
-Sluice is copyright 2012-2014 Snowplow Analytics Ltd.
+Sluice is copyright 2012-2015 Snowplow Analytics Ltd.
 
 Licensed under the [Apache License, Version 2.0] [license] (the "License");
 you may not use this software except in compliance with the License.
@@ -65,6 +66,9 @@ limitations under the License.
 
 [bundler]: http://gembundler.com/
 
+[license-image]: http://img.shields.io/badge/license-Apache--2-blue.svg?style=flat
+[license]: http://www.apache.org/licenses/LICENSE-2.0
+
 [snowplow-analytics]: http://snowplowanalytics.com
 [alexanderdean]: https://github.com/alexanderdean
 [mtibben]: https://github.com/mtibben
@@ -73,4 +77,7 @@ limitations under the License.
 [emr-etl-runner]: https://github.com/snowplow/snowplow/tree/master/3-enrich/emr-etl-runner
 [storage-loader]: https://github.com/snowplow/snowplow/tree/master/4-storage/storage-loader
 
+[vagrant-install]: http://docs.vagrantup.com/v2/installation/index.html
+[virtualbox-install]: https://www.virtualbox.org/wiki/Downloads
+
 [license]: http://www.apache.org/licenses/LICENSE-2.0

Vagrantfile

@@ -0,0 +1,23 @@
+Vagrant.configure("2") do |config|
+
+  config.vm.box = "ubuntu/trusty64"
+  config.vm.hostname = "sluice"
+  config.ssh.forward_agent = true
+
+  config.vm.provider :virtualbox do |vb|
+    vb.name = Dir.pwd().split("/")[-1] + "-" + Time.now.to_f.to_i.to_s
+    vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
+    vb.customize [ "guestproperty", "set", :id, "--timesync-threshold", 10000 ]
+    vb.memory = 2048
+  end
+
+  config.vm.provision :shell do |sh|
+    sh.path = "vagrant/up.bash"
+  end
+
+  # Requires Vagrant 1.7.0+
+  config.push.define "gem", strategy: "local-exec" do |push|
+    push.script = "vagrant/push.bash"
+  end
+
+end

lib/sluice/storage/storage.rb

@@ -38,13 +38,15 @@ def files_between(start_date, end_date, date_format, file_ext=nil)
     end
     module_function :files_between
 
-    # Add a trailing slash to a path if missing
+    # Add a trailing slash to a path if missing.
+    # Tolerates a nil path.
     #
     # Parameters:
     # +path+:: path to add a trailing slash to
     def trail_slash(path)
-
-      path[-1].chr != '/' ? path << '/' : path
+      unless path.nil?
+        path[-1].chr != '/' ? path << '/' : path
+      end
     end
     module_function :trail_slash
 

lib/sluice/version.rb

@@ -15,5 +15,5 @@
 
 module Sluice
   NAME    = "sluice"
-  VERSION = "0.2.1"
+  VERSION = "0.2.2"
 end

sluice.gemspec

@@ -36,7 +36,7 @@ Gem::Specification.new do |gem|
 
   # Dependencies
   gem.add_dependency 'contracts', '~> 0.4'
-  gem.add_dependency 'fog', '~> 1.22'
+  gem.add_dependency 'fog', '1.24'
 
   gem.add_development_dependency "rspec", "~> 2.14", ">= 2.14.1"
   gem.add_development_dependency "rspec-nc"

vagrant/.gitignore

@@ -0,0 +1,3 @@
+oss-playbooks
+ansible
+.peru

vagrant/ansible.hosts

@@ -0,0 +1,2 @@
+[vagrant]
+127.0.0.1:2222

vagrant/peru.yaml

@@ -0,0 +1,14 @@
+imports:
+    ansible: ansible
+    ansible_playbooks: oss-playbooks
+
+curl module ansible:
+    # Equivalent of git cloning tags/v1.6.6 but much, much faster
+    url: https://codeload.github.com/ansible/ansible/zip/69d85c22c7475ccf8169b6ec9dee3ee28c92a314
+    unpack: zip
+    export: ansible-69d85c22c7475ccf8169b6ec9dee3ee28c92a314
+
+git module ansible_playbooks:
+    url: https://github.com/snowplow/ansible-playbooks.git
+    # Comment out to fetch a specific rev instead of master:
+    # rev: xxx

vagrant/push.bash

@@ -0,0 +1,79 @@
+#!/bin/bash
+set -e
+
+# Constants
+rubygems_user=snowplow
+rubygems_gem=sluice
+
+# Check if our Vagrant box is running. Expects `vagrant status` to look like:
+#
+# > Current machine states:
+# >
+# > default                   poweroff (virtualbox)
+# >
+# > The VM is powered off. To restart the VM, simply run `vagrant up`
+function running {
+    set +e
+    vagrant status | sed -n 3p | grep -q "^default\s*running (virtualbox)$"
+    local is_running=$?
+    set -e
+    echo $is_running
+}
+
+# Reads the version out of the version.rb
+function parse_version {
+    cat "lib/${rubygems_gem}/version.rb" | awk '/  VERSION =/ {v=$3; gsub(/\047/, "", v)} END {print v}'
+}
+
+# Installs RubyGems.org credentials in our guest
+#
+# Parameters:
+# 1. rubygems_password
+function install_creds {
+    vagrant ssh -c "curl -u ${rubygems_user}:${1} https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials"
+    vagrant ssh -c "chmod 0600 ~/.gem/credentials"
+}
+
+# Builds our gem
+function build_gem {
+    vagrant ssh -c "cd /vagrant && gem build ${rubygems_gem}.gemspec"
+}
+
+# Installs our gem
+#
+# Parameters:
+# 1. rubygems_version
+function install_gem {
+    vagrant ssh -c "cd /vagrant && sudo gem install ./${rubygems_gem}-${1}.gem"
+}
+
+# Pushes our gem to RubyGems.org
+#
+# Parameters:
+# 1. rubygems_version
+function push_gem {
+    vagrant ssh -c "cd /vagrant && gem push ./${rubygems_gem}-${1}.gem"
+}
+
+
+# Move to the parent directory of this script
+source="${BASH_SOURCE[0]}"
+while [ -h "${source}" ] ; do source="$(readlink "${source}")"; done
+dir="$( cd -P "$( dirname "${source}" )/.." && pwd )"
+cd ${dir}
+
+# Precondition for running
+if [ $(running) != "0" ]; then
+    echo "Vagrant guest must be running to push"
+    exit 1
+fi
+
+# Can't pass args thru vagrant push so have to prompt
+read -e -p "Please enter password for RubyGems.org user ${rubygems_user}: " rubygems_password
+
+# Build, install & push
+build_gem
+rubygems_version=$(parse_version)
+install_gem ${rubygems_version}
+install_creds ${rubygems_password}
+push_gem ${rubygems_version}

vagrant/up.bash

@@ -0,0 +1,50 @@
+#!/bin/bash
+set -e
+
+vagrant_dir=/vagrant/vagrant
+bashrc=/home/vagrant/.bashrc
+
+echo "========================================"
+echo "INSTALLING PERU AND ANSIBLE DEPENDENCIES"
+echo "----------------------------------------"
+apt-get update
+apt-get install -y language-pack-en git unzip libyaml-dev python3-pip python-yaml python-paramiko python-jinja2
+
+echo "==============="
+echo "INSTALLING PERU"
+echo "---------------"
+sudo pip3 install peru
+
+echo "======================================="
+echo "CLONING ANSIBLE AND PLAYBOOKS WITH PERU"
+echo "---------------------------------------"
+cd ${vagrant_dir} && peru sync -v
+echo "... done"
+
+env_setup=${vagrant_dir}/ansible/hacking/env-setup
+hosts=${vagrant_dir}/ansible.hosts
+
+echo "==================="
+echo "CONFIGURING ANSIBLE"
+echo "-------------------"
+touch ${bashrc}
+echo "source ${env_setup}" >> ${bashrc}
+echo "export ANSIBLE_HOSTS=${hosts}" >> ${bashrc}
+echo "... done"
+
+echo "=========================================="
+echo "RUNNING PLAYBOOKS WITH ANSIBLE*"
+echo "* no output while each playbook is running"
+echo "------------------------------------------"
+while read pb; do
+    su - -c "source ${env_setup} && ${vagrant_dir}/ansible/bin/ansible-playbook ${vagrant_dir}/${pb} --connection=local --inventory-file=${hosts}" vagrant
+done <${vagrant_dir}/up.playbooks
+
+guidance=${vagrant_dir}/up.guidance
+
+if [ -f ${guidance} ]; then
+    echo "==========="
+    echo "PLEASE READ"
+    echo "-----------"
+    cat $guidance
+fi

vagrant/up.guidance

@@ -0,0 +1,5 @@
+To get started:
+vagrant ssh
+cd /vagrant
+bundle install
+rspec

vagrant/up.playbooks

@@ -0,0 +1 @@
+oss-playbooks/ruby-rvm.yml