diff --git a/.gitignore b/.gitignore index d87d4be..4380e30 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ spec/reports test/tmp test/version_tmp tmp + +# Vagrant +.vagrant diff --git a/CHANGELOG b/CHANGELOG index 087942e..45b07dd 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,10 @@ +Version 0.2.2 (2015-04-08) +-------------------------- +Now tolerates nil passed to trail_slash (#38) +Added License button to README (#34) +Added Vagrant quickstart to README (#36) +Added dedicated Vagrant setup (#35) + Version 0.2.1 (2014-05-20) -------------------------- Renamed lambda and alter_filename_lambda in rename_file (#27) diff --git a/README.md b/README.md index bb74afe..aa0262b 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ [![Build Status](https://travis-ci.org/snowplow/sluice.png)](https://travis-ci.org/snowplow/sluice) [![Code Climate](https://codeclimate.com/github/snowplow/sluice.png)](https://codeclimate.com/github/snowplow/sluice) [![Coverage Status](https://coveralls.io/repos/snowplow/sluice/badge.png?branch=master)](https://coveralls.io/r/snowplow/sluice?branch=master) +[![License][license-image]][license] Sluice is a Ruby gem (built with [Bundler] [bundler]) to help you build cloud-friendly ETL (extract, transform, load) processes. @@ -25,34 +26,34 @@ Sluice has been extracted from a pair of Ruby ETL applications built by the [Sno Or in your Gemfile: - gem 'sluice', '~> 0.2.0' + gem 'sluice', '~> 0.2.2' ## Usage Rubydoc and usage examples to come. -## Hacking and contributing +## Developer quickstart -To hack on Sluice locally: +Assuming git, **[Vagrant] [vagrant-install]** and **[VirtualBox] [virtualbox-install]** installed: - $ gem build sluice.gemspec - $ sudo gem install sluice-0.2.0.gem +```bash + host$ git clone https://github.com/snowplow/sluice.git + host$ cd sluice + host$ vagrant up && vagrant ssh +guest$ cd /vagrant +guest$ gem install bundler +guest$ rspec +``` -To contribute: +## Publishing -1. Fork it -2. Create your feature branch (`git checkout -b my-new-feature`) -3. Commit your changes (`git commit -am 'Added some feature'`) -4. Push to the branch (`git push origin my-new-feature`) -5. Create new Pull Request - -## Credits and thanks - -Sluice was developed by [Alex Dean] [alexanderdean] ([Snowplow Analytics] [snowplow-analytics]) and [Michael Tibben] [mtibben] ([99designs] [99designs]). +```bash + host$ vagrant push +``` ## Copyright and license -Sluice is copyright 2012-2014 Snowplow Analytics Ltd. +Sluice is copyright 2012-2015 Snowplow Analytics Ltd. Licensed under the [Apache License, Version 2.0] [license] (the "License"); you may not use this software except in compliance with the License. @@ -65,6 +66,9 @@ limitations under the License. [bundler]: http://gembundler.com/ +[license-image]: http://img.shields.io/badge/license-Apache--2-blue.svg?style=flat +[license]: http://www.apache.org/licenses/LICENSE-2.0 + [snowplow-analytics]: http://snowplowanalytics.com [alexanderdean]: https://github.com/alexanderdean [mtibben]: https://github.com/mtibben @@ -73,4 +77,7 @@ limitations under the License. [emr-etl-runner]: https://github.com/snowplow/snowplow/tree/master/3-enrich/emr-etl-runner [storage-loader]: https://github.com/snowplow/snowplow/tree/master/4-storage/storage-loader +[vagrant-install]: http://docs.vagrantup.com/v2/installation/index.html +[virtualbox-install]: https://www.virtualbox.org/wiki/Downloads + [license]: http://www.apache.org/licenses/LICENSE-2.0 diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..63bfd02 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,23 @@ +Vagrant.configure("2") do |config| + + config.vm.box = "ubuntu/trusty64" + config.vm.hostname = "sluice" + config.ssh.forward_agent = true + + config.vm.provider :virtualbox do |vb| + vb.name = Dir.pwd().split("/")[-1] + "-" + Time.now.to_f.to_i.to_s + vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] + vb.customize [ "guestproperty", "set", :id, "--timesync-threshold", 10000 ] + vb.memory = 2048 + end + + config.vm.provision :shell do |sh| + sh.path = "vagrant/up.bash" + end + + # Requires Vagrant 1.7.0+ + config.push.define "gem", strategy: "local-exec" do |push| + push.script = "vagrant/push.bash" + end + +end diff --git a/lib/sluice/storage/storage.rb b/lib/sluice/storage/storage.rb index 7a5c185..3eb2b82 100644 --- a/lib/sluice/storage/storage.rb +++ b/lib/sluice/storage/storage.rb @@ -38,13 +38,15 @@ def files_between(start_date, end_date, date_format, file_ext=nil) end module_function :files_between - # Add a trailing slash to a path if missing + # Add a trailing slash to a path if missing. + # Tolerates a nil path. # # Parameters: # +path+:: path to add a trailing slash to def trail_slash(path) - - path[-1].chr != '/' ? path << '/' : path + unless path.nil? + path[-1].chr != '/' ? path << '/' : path + end end module_function :trail_slash diff --git a/lib/sluice/version.rb b/lib/sluice/version.rb index 74b36ab..3d628c8 100644 --- a/lib/sluice/version.rb +++ b/lib/sluice/version.rb @@ -15,5 +15,5 @@ module Sluice NAME = "sluice" - VERSION = "0.2.1" + VERSION = "0.2.2" end \ No newline at end of file diff --git a/sluice.gemspec b/sluice.gemspec index 7f395b1..50f3619 100644 --- a/sluice.gemspec +++ b/sluice.gemspec @@ -36,7 +36,7 @@ Gem::Specification.new do |gem| # Dependencies gem.add_dependency 'contracts', '~> 0.4' - gem.add_dependency 'fog', '~> 1.22' + gem.add_dependency 'fog', '1.24' gem.add_development_dependency "rspec", "~> 2.14", ">= 2.14.1" gem.add_development_dependency "rspec-nc" diff --git a/vagrant/.gitignore b/vagrant/.gitignore new file mode 100644 index 0000000..5b164d3 --- /dev/null +++ b/vagrant/.gitignore @@ -0,0 +1,3 @@ +oss-playbooks +ansible +.peru diff --git a/vagrant/ansible.hosts b/vagrant/ansible.hosts new file mode 100644 index 0000000..588fa08 --- /dev/null +++ b/vagrant/ansible.hosts @@ -0,0 +1,2 @@ +[vagrant] +127.0.0.1:2222 diff --git a/vagrant/peru.yaml b/vagrant/peru.yaml new file mode 100644 index 0000000..e7fdf41 --- /dev/null +++ b/vagrant/peru.yaml @@ -0,0 +1,14 @@ +imports: + ansible: ansible + ansible_playbooks: oss-playbooks + +curl module ansible: + # Equivalent of git cloning tags/v1.6.6 but much, much faster + url: https://codeload.github.com/ansible/ansible/zip/69d85c22c7475ccf8169b6ec9dee3ee28c92a314 + unpack: zip + export: ansible-69d85c22c7475ccf8169b6ec9dee3ee28c92a314 + +git module ansible_playbooks: + url: https://github.com/snowplow/ansible-playbooks.git + # Comment out to fetch a specific rev instead of master: + # rev: xxx diff --git a/vagrant/push.bash b/vagrant/push.bash new file mode 100755 index 0000000..9c3fa4a --- /dev/null +++ b/vagrant/push.bash @@ -0,0 +1,79 @@ +#!/bin/bash +set -e + +# Constants +rubygems_user=snowplow +rubygems_gem=sluice + +# Check if our Vagrant box is running. Expects `vagrant status` to look like: +# +# > Current machine states: +# > +# > default poweroff (virtualbox) +# > +# > The VM is powered off. To restart the VM, simply run `vagrant up` +function running { + set +e + vagrant status | sed -n 3p | grep -q "^default\s*running (virtualbox)$" + local is_running=$? + set -e + echo $is_running +} + +# Reads the version out of the version.rb +function parse_version { + cat "lib/${rubygems_gem}/version.rb" | awk '/ VERSION =/ {v=$3; gsub(/\047/, "", v)} END {print v}' +} + +# Installs RubyGems.org credentials in our guest +# +# Parameters: +# 1. rubygems_password +function install_creds { + vagrant ssh -c "curl -u ${rubygems_user}:${1} https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials" + vagrant ssh -c "chmod 0600 ~/.gem/credentials" +} + +# Builds our gem +function build_gem { + vagrant ssh -c "cd /vagrant && gem build ${rubygems_gem}.gemspec" +} + +# Installs our gem +# +# Parameters: +# 1. rubygems_version +function install_gem { + vagrant ssh -c "cd /vagrant && sudo gem install ./${rubygems_gem}-${1}.gem" +} + +# Pushes our gem to RubyGems.org +# +# Parameters: +# 1. rubygems_version +function push_gem { + vagrant ssh -c "cd /vagrant && gem push ./${rubygems_gem}-${1}.gem" +} + + +# Move to the parent directory of this script +source="${BASH_SOURCE[0]}" +while [ -h "${source}" ] ; do source="$(readlink "${source}")"; done +dir="$( cd -P "$( dirname "${source}" )/.." && pwd )" +cd ${dir} + +# Precondition for running +if [ $(running) != "0" ]; then + echo "Vagrant guest must be running to push" + exit 1 +fi + +# Can't pass args thru vagrant push so have to prompt +read -e -p "Please enter password for RubyGems.org user ${rubygems_user}: " rubygems_password + +# Build, install & push +build_gem +rubygems_version=$(parse_version) +install_gem ${rubygems_version} +install_creds ${rubygems_password} +push_gem ${rubygems_version} diff --git a/vagrant/up.bash b/vagrant/up.bash new file mode 100755 index 0000000..7450ae8 --- /dev/null +++ b/vagrant/up.bash @@ -0,0 +1,50 @@ +#!/bin/bash +set -e + +vagrant_dir=/vagrant/vagrant +bashrc=/home/vagrant/.bashrc + +echo "========================================" +echo "INSTALLING PERU AND ANSIBLE DEPENDENCIES" +echo "----------------------------------------" +apt-get update +apt-get install -y language-pack-en git unzip libyaml-dev python3-pip python-yaml python-paramiko python-jinja2 + +echo "===============" +echo "INSTALLING PERU" +echo "---------------" +sudo pip3 install peru + +echo "=======================================" +echo "CLONING ANSIBLE AND PLAYBOOKS WITH PERU" +echo "---------------------------------------" +cd ${vagrant_dir} && peru sync -v +echo "... done" + +env_setup=${vagrant_dir}/ansible/hacking/env-setup +hosts=${vagrant_dir}/ansible.hosts + +echo "===================" +echo "CONFIGURING ANSIBLE" +echo "-------------------" +touch ${bashrc} +echo "source ${env_setup}" >> ${bashrc} +echo "export ANSIBLE_HOSTS=${hosts}" >> ${bashrc} +echo "... done" + +echo "==========================================" +echo "RUNNING PLAYBOOKS WITH ANSIBLE*" +echo "* no output while each playbook is running" +echo "------------------------------------------" +while read pb; do + su - -c "source ${env_setup} && ${vagrant_dir}/ansible/bin/ansible-playbook ${vagrant_dir}/${pb} --connection=local --inventory-file=${hosts}" vagrant +done <${vagrant_dir}/up.playbooks + +guidance=${vagrant_dir}/up.guidance + +if [ -f ${guidance} ]; then + echo "===========" + echo "PLEASE READ" + echo "-----------" + cat $guidance +fi diff --git a/vagrant/up.guidance b/vagrant/up.guidance new file mode 100644 index 0000000..85e564e --- /dev/null +++ b/vagrant/up.guidance @@ -0,0 +1,5 @@ +To get started: +vagrant ssh +cd /vagrant +bundle install +rspec diff --git a/vagrant/up.playbooks b/vagrant/up.playbooks new file mode 100644 index 0000000..fa931bd --- /dev/null +++ b/vagrant/up.playbooks @@ -0,0 +1 @@ +oss-playbooks/ruby-rvm.yml