Permalink
Browse files

add an uploader, refactor config, add a read me

  • Loading branch information...
Sven Fuchs
Sven Fuchs committed Oct 2, 2011
1 parent 031c0f1 commit f85db52a7118ca5cf7a957cca91678c6cbc589c9
Showing with 237 additions and 40 deletions.
  1. +5 −2 Gemfile
  2. +9 −0 Gemfile.lock
  3. +42 −0 README.md
  4. +1 −0 lib/travis/boxes.rb
  5. +20 −16 lib/travis/boxes/cli/vagrant.rb
  6. +43 −15 lib/travis/boxes/config.rb
  7. +39 −0 lib/travis/boxes/upload.rb
  8. +55 −7 spec/config_spec.rb
  9. +23 −0 spec/upload_spec.rb
View
@@ -1,7 +1,10 @@
+source :rubygems
+
+gem 'thor'
+gem 'vagrant', '~> 0.8.0'
gem 'hashr', '~> 0.0.13'
gem 'multi_json'
-gem 'vagrant', '~> 0.8.0'
-gem 'thor'
+gem 'aws-s3'
group :test do
gem 'rspec'
View
@@ -1,12 +1,19 @@
GEM
+ remote: http://rubygems.org/
specs:
archive-tar-minitar (0.5.2)
+ aws-s3 (0.6.2)
+ builder
+ mime-types
+ xml-simple
+ builder (3.0.0)
diff-lcs (1.1.3)
erubis (2.7.0)
ffi (1.0.9)
hashr (0.0.16)
i18n (0.6.0)
json (1.5.4)
+ mime-types (1.16)
multi_json (1.0.3)
net-scp (1.0.4)
net-ssh (>= 1.99.1)
@@ -31,11 +38,13 @@ GEM
virtualbox (~> 0.9.1)
virtualbox (0.9.2)
ffi (~> 1.0.9)
+ xml-simple (1.1.0)
PLATFORMS
ruby
DEPENDENCIES
+ aws-s3
hashr (~> 0.0.13)
multi_json
rspec
View
@@ -0,0 +1,42 @@
+Support for Travis CI worker box maintenance.
+
+Use the provided `thor` tasks to build worker base boxes and upload them to S3. They can then be distributed to the worker machines and used for updating the vms.
+
+Base boxes are built per "environment" (i.e. worker type, e.g. "staging", "ruby", "rails", ...)
+
+E.g. for rebuilding the staging base box use:
+
+ $ thor travis:box:build -e staging -b bases/lucid32.box
+
+Configuration for the boxes is in the local and shared `config/*.yml` files.
+
+The shared file `config/worker.base.yml` will be used for configuration common to all boxes and it will be merged with the shared file `config/[environment].yml`.
+
+The result is then also merged with
+
+* the "base" section from the local file `config/worker.yml`
+* the respective environment section (e.g. "staging") from the local file `config/worker.yml`
+
+The file `config/worker.yml` is meant to be *local* and should not be checked in.
+
+Example:
+
+ # config/worker.base.yml (in travis-boxes)
+ foo: foo
+
+ # config/worker.staging.yml (in travis-boxes)
+ bar: bar
+
+ # config/worker.yml (in the current working directory)
+ base:
+ secret: secret
+ staging:
+ another_secret: another_secret
+
+ # Travis::Boxes::Config.new.staging
+ {
+ 'foo' => 'Foo',
+ 'bar' => 'bar',
+ 'secret' => 'secret',
+ 'another_secret' => 'another_secret',
+ }
View
@@ -2,5 +2,6 @@ module Travis
module Boxes
autoload :Cli, 'travis/boxes/cli'
autoload :Config, 'travis/boxes/config'
+ autoload :Upload, 'travis/boxes/upload'
end
end
@@ -1,17 +1,17 @@
require 'archive/tar/minitar'
require 'json'
+require 'right_aws'
require 'travis/boxes'
-require 'virtualbox'
module Travis
module Boxes
module Cli
class Vagrant < Thor
- namespace "travis:boxes:vagrant"
+ namespace "travis:box"
include Cli
- desc 'rebuild', 'Rebuild the base box'
+ desc 'build', 'Build a base box'
method_option :env, :aliases => '-e', :default => 'development', :desc => 'Environment the box is built for (e.g staging)'
method_option :base, :aliases => '-b', :desc => 'Base box for this box (e.g. lucid32.box)'
method_option :upload, :aliases => '-u', :desc => 'Upload the box'
@@ -29,14 +29,21 @@ def rebuild
upload if upload?
end
+ desc 'upload', 'Upload a base box'
+ method_option :env, :aliases => '-e', :default => 'development', :desc => 'Environment the box is built for (e.g staging)'
+
+ def upload
+ Travis::Boxes::Upload.new(env, config.s3).perform
+ end
+
protected
def vbox
@vbox ||= Vbox.new('', options)
end
def config
- @config ||= Travis::Boxes::Config.new(env)
+ @config ||= Travis::Boxes::Config.new[env]
end
def env
@@ -52,7 +59,7 @@ def base
end
def target
- @target ||= "boxes/#{env}.box" # -#{Time.now.strftime('%Y%m%d%H%M%S')}
+ @target ||= "boxes/#{env}.box"
end
def download
@@ -79,14 +86,6 @@ def halt
run "vagrant halt #{env}"
end
- # def immute_disk
- # run <<-sh
- # VBoxManage storageattach #{uuid} --storagectl "SATA Controller" --port 0 --device 0 --medium none
- # VBoxManage modifyhd ~/VirtualBox\\\\ VMs/#{uuid}/box-disk1.vmdk/#{name} --type immutable
- # VBoxManage storageattach #{uuid} --storagectl "SATA Controller" --port 0 --device 0 --medium #{name} --type hdd
- # sh
- # end
-
def package_box
run <<-sh
vagrant package --base #{uuid}
@@ -95,13 +94,18 @@ def package_box
sh
end
- def upload
- end
-
def uuid
meta = JSON.parse(File.read('.vagrant'))
meta['active'][env] || raise("could not find #{env} uuid in #{meta.inspect}")
end
+
+ # def immute_disk
+ # run <<-sh
+ # VBoxManage storageattach #{uuid} --storagectl "SATA Controller" --port 0 --device 0 --medium none
+ # VBoxManage modifyhd ~/VirtualBox\\\\ VMs/#{uuid}/box-disk1.vmdk/#{name} --type immutable
+ # VBoxManage storageattach #{uuid} --storagectl "SATA Controller" --port 0 --device 0 --medium #{name} --type hdd
+ # sh
+ # end
end
end
end
View
@@ -3,30 +3,58 @@
module Travis
module Boxes
- class Config < Hashr
- define :base => 'lucid32_new.box',
- :cookbooks => 'vendor/travis-cookbooks',
- :json => {},
- :recipes => []
-
- def initialize(type)
- super(read(type))
+ class Config
+ class Environment < Hashr
+ define :base => 'lucid32_new.box',
+ :cookbooks => 'vendor/travis-cookbooks',
+ :json => {},
+ :recipes => [],
+ :s3 => { :bucket => 'travis-boxes' }
+ end
+
+ attr_reader :environments
+
+ def initialize
+ @environments = {}
+ end
+
+ def environment(name)
+ environments[name.to_sym] ||= Environment.new(read(name.to_s))
+ end
+ alias :[] :environment
+
+ def method_missing(name, *args, &block)
+ args.empty? ? environment(name) : super
end
protected
- def read(type)
- read_yml('base').merge(read_yml(type))
+ def read(name)
+ base.merge(env(name)).merge((local['base'] || {}).merge(local[name] || {})).merge(:env => name)
end
- def read_yml(type)
- YAML.load_file(path(type)) || {}
+ def base
+ read_yml('base', true)
end
- def path(type)
- filename = ['config/worker', type, 'yml'].compact.join('.')
- path = File.expand_path(filename)
+ def env(name)
+ read_yml(name, true)
+ end
+
+ def local
+ read_yml
+ end
+
+ def read_yml(name = nil)
+ path = self.path(name)
File.exists?(path) ? path : raise("Could not find a configuration file #{path}")
+ YAML.load_file(path) || {}
+ end
+
+ def path(name = nil)
+ directory = name ? File.expand_path('../../../..', __FILE__) : '.'
+ filename = ['config/worker', name, 'yml'].compact.join('.')
+ [directory, filename].join('/')
end
end
end
View
@@ -0,0 +1,39 @@
+module Travis
+ module Boxes
+ class Upload
+ attr_reader :env, :config
+
+ def initialize(env, config)
+ @env = env
+ @config = config
+ end
+
+ def perform
+ s3.put(bucket, target, source.open)
+ end
+
+ protected
+
+ def s3
+ RightAws::S3.new(config.access_key_id, config.secret_access_key).interface
+ end
+
+ def bucket
+ config.bucket
+ end
+
+ def source
+ Pathname.new("boxes/#{env}.box")
+ end
+
+ def target
+ "boxes/#{env}/#{timestamp}.box"
+ end
+
+ def timestamp
+ Time.now.strftime('%Y%m%d%H%M%S')
+ end
+ end
+ end
+end
+
View
@@ -1,29 +1,77 @@
require 'travis/boxes'
describe Travis::Boxes::Config do
- let :config do
- Travis::Boxes::Config.new('test')
- end
+ let(:config) { Travis::Boxes::Config.new }
describe 'defaults' do
before :each do
Travis::Boxes::Config.any_instance.stub(:read).and_return({})
end
it ':base defaults "lucid32_new.box"' do
- config.base.should == 'lucid32_new.box'
+ config.test.base.should == 'lucid32_new.box'
end
it ':cookbooks defaults to "vendor/travis-cookbooks"' do
- config.cookbooks.should == 'vendor/travis-cookbooks'
+ config.test.cookbooks.should == 'vendor/travis-cookbooks'
end
it ':json defaults to {}' do
- config.cookbooks.should == 'vendor/travis-cookbooks'
+ config.test.cookbooks.should == 'vendor/travis-cookbooks'
end
it ':recipes defaults to []' do
- config.cookbooks.should == 'vendor/travis-cookbooks'
+ config.test.cookbooks.should == 'vendor/travis-cookbooks'
+ end
+ end
+
+ describe 'merging' do
+ DATA_STUBS = {
+ 'local' => { 'base' => { 'secret' => 'secret' }, 'staging' => { 'another_secret' => 'another_secret' } },
+ 'base' => { 'foo' => 'foo' },
+ 'env' => { 'bar' => 'bar' }
+ }
+
+ before :each do
+ config.stub(DATA_STUBS)
+ end
+
+ it 'merges base stuff' do
+ config.staging.foo.should == 'foo'
+ end
+
+ it 'merges env stuff' do
+ config.staging.bar.should == 'bar'
+ end
+
+ it 'merges local base stuff' do
+ config.staging.secret.should == 'secret'
+ end
+
+ it 'merges local env stuff' do
+ config.staging.another_secret.should == 'another_secret'
+ end
+
+ it 'adds the environment to the configuration' do
+ config.staging.env.should == 'staging'
+ end
+ end
+
+ describe 'config file paths' do
+ before :each do
+ File.stub(:exists?).and_return(true)
+ end
+
+ it 'shared config/worker.base.yml' do
+ config.send(:path, 'base').should == File.expand_path('../../config/worker.base.yml', __FILE__)
+ end
+
+ it 'shared config/worker.[environment].yml' do
+ config.send(:path, 'staging').should == File.expand_path('../../config/worker.staging.yml', __FILE__)
+ end
+
+ it 'local config/worker.yml' do
+ config.send(:path).should == './config/worker.yml'
end
end
end
Oops, something went wrong.

0 comments on commit f85db52

Please sign in to comment.