Permalink
Browse files

New BOSH blobs experience

Simplified blobs management:

- no need to sync manually, 'create release' now syncs blobs
  before trying to generate anything;

- old blobstore commands are now retired in favor of two new commands:
  'bosh add blob <local_file> [<blob_dir>]' moves blob to blobs/<blob_dir>
  'bosh upload blobs' finds and uploads new/updated blobs;

- 'bosh blobs' shows what's going on and whether there are blobs that
   need to be uploaded.

Change-Id: I019005bfb8583c087ee6427f39463ef29be4569d
  • Loading branch information...
1 parent 6a8ebad commit 6926aa2d9a9a0a23525844c89569c64064291833 @olegshaldybin olegshaldybin committed Apr 20, 2012
View
2 cli/Gemfile.lock
@@ -1,7 +1,7 @@
PATH
remote: .
specs:
- bosh_cli (0.16)
+ bosh_cli (0.17)
@drnic
Cloud Foundry member
drnic added a line comment Apr 27, 2012

I'm personally not a fan of bumping gem versions in other commits. A nice clean "Bumped to 0.17" commit is easy to spot and easy to tag.

@olegshaldybin
olegshaldybin added a line comment Apr 27, 2012

I totally agree! Just need to start practicing what I preach I guess :-)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
blobstore_client (~> 0.3.13)
highline (~> 1.6.2)
httpclient (>= 2.2.4, <= 2.2.4)
View
2 cli/lib/cli.rb
@@ -64,6 +64,8 @@ module Cli
require "cli/release_compiler"
require "cli/release_tarball"
+require "cli/blob_manager"
+
require "cli/command_definition"
require "cli/runner"
View
376 cli/lib/cli/blob_manager.rb
@@ -0,0 +1,376 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Cli
+ # In order to avoid storing large objects in git repo,
+ # release might save them in the blobstore instead.
+ # BlobManager encapsulates most of the blob operations.
+ class BlobManager
+ DEFAULT_INDEX_NAME = "blobs.yml"
+
+ attr_reader :new_blobs, :updated_blobs
+
+ # @param [Bosh::Cli::Release] release BOSH Release object
+ def initialize(release)
+ @release = release
+ @index_file = File.join(@release.dir, "config", DEFAULT_INDEX_NAME)
+
+ legacy_index_file = File.join(@release.dir, "blob_index.yml")
+
+ if File.exists?(legacy_index_file)
+ if File.exists?(@index_file)
+ err("Found both new and legacy blob index, please fix it")
+ end
+ FileUtils.mv(legacy_index_file, @index_file)
+ end
+
+ if File.file?(@index_file)
+ @index = load_yaml_file(@index_file)
+ else
+ @index = {}
+ end
+
+ @src_dir = File.join(@release.dir, "src")
+ unless File.directory?(@src_dir)
+ err("`src' directory is missing")
+ end
+
+ @storage_dir = File.join(@release.dir, ".blobs")
+ unless File.directory?(@storage_dir)
+ FileUtils.mkdir(@storage_dir)
+ end
+
+ @blobs_dir = File.join(@release.dir, "blobs")
+ unless File.directory?(@blobs_dir)
+ FileUtils.mkdir(@blobs_dir)
+ end
+
+ @blobstore = @release.blobstore
+ if @blobstore.nil?
+ err("Blobstore is not configured")
+ end
+
+ @new_blobs = []
+ @updated_blobs = []
+ end
+
+ # Returns a list of blobs that need to be uploaded
+ # @return [Array]
+ def blobs_to_upload
+ @new_blobs + @updated_blobs
+ end
+
+ # Returns whether blobs directory is dirty
+ # @return Boolean
+ def dirty?
+ @new_blobs.size > 0 || @updated_blobs.size > 0
+ end
+
+ # Prints out blobs status
+ # @return [void]
+ def print_status
+ total_file_size = @index.inject(0) do |total, (_, entry)|
+ total += entry["size"].to_i
+ total
+ end
+
+ say("Total: #{@index.size}, #{pretty_size(total_file_size)}")
+ process_blobs_directory
+
+ unless dirty?
+ say("No blobs to upload".green)
+ return
+ end
+
+ nl
+ say("You have some blobs that need to be uploaded:")
+ @new_blobs.each do |blob|
+ size = File.size(File.join(@blobs_dir, blob))
+ say("%s\t%s\t%s" % ["new".green, blob, pretty_size(size)])
+ end
+
+ @updated_blobs.each do |blob|
+ size = File.size(File.join(@blobs_dir, blob))
+ say("%s\t%s\t%s" % ["new version".yellow, blob, pretty_size(size)])
+ end
+
+ nl
+ say("When ready please run `#{"bosh upload blobs".green}'")
+ end
+
+ # Registers a file as BOSH blob
+ # @param [String] local_path Local file path
+ # @param [String] blob_path Blob path relative to blobs directory
+ # @return [void]
+ def add_blob(local_path, blob_path)
+ unless File.exists?(local_path)
+ err("File `#{local_path}' not found")
+ end
+
+ if File.directory?(local_path)
+ err("`#{local_path}' is a directory")
+ end
+
+ if blob_path[0..0] == "/"
+ err("Blob path should be a relative path")
+ end
+
+ if blob_path[0..5] == "blobs/"
+ err("Blob path should not start with `blobs/'")
+ end
+
+ blob_dst = File.join(@blobs_dir, blob_path)
+
+ if File.directory?(blob_dst)
+ err("`#{blob_dst}' is a directory, please pick a different path")
+ end
+
+ update = false
+ if File.exists?(blob_dst)
+ if file_checksum(blob_dst) == file_checksum(local_path)
+ err("Already tracking the same version of `#{blob_path}'")
+ end
+ update = true
+ FileUtils.rm(blob_dst)
+ end
+
+ FileUtils.mkdir_p(File.dirname(blob_dst))
+ FileUtils.cp(local_path, blob_dst, :preserve => true)
+ FileUtils.chmod(0644, blob_dst)
+ if update
+ say("Updated #{blob_path.yellow}")
+ else
+ say("Added #{blob_path.yellow}")
+ end
+
+ say("When you are done testing the new blob, please run\n" +
+ "`#{"bosh upload blobs".green}' and commit changes.")
+ end
+
+ # Synchronizes the contents of blobs directory with blobs index.
+ # @return [void]
+ def sync
+ say("Syncing blobs...")
+ remove_symlinks
+ process_blobs_directory
+ process_index
+ end
+
+ # Processes all files in blobs directory and only leaves non-symlinks.
+ # Marks blobs as dirty if there are any non-symlink files.
+ # @return [void]
+ def process_blobs_directory
+ @updated_blobs = []
+ @new_blobs = []
+
+ Dir[File.join(@blobs_dir, "**", "*")].each do |file|
+ next if File.directory?(file) || File.symlink?(file)
+ # We don't care about symlinks because they represent blobs
+ # that are already tracked.
+ # Regular files are more interesting: it's either a new version
+ # of an existing blob or a completely new blob.
+ path = strip_blobs_dir(file)
+
+ if File.exists?(File.join(@src_dir, path))
+ err("File `#{path}' is in both `blobs' and `src' directory.\n" +
+ "Please fix release repo before proceeding")
+ end
+
+ if @index.has_key?(path)
+ if file_checksum(file) == @index[path]["sha"]
+ # Already have exactly the same file in the index,
+ # no need to keep it around. Also handles the migration
+ # scenario for people with old blobs checked out.
+ local_path = File.join(@storage_dir, @index[path]["sha"])
+ if File.exists?(local_path)
+ FileUtils.rm_rf(file)
+ else
+ FileUtils.mv(file, local_path)
+ end
+ install_blob(local_path, path, @index[path]["sha"])
+ else
+ @updated_blobs << path
+ end
+ else
+ @new_blobs << path
+ end
+ end
+ end
+
+ # Removes all symlinks from blobs directory
+ # @return [void]
+ def remove_symlinks
+ Dir[File.join(@blobs_dir, "**", "*")].each do |file|
+ FileUtils.rm_rf(file) if File.symlink?(file)
+ end
+ end
+
+ # Processes blobs index, fetches any missing or mismatched blobs,
+ # establishes symlinks in blobs directory to any files present in index.
+ # @return [void]
+ def process_index
+ @index.each_pair do |path, entry|
+ if File.exists?(File.join(@src_dir, path))
+ err("File `#{path}' is in both blob index and src directory.\n" +
+ "Please fix release repo before proceeding")
+ end
+
+ local_path = File.join(@storage_dir, entry["sha"])
+ need_download = true
+
+ if File.exists?(local_path)
+ checksum = file_checksum(local_path)
+ if checksum == entry["sha"]
+ need_download = false
+ else
+ progress(path, "checksum mismatch, re-downloading...\n".red)
+ end
+ end
+
+ if need_download
+ local_path = download_blob(path)
+ end
+
+ install_blob(local_path, path, entry["sha"])
+ end
+ end
+
+ # Uploads blob to a blobstore, updates blobs index.
+ # @param [String] path Blob path relative to blobs dir
+ def upload_blob(path)
+ blob_path = File.join(@blobs_dir, path)
+
+ unless File.exists?(blob_path)
+ err("Cannot upload blob, local file `#{blob_path}' doesn't exist")
+ end
+
+ if File.symlink?(blob_path)
+ err("`#{blob_path}' is a symlink")
+ end
+
+ object_id = rand(36**9).to_s(36)
+ checksum = file_checksum(blob_path)
+
+ progress(path, "uploading...")
+ object_id = @blobstore.create(File.open(blob_path, "r"))
+ progress(path, "uploaded\n".green)
+
+ @index[path] = {
+ "object_id" => object_id,
+ "sha" => checksum,
+ "size" => File.size(blob_path)
+ }
+
+ update_index
+ install_blob(blob_path, path, checksum)
+ object_id
+ end
+
+ # Downloads blob from a blobstore
+ # @param [String] path Downloaded blob file path
+ def download_blob(path)
+ unless @index.has_key?(path)
+ err("Unknown blob path `#{path}'")
+ end
+
+ blob = @index[path]
+ size = blob["size"].to_i
+ tmp_file = Tempfile.new("bosh-blob")
+
+ download_label = "downloading"
+ if size > 0
+ download_label += " " + pretty_size(size)
+ end
+
+ progress_bar = Thread.new do
+ loop do
+ break unless size > 0
+ if File.exists?(tmp_file.path)
+ pct = 100 * File.size(tmp_file.path).to_f / size
+ progress(path, "#{download_label} (#{pct.to_i}%)...")
+ end
+ sleep(0.2)
+ end
+ end
+
+ progress(path, "#{download_label}...")
+ @blobstore.get(blob["object_id"], tmp_file)
+ tmp_file.close
+ progress_bar.kill
+ progress(path, "downloaded\n".green)
+
+ if file_checksum(tmp_file.path) != blob["sha"]
+ err("Checksum mismatch for downloaded blob `#{path}'")
+ end
+
+ tmp_file.path
+ end
+
+ private
+
+ # Renders blob operation progress
+ # @param [String] path Blob path relative to blobs dir
+ # @param [String] label Operation happening to a blob
+ def progress(path, label)
+ say("\r", " " * 80)
+ say("\r#{path.truncate(40).yellow} #{label}", "")
+ end
+
+ # @param [String] src Path to a file containing the blob
+ # @param [String] dst Resulting blob path relative to blobs dir
+ # @param [String] checksum Blob checksum
+ def install_blob(src, dst, checksum)
+ store_path = File.join(@storage_dir, checksum)
+ symlink_path = File.join(@blobs_dir, dst)
+
+ FileUtils.chmod(0644, src)
+
+ unless File.exists?(store_path) && realpath(src) == realpath(store_path)
+ # Move blob to a storage dir if it's not there yet
+ FileUtils.mv(src, store_path)
+ end
+
+ unless File.exists?(symlink_path) && !File.symlink?(symlink_path)
+ FileUtils.mkdir_p(File.dirname(symlink_path))
+ FileUtils.rm_rf(symlink_path)
+ FileUtils.ln_s(store_path, symlink_path)
+ end
+ end
+
+ # Returns blob path relative to blobs dir, fails if blob is not in blobs
+ # dir.
+ # @param [String] path Absolute or relative blob path
+ def strip_blobs_dir(path)
+ blob_path = realpath(path)
+ blobs_dir = realpath(@blobs_dir)
+
+ if blob_path[0..blobs_dir.size] == blobs_dir + "/"
+ blob_path[blobs_dir.size+1..-1]
+ else
+ err("File `#{blob_path}' is not under `blobs' directory")
+ end
+ end
+
+ # Updates blobs index
+ def update_index
+ yaml = YAML.dump(@index).gsub(/\s*$/, "")
+
+ index_file = Tempfile.new("blob_index")
+ index_file.puts(yaml)
+ index_file.close
+
+ FileUtils.mv(index_file.path, @index_file)
+ end
+
+ # Returns file SHA1 checksum
+ # @param [String] path File path
+ def file_checksum(path)
+ Digest::SHA1.file(path).hexdigest
+ end
+
+ # Returns real file path (resolves symlinks)
+ # @param [String] path File path
+ def realpath(path)
+ Pathname.new(path).realpath.to_s
+ end
+ end
+end
View
78 cli/lib/cli/commands/base.rb
@@ -37,6 +37,10 @@ def release
@release = Bosh::Cli::Release.new(@work_dir)
end
+ def blob_manager
+ @blob_manager ||= Bosh::Cli::BlobManager.new(release)
+ end
+
def blobstore
release.blobstore
end
@@ -166,80 +170,6 @@ def normalize_url(url)
URI.parse(url).to_s
end
- def check_if_blobs_supported
- check_if_release_dir
- unless File.directory?(BLOBS_DIR)
- err("Can't find blob directory '#{BLOBS_DIR}'.")
- end
-
- unless File.file?(BLOBS_INDEX_FILE)
- err("Can't find '#{BLOBS_INDEX_FILE}'")
- end
- end
-
- def check_dirty_blobs
- if File.file?(File.join(work_dir, BLOBS_INDEX_FILE)) && blob_status != 0
- err("Your 'blobs' directory is not in sync. " +
- "Resolve using 'bosh sync blobs' command")
- end
- end
-
- def get_blobs_index
- load_yaml_file(File.join(work_dir, BLOBS_INDEX_FILE))
- end
-
- def blob_status(verbose = false)
- check_if_blobs_supported
- untracked = []
- modified = []
- tracked= []
- unsynced = []
-
- local_blobs = {}
- Dir.chdir(BLOBS_DIR) do
- Dir.glob("**/*").select { |entry| File.file?(entry) }.each do |file|
- local_blobs[file] = Digest::SHA1.file(file).hexdigest
- end
- end
- remote_blobs = get_blobs_index
-
- local_blobs.each do |blob_name, blob_sha|
- if remote_blobs[blob_name].nil?
- untracked << blob_name
- elsif blob_sha != remote_blobs[blob_name]["sha"]
- modified << blob_name
- else
- tracked << blob_name
- end
- end
-
- remote_blobs.each_key do |blob_name|
- unsynced << blob_name if local_blobs[blob_name].nil?
- end
-
- changes = modified.size + untracked.size + unsynced.size
- return changes unless verbose
-
- if modified.size > 0
- say("\nModified blobs ('bosh upload blob' to update): ".green)
- modified.each { |blob| say(blob) }
- end
-
- if untracked.size > 0
- say("\nNew blobs ('bosh upload blob' to add): ".green)
- untracked.each { |blob| say(blob) }
- end
-
- if unsynced.size > 0
- say("\nMissing blobs ('bosh sync blobs' to fetch) : ".green)
- unsynced.each { |blob| say(blob) }
- end
-
- if changes == 0
- say("\nRelease blobs are up to date".green)
- end
- changes
- end
end
end
end
View
125 cli/lib/cli/commands/blob.rb
@@ -1,125 +0,0 @@
-# Copyright (c) 2009-2012 VMware, Inc.
-
-module Bosh::Cli::Command
- class Blob < Base
-
- def upload_blob(*params)
- check_if_blobs_supported
- force = !params.delete("--force").nil?
-
- blobs = params.map{ |param| get_blob_name(param) }
- total = blobs.size
- blob_index = get_blobs_index
-
- blobs.each_with_index do |blob_name, idx|
- count = idx + 1
- blob_file = File.join(BLOBS_DIR, blob_name)
- blob_sha = Digest::SHA1.file(blob_file).hexdigest
-
- if blob_index[blob_name] && !force
- # We already have this binary on record
- if blob_index[blob_name]["sha"] == blob_sha
- say("[#{count}/#{total}] Skipping #{blob_name}".green)
- next
- end
- # Local copy is different from the remote copy
- if interactive?
- confirm = ask("\nBlob #{blob_name} changed, " +
- "do you want to update the binary [yN]: ")
- if confirm.empty? || !(confirm =~ /y(es)?$/i)
- say("[#{count}/#{total}] Skipping #{blob_name}".green)
- next
- end
- end
- end
-
- # TODO: We could use the sha and try to avoid
- # uploading duplicated objects.
- say("[#{count}/#{total}] Uploading #{blob_name}".green)
- blob_id = blobstore.create(File.open(blob_file, "r"))
- blob_index[blob_name] = { "object_id" => blob_id, "sha" => blob_sha }
- end
-
- # update the index file
- index_file = Tempfile.new("tmp_blob_index")
- dump_yaml_to_file(blob_index, index_file)
- index_file.close
- FileUtils.mv(index_file.path, File.join(work_dir, BLOBS_INDEX_FILE))
- end
-
- def sync_blobs(*options)
- check_if_blobs_supported
- force = options.include?("--force")
-
- blob_index = get_blobs_index
- total = blob_index.size
- count = 0
-
- blob_index.each_pair do |name, blob_info|
- count += 1
- blob_file = File.join(work_dir, BLOBS_DIR, name)
-
- # check if we have conflicting blobs
- if File.file?(blob_file) && !force
- blob_sha = Digest::SHA1.file(blob_file).hexdigest
- if blob_sha == blob_info["sha"]
- say("[#{count}/#{total}] Skipping blob #{name}".green)
- next
- end
-
- if interactive?
- confirm = ask("\nLocal blob (#{name}) conflicts with " +
- "remote object, overwrite local copy? [yN]: ")
- if confirm.empty? || !(confirm =~ /y(es)?$/i)
- say("[#{count}/#{total}] Skipping blob #{name}".green)
- next
- end
- end
- end
- say("[#{count}/#{total}] Updating #{blob_file}".green)
- fetch_blob(blob_file, blob_info)
- end
- end
-
- def blobs_info
- blob_status(true)
- end
-
- private
-
- # Sanity check the input file and returns the blob_name
- def get_blob_name(file)
- err("Invalid file #{file}") unless File.file?(file)
- blobs_dir = File.join(realpath(work_dir), "#{BLOBS_DIR}/")
- file_path = realpath(File.expand_path(file))
-
- if file_path[0..blobs_dir.length - 1] != blobs_dir
- err("#{file_path} is NOT under #{blobs_dir}")
- end
- file_path[blobs_dir.length..file_path.length]
- end
-
- # Download the blob (blob_info) into dst_file
- def fetch_blob(dst_file, blob_info)
- object_id = blob_info["object_id"]
-
- # fetch the blob
- new_blob = Tempfile.new("new_blob_file")
- blobstore.get(object_id, new_blob)
- new_blob.close
-
- if blob_info["sha"] != Digest::SHA1.file(new_blob.path).hexdigest
- err("Fatal error: " +
- "Inconsistent checksum for object #{blob_info["object_id"]}")
- end
-
- FileUtils.mkdir_p(File.dirname(dst_file))
- FileUtils.chmod(0644, new_blob.path)
- FileUtils.mv(new_blob.path, dst_file)
- end
-
- def realpath(path)
- Pathname.new(path).realpath.to_s
- end
- end
-end
View
47 cli/lib/cli/commands/blob_management.rb
@@ -0,0 +1,47 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Cli::Command
+ class BlobManagement < Base
+
+ # Prints out blobs status
+ def status
+ blob_manager.print_status
+ end
+
+ # Adds blob to managed blobs
+ # @param [String] local_path Local file path
+ # @param [optional, String] blob_dir Directory to store blob in, relative
+ # to blobs dir
+ def add(local_path, blob_dir = nil)
+ blob_path = File.basename(local_path)
+ if blob_dir
+ # We don't need about blobs prefix,
+ # but it might be handy for people who rely on auto-completion
+ if blob_dir[0..5] == "blobs/"
+ blob_dir = blob_dir[6..-1]
+ end
+ blob_path = File.join(blob_dir, blob_path)
+ end
+ blob_manager.add_blob(local_path, blob_path)
+ end
+
+ # Uploads all blobs that need to be uploaded
+ def upload
+ blob_manager.print_status
+
+ blob_manager.blobs_to_upload.each do |blob|
+ nl
+ if confirmed?("Upload blob #{blob.yellow}?")
+ blob_manager.upload_blob(blob)
+ end
+ end
+ end
+
+ # Syncs blobs with blobstore
+ def sync
+ blob_manager.sync
+ blob_manager.print_status
+ end
+
+ end
+end
View
13 cli/lib/cli/commands/release.rb
@@ -213,11 +213,18 @@ def create_from_spec(*options)
exit(1)
end
- if !force
- check_dirty_blobs
- check_if_dirty_state
+ blob_manager.sync
+ if blob_manager.dirty?
+ blob_manager.print_status
+ if force
+ say("Proceeding with dirty blobs as '--force' is given".red)
+ else
+ err("Please use '--force' or upload new blobs")
+ end
end
+ check_if_dirty_state unless force
+
confirmation = "Are you sure you want to " +
"generate #{'final'.red} version? "
View
24 cli/lib/cli/runner.rb
@@ -537,24 +537,28 @@ def define_commands
route :cloud_check, :perform
end
- command :upload_blob do
- usage "upload blob <blobs>"
- desc "Upload given blob to the blobstore"
- option "--force", "bypass duplicate checking"
- route :blob, :upload_blob
+ command :add_blob do
+ usage "add blob <local_path> [<blob_dir>]"
+ desc "Add a local file as BOSH blob"
+ route :blob_management, :add
+ end
+
+ command :upload_blobs do
+ usage "upload blobs"
+ desc "Upload new and updated blobs to the blobstore"
+ route :blob_management, :upload
end
command :sync_blobs do
usage "sync blobs"
desc "Sync blob with the blobstore"
- option "--force", "overwrite all local copies with the remote blob"
- route :blob, :sync_blobs
+ route :blob_management, :sync
end
- command :blobs do
+ command :blobs_status do
usage "blobs"
- desc "Print blob status"
- route :blob, :blobs_info
+ desc "Print current blobs status"
+ route :blob_management, :status
end
def define_plugin_commands
View
5 cli/lib/cli/templates/help_message.erb
@@ -70,6 +70,7 @@ Remote access
<%= command_usage(:ssh_cleanup) %>
Blob
-<%= command_usage(:upload_blob) %>
+<%= command_usage(:add_blob) %>
+<%= command_usage(:upload_blobs) %>
<%= command_usage(:sync_blobs) %>
-<%= command_usage(:blobs) %>
+<%= command_usage(:blobs_status) %>
View
2 cli/lib/cli/version.rb
@@ -2,6 +2,6 @@
module Bosh
module Cli
- VERSION = "0.16"
+ VERSION = "0.17"
end
end
View
290 cli/spec/unit/blob_manager_spec.rb
@@ -0,0 +1,290 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+require "spec_helper"
+
+describe Bosh::Cli::BlobManager do
+
+ def make_manager(release)
+ Bosh::Cli::BlobManager.new(release)
+ end
+
+ before(:each) do
+ @blobstore = mock("blobstore")
+ @dir = Dir.mktmpdir
+ @src_dir = FileUtils.mkdir(File.join(@dir, "src"))
+ @config_dir = File.join(@dir, "config")
+ FileUtils.mkdir(@config_dir)
+ @blobs_dir = File.join(@dir, "blobs")
+ @release = mock("release", :dir => @dir, :blobstore => @blobstore)
+ end
+
+ describe "initialization" do
+ it "fails if 'src' directory is missing" do
+ FileUtils.rm_rf(@src_dir)
+ expect {
+ make_manager(@release)
+ }.to raise_error("`src' directory is missing")
+ end
+
+ it "fails if blobstore is not configured" do
+ @release.stub!(:blobstore).and_return(nil)
+ expect {
+ make_manager(@release)
+ }.to raise_error("Blobstore is not configured")
+ end
+
+ it "creates necessary directories in release dir" do
+ make_manager(@release)
+ File.directory?(File.join(@dir, "blobs")).should be_true
+ File.directory?(File.join(@dir, ".blobs")).should be_true
+ end
+
+ it "has dirty flag cleared and upload list empty" do
+ manager = make_manager(@release)
+ manager.dirty?.should be_false
+ manager.blobs_to_upload.should == []
+ end
+
+ it "doesn't like bad index file'" do
+ File.open(File.join(@config_dir, "blobs.yml"), "w") do |f|
+ YAML.dump("string", f)
+ end
+ expect {
+ make_manager(@release)
+ }.to raise_error(/Incorrect file format/)
+ end
+
+ it "migrates legacy index file" do
+ legacy_file = File.join(@release.dir, "blob_index.yml")
+ test_hash = { "foo" => "bar" }
+
+ File.open(legacy_file, "w") do |f|
+ YAML.dump({ "foo" => "bar" }, f)
+ end
+
+ make_manager(@release)
+ File.exists?(legacy_file).should be_false
+ YAML.load_file(File.join(@config_dir, "blobs.yml")).should == test_hash
+ end
+ end
+
+ describe "adding a blob" do
+ before(:each) do
+ @manager = make_manager(@release)
+ @blob = Tempfile.new("blob")
+ @blob.write("blob contents")
+ @blob.close
+ end
+
+ it "cannot add non-existing file" do
+ expect {
+ @manager.add_blob("tmp/foobar.tgz", "test")
+ }.to raise_error("File `tmp/foobar.tgz' not found")
+ end
+
+ it "cannot add directory" do
+ tmp_dir = Dir.mktmpdir
+ expect {
+ @manager.add_blob(tmp_dir, "test")
+ }.to raise_error("`#{tmp_dir}' is a directory")
+ end
+
+ it "cannot use absolute path as blob destination" do
+ expect {
+ @manager.add_blob(@blob.path, "/test")
+ }.to raise_error("Blob path should be a relative path")
+ end
+
+ it "cannot use 'blobs' prefix for blob destination" do
+ expect {
+ @manager.add_blob(@blob.path, "blobs/foo/bar")
+ }.to raise_error("Blob path should not start with `blobs/'")
+ end
+
+ it "cannot use directory as blob destination" do
+ foo_dir = File.join(@blobs_dir, "foo")
+ FileUtils.mkdir(foo_dir)
+ expect {
+ @manager.add_blob(@blob.path, "foo")
+ }.to raise_error("`#{foo_dir}' is a directory, " +
+ "please pick a different path")
+ end
+
+ it "adds blob to a blobs directory" do
+ blob_dst = File.join(@blobs_dir, "foo", "blob")
+ @manager.add_blob(@blob.path, "foo/blob")
+ File.exists?(blob_dst).should be_true
+ File.read(blob_dst).should == "blob contents"
+ File.symlink?(blob_dst).should be_false
+ File.stat(blob_dst).mode.to_s(8)[-4..-1].should == "0644"
+ File.exists?(@blob.path).should be_true # original still exists
+
+ @manager.process_blobs_directory
+ @manager.dirty?.should be_true
+ @manager.new_blobs.should == %w(foo/blob)
+ @manager.updated_blobs.should == []
+ end
+
+ it "prevents double adds of the same file" do
+ @manager.add_blob(@blob.path, "foo/blob")
+ expect {
+ @manager.add_blob(@blob.path, "foo/blob")
+ }.to raise_error(/Already tracking/)
+ end
+
+ it "updates blob" do
+ new_blob = Tempfile.new("new-blob")
+ new_blob.write("foobar")
+ new_blob.close
+ blob_dst = File.join(@blobs_dir, "foo", "blob")
+ @manager.add_blob(@blob.path, "foo/blob")
+ File.read(blob_dst).should == "blob contents"
+ @manager.add_blob(new_blob.path, "foo/blob")
+ File.read(blob_dst).should == "foobar"
+ end
+ end
+
+ describe "downloading a blob" do
+ it "cannot download blob if path is not in index" do
+ @manager = make_manager(@release)
+
+ expect {
+ @manager.download_blob("foo")
+ }.to raise_error(/Unknown blob path/)
+ end
+
+ it "downloads blob from blobstore" do
+ index = {
+ "foo" => {
+ "size" => "1000",
+ "object_id" => "deadbeef",
+ "sha" => Digest::SHA1.hexdigest("blob contents")
+ }
+ }
+
+ tmp_file = Tempfile.new("mock-blob")
+ Tempfile.stub!(:new).with("bosh-blob").and_return(tmp_file)
+
+ File.open(File.join(@config_dir, "blobs.yml"), "w") do |f|
+ YAML.dump(index, f)
+ end
+
+ @manager = make_manager(@release)
+ @blobstore.should_receive(:get).with("deadbeef", tmp_file).
+ and_return { tmp_file.write("blob contents"); tmp_file.close }
+
+ path = @manager.download_blob("foo")
+ File.read(path).should == "blob contents"
+ end
+ end
+
+ describe "uploading a blob" do
+ before(:each) do
+ @manager = make_manager(@release)
+ end
+
+ it "needs blob path to exist" do
+ expect {
+ @manager.upload_blob("foo")
+ }.to raise_error(/doesn't exist/)
+ end
+
+ it "doesn't follow symlinks" do
+ FileUtils.touch(File.join(@dir, "blob"))
+ FileUtils.ln_s(File.join(@dir, "blob"), File.join(@blobs_dir, "foo"))
+ expect {
+ @manager.upload_blob("foo")
+ }.to raise_error(/is a symlink/)
+ end
+
+ it "uploads file to a blobstore, updates index and symlinks blob" do
+ new_blob = File.join(@dir, "blob")
+ File.open(new_blob, "w") { |f| f.write("test blob") }
+ @manager.add_blob(new_blob, "foo")
+
+ @blobstore.should_receive(:create).and_return("deadbeef")
+ @manager.upload_blob("foo").should == "deadbeef"
+
+ blob_dst = File.join(@blobs_dir, "foo")
+ checksum = Digest::SHA1.hexdigest("test blob")
+
+ File.symlink?(blob_dst).should be_true
+ File.readlink(blob_dst).should == File.join(@dir, ".blobs", checksum)
+ File.read(blob_dst).should == "test blob"
+ end
+ end
+
+ describe "syncing blobs" do
+ it "includes several steps" do
+ @manager = make_manager(@release)
+ @manager.should_receive(:remove_symlinks).ordered
+ @manager.should_receive(:process_blobs_directory).ordered
+ @manager.should_receive(:process_index).ordered
+ @manager.sync
+ end
+
+ it "processes blobs directory" do
+ @manager = make_manager(@release)
+ @blobstore.stub!(:create).and_return("new-object-id")
+
+ new_blob = Tempfile.new("new-blob")
+ new_blob.write("test")
+ new_blob.close
+
+ @manager.add_blob(new_blob.path, "foo")
+ @manager.process_blobs_directory
+ @manager.new_blobs.should == %w(foo)
+
+ @manager.add_blob(new_blob.path, "bar")
+ @manager.process_blobs_directory
+ @manager.new_blobs.sort.should == %w(bar foo)
+
+ @manager.upload_blob("bar")
+
+ new_blob.open
+ new_blob.write("stuff")
+ new_blob.close
+
+ @manager.add_blob(new_blob.path, "bar")
+ @manager.process_blobs_directory
+ @manager.new_blobs.sort.should == %w(foo)
+ @manager.updated_blobs.sort.should == %w(bar)
+ end
+
+ it "downloads missing blobs" do
+ index = {
+ "foo" => {
+ "size" => 1000,
+ "sha" => Digest::SHA1.hexdigest("foo"),
+ "object_id" => "da-foo"
+ },
+ "bar" => {
+ "size" => 500,
+ "sha" => Digest::SHA1.hexdigest("bar"),
+ "object_id" => "da-bar"
+ }
+ }
+
+ File.open(File.join(@config_dir, "blobs.yml"), "w") do |f|
+ YAML.dump(index, f)
+ end
+
+ foo = Tempfile.new("foo")
+ foo.write("foo")
+ foo.close
+
+ bar = Tempfile.new("bar")
+ bar.write("bar")
+ bar.close
+
+ @manager = make_manager(@release)
+ @manager.should_receive(:download_blob).with("foo").and_return(foo.path)
+ @manager.should_receive(:download_blob).with("bar").and_return(bar.path)
+
+ @manager.process_index
+
+ File.read(File.join(@blobs_dir, "foo")).should == "foo"
+ File.read(File.join(@blobs_dir, "bar")).should == "bar"
+ end
+ end
+end
View
207 cli/spec/unit/cli_commands_spec.rb
@@ -6,8 +6,8 @@
before :each do
@config = File.join(Dir.mktmpdir, "bosh_config")
- @cache = File.join(Dir.mktmpdir, "bosh_cache")
- @opts = { :config => @config, :cache_dir => @cache }
+ @cache = File.join(Dir.mktmpdir, "bosh_cache")
+ @opts = { :config => @config, :cache_dir => @cache }
end
describe Bosh::Cli::Command::Misc do
@@ -300,181 +300,48 @@
end
- describe Bosh::Cli::Command::Blob do
+ describe Bosh::Cli::Command::BlobManagement do
before :each do
- @cmd = Bosh::Cli::Command::Blob.new(@opts)
- @release_dir = Dir.mktmpdir
- @blob_dir = File.join(@release_dir, "blobs/")
- @cmd.stub!(:work_dir).and_return(@release_dir)
- @blobstore = mock("blobstore")
- @cmd.stub!(:blobstore).and_return(@blobstore)
- FileUtils.mkdir(@blob_dir)
+ @cmd = Bosh::Cli::Command::BlobManagement.new(@opts)
+ @blob_manager = mock("blob manager")
+ @release = mock("release")
+
+ @cmd.should_receive(:check_if_release_dir)
+ Bosh::Cli::Release.stub!(:new).and_return(@release)
+ Bosh::Cli::BlobManager.stub!(:new).with(@release).
+ and_return(@blob_manager)
end
- after :each do
- FileUtils.rm_rf(@release_dir)
+ it "prints blobs status" do
+ @blob_manager.should_receive(:print_status)
+ @cmd.status
end
- it "refuse to run outside of the release directory" do
- lambda {
- @cmd.upload_blob("foo")
- }.should raise_error(Bosh::Cli::CliExit,
- "Sorry, your current directory doesn't " +
- "look like release directory".red)
+ it "adds blob under provided directory" do
+ @blob_manager.should_receive(:add_blob).with("foo/bar.tgz", "bar/bar.tgz")
+ @cmd.add("foo/bar.tgz", "bar")
+ end
- lambda {
- @cmd.sync_blobs
- }.should raise_error(Bosh::Cli::CliExit,
- "Sorry, your current directory doesn't " +
- "look like release directory".red)
+ it "adds blob with no directory provided" do
+ @blob_manager.should_receive(:add_blob).with("foo/bar.tgz", "bar.tgz")
+ @cmd.add("foo/bar.tgz")
+ end
- lambda {
- @cmd.blobs_info
- }.should raise_error(Bosh::Cli::CliExit,
- "Sorry, your current directory doesn't " +
- "look like release directory".red)
- end
-
- it "refuse to upload blob outside of release/blobs" do
- Dir.chdir(@release_dir) do
- FileUtils.touch("test.tgz")
- @cmd.should_receive(:check_if_blobs_supported).and_return(true)
- blob_path = Pathname.new(@release_dir).realpath.to_s
- blobs_dir = Pathname.new(@blob_dir).realpath.to_s
- lambda {
- @cmd.upload_blob("test.tgz")
- }.should raise_error(Bosh::Cli::CliExit,
- "#{File.join(blob_path, "test.tgz")} is " +
- "NOT under #{blobs_dir}/")
- end
- end
-
- it "upload new blob to blobstore" do
- Dir.chdir(@release_dir) do
- FileUtils.mkdir("./blobs/test")
- blob = FileUtils.touch("./blobs/test/test.tgz")
- @cmd.should_receive(:get_blobs_index).and_return({})
- @cmd.should_receive(:check_if_blobs_supported).and_return(true)
- @blobstore.should_receive(:create).and_return(2)
- @cmd.upload_blob("./blobs/test/test.tgz")
- YAML.load_file("blob_index.yml").should == {
- "test/test.tgz" => {
- "object_id" => 2,
- "sha" => Digest::SHA1.file(blob.first).hexdigest
- }
- }
- end
- end
-
- it "should skip upload if blob already exists" do
- Dir.chdir(@release_dir) do
- FileUtils.mkdir("./blobs/test")
- blob = FileUtils.touch("./blobs/test/test.tgz")
- @cmd.should_receive(:check_if_blobs_supported).and_return(true)
- @cmd.should_receive(:get_blobs_index).and_return('test/test.tgz' => {
- "sha" => Digest::SHA1.file(blob.first).hexdigest,
- "object_id" => 2
- })
- @blobstore.should_not_receive(:create)
- @cmd.upload_blob("./blobs/test/test.tgz")
- end
- end
-
- it "should ask user if blob was modified" do
- Dir.chdir(@release_dir) do
- FileUtils.mkdir("./blobs/test")
- blob = FileUtils.touch("./blobs/test/test.tgz")
- @cmd.should_receive(:check_if_blobs_supported).and_return(true)
- @cmd.should_receive(:get_blobs_index).and_return('test/test.tgz' => {
- "sha" => 1,
- "object_id" => 2
- })
- @cmd.should_receive(:ask).and_return("no")
- @blobstore.should_not_receive(:create)
- @cmd.upload_blob("./blobs/test/test.tgz")
- end
- end
-
- it "should sync if file is not present" do
- Dir.chdir(@release_dir) do
- @cmd.should_receive(:check_if_blobs_supported).and_return(true)
- @cmd.should_receive(:get_blobs_index).and_return('test/test.tgz' => {
- "sha" => 1,
- "object_id" => 2
- })
- @cmd.should_receive(:fetch_blob).
- with(File.join(@release_dir, "blobs", "test/test.tgz"),
- { "sha" => 1, "object_id" => 2 })
- @cmd.sync_blobs
- end
- end
-
- it "should not sync if the same file is present" do
- Dir.chdir(@release_dir) do
- FileUtils.mkdir("./blobs/test")
- blob = FileUtils.touch("./blobs/test/test.tgz")
- @cmd.should_receive(:check_if_blobs_supported).and_return(true)
- @cmd.should_receive(:get_blobs_index).and_return('test/test.tgz' => {
- "sha" => Digest::SHA1.file(blob.first).hexdigest,
- "object_id" => 2
- })
- @cmd.should_not_receive(:fetch_blob)
- @cmd.sync_blobs
- end
- end
-
- it "should ask user if blob sha is different" do
- Dir.chdir(@release_dir) do
- FileUtils.mkdir("./blobs/test")
- blob = FileUtils.touch("./blobs/test/test.tgz")
- @cmd.should_receive(:check_if_blobs_supported).and_return(true)
- @cmd.should_receive(:get_blobs_index).and_return('test/test.tgz' => {
- "sha" => 1,
- "object_id" => 2
- })
- @cmd.should_receive(:ask).and_return("")
- @cmd.should_not_receive(:fetch_blob)
- @cmd.sync_blobs
- end
- end
-
- it "reports untracked blobs" do
- Dir.chdir(@release_dir) do
- FileUtils.mkdir("./blobs/test")
- FileUtils.touch("./blobs/test/test.tgz")
- @cmd.should_receive(:check_if_blobs_supported).and_return(true)
- @cmd.should_receive(:get_blobs_index).and_return({})
- @cmd.should_receive(:say).
- with("\nNew blobs ('bosh upload blob' to add): ".green)
- @cmd.should_receive(:say).with("test/test.tgz")
- @cmd.blobs_info
- end
- end
-
- it "reports modified blobs" do
- Dir.chdir(@release_dir) do
- FileUtils.mkdir("./blobs/test")
- FileUtils.touch("./blobs/test/test.tgz")
- @cmd.should_receive(:check_if_blobs_supported).and_return(true)
- @cmd.should_receive(:get_blobs_index).
- and_return({ "test/test.tgz" => { "sha" => 1, "object_id" => 2}})
- @cmd.should_receive(:say).
- with("\nModified blobs ('bosh upload blob' to update): ".green)
- @cmd.should_receive(:say).with("test/test.tgz")
- @cmd.blobs_info
- end
- end
-
- it "reports unsynced blobs" do
- Dir.chdir(@release_dir) do
- @cmd.should_receive(:check_if_blobs_supported).and_return(true)
- @cmd.should_receive(:get_blobs_index).
- and_return({ "test/test.tgz" => { "sha" => 1, "object_id" => 2}})
- @cmd.should_receive(:say).
- with("\nMissing blobs ('bosh sync blobs' to fetch) : ".green)
- @cmd.should_receive(:say).with("test/test.tgz")
- @cmd.blobs_info
- end
+ it "uploads blobs" do
+ @blob_manager.should_receive(:print_status)
+ @blob_manager.stub!(:blobs_to_upload).and_return(%w(foo bar baz))
+ @blob_manager.should_receive(:upload_blob).with("foo")
+ @blob_manager.should_receive(:upload_blob).with("bar")
+ @blob_manager.should_receive(:upload_blob).with("baz")
+
+ @cmd.should_receive(:confirmed?).exactly(3).times.and_return(true)
+ @cmd.upload
+ end
+
+ it "syncs blobs" do
+ @blob_manager.should_receive(:sync).ordered
+ @blob_manager.should_receive(:print_status).ordered
+ @cmd.sync
end
end
View
5 cli/spec/unit/runner_spec.rb
@@ -93,6 +93,11 @@ def test_cmd(args, namespace, action, cmd_args = [])
test_cmd(["tasks"], :task, :list_running)
test_cmd(["tasks", "recent"], :task, :list_recent)
test_cmd(["tasks", "recent", "42"], :task, :list_recent, ["42"])
+
+ test_cmd(%w(blobs), :blob_management, :status)
+ test_cmd(%w(add blob foo bar), :blob_management, :add, %w(foo bar))
+ test_cmd(%w(upload blobs), :blob_management, :upload)
+ test_cmd(%w(sync blobs), :blob_management, :sync)
end
it "cancels running task and quits when ctrl-c is issued " +
View
5 spec/cli_spec.rb
@@ -14,6 +14,11 @@ def expect_output(cmd, expected_output)
format_output(run_bosh(cmd)).should == format_output(expected_output)
end
+ it "shows help message" do
+ run_bosh("help")
+ $?.should == 0
+ end
+
it "shows status" do
expect_output("status", <<-OUT)
Target not set

0 comments on commit 6926aa2

Please sign in to comment.