Permalink
Browse files

Public stemcell uploader.

Change-Id: I86e73d21329fea8eb165ed99d97eebe214111bcd
  • Loading branch information...
1 parent a2e6738 commit 7369cf0bde715923c48fecb924bf8c1415a3691e @lisbakke lisbakke committed with olegshaldybin Mar 22, 2012
View
@@ -0,0 +1 @@
+.stemcell_builds/public_stemcell_config.yml
@@ -0,0 +1,20 @@
+---
+public_stemcells_index.yml:
+ object_id: eyJvaWQiOiI0ZTRlNzhiY2EyMWUxMjEyMDRlNGU4NmVlMTUxYmMwNGY2YTE5%0AY2U0NmIyMiIsInNpZyI6Ik5KdUFyOWM4ZU9pZDdkS0ZtT0VON2JtekFsST0i%0AfQ==%0A
+ url: https://blob.cfblob.com/rest/objects/4e4e78bca21e121204e4e86ee151bc04f6a19ce46b22?uid=bb6a0c89ef4048a8a0f814e25385d1c5/user1&expires=1893484800&signature=NJuAr9c8eOid7dKFmOEN7bmzAlI=
+ sha: c616046a1d7e7a942b9e98ecde314b789bc3e263
+bosh-stemcell-0.3.0.tgz:
+ object_id: eyJvaWQiOiI0ZTRlNzhiY2E0MWUxMjIyMDRlNGU5ODYzZDA3NjMwNGY2Yjdj%0AMGYwMGFmYiIsInNpZyI6Iks1VnVVVFVqYzBEcVM0ejZhUnRxUlBzbXcxdz0i%0AfQ==%0A
+ url: https://blob.cfblob.com/rest/objects/4e4e78bca41e122204e4e9863d076304f6b7c0f00afb?uid=bb6a0c89ef4048a8a0f814e25385d1c5/user1&expires=1893484800&signature=K5VuUTUjc0DqS4z6aRtqRPsmw1w=
+ sha: 283be73063d30482c8a9e68f0aa80532207e172d
+ size: 304976057
+bosh-stemcell-0.4.4.tgz:
+ object_id: eyJvaWQiOiI0ZTRlNzhiY2E1MWUxMjEwMDRlNGU3ZDUxOTA2Y2QwNGY2Yjdm%0AYzgxNjFiOCIsInNpZyI6IlZMOUYybktzZ1ZsbWZYakY0blBISkg0cjE0ND0i%0AfQ==%0A
+ url: https://blob.cfblob.com/rest/objects/4e4e78bca51e121004e4e7d51906cd04f6b7fc8161b8?uid=bb6a0c89ef4048a8a0f814e25385d1c5/user1&expires=1893484800&signature=VL9F2nKsgVlmfXjF4nPHJH4r144=
+ sha: ef7dbe0a9dfeff52b5bab03c7c5f9e859e6ce5e8
+ size: 282851221
+bosh-stemcell-0.4.7.tgz:
+ object_id: eyJvaWQiOiI0ZTRlNzhiY2EyMWUxMjIwMDRlNGU4ZWM2NDdhNTQwNGY2Yjhh%0AZDY0ZDNjOCIsInNpZyI6Ik9oQkhLejV0OVNERlZFQkhoemVqeW5taDEyMD0i%0AfQ==%0A
+ url: https://blob.cfblob.com/rest/objects/4e4e78bca21e122004e4e8ec647a5404f6b8ad64d3c8?uid=bb6a0c89ef4048a8a0f814e25385d1c5/user1&expires=1893484800&signature=OhBHKz5t9SDFVEBHhzejynmh120=
+ sha: 25597777a8e82d3e95ce1890ead8973b1ceab22d
+ size: 283901280
View
@@ -1,18 +1,20 @@
source :rubygems
-gem "rake"
+gem "blobstore_client", "~> 0.3.11"
+gem "highline"
+gem "monit_api"
gem "nats", "=0.4.22.beta.4"
-gem "yajl-ruby", :require => "yajl"
-gem "uuidtools"
gem "netaddr"
-gem "blobstore_client", "~> 0.3.11"
gem "posix-spawn"
-gem "monit_api"
-gem "thin"
-gem "sinatra"
+gem "rake"
gem "rack-test"
+gem "ruby-atmos-pure"
gem "sigar", ">=0.7.2"
gem "bosh_encryption", ">=0.0.3"
+gem "sinatra"
+gem "thin"
+gem "uuidtools"
+gem "yajl-ruby", :require => "yajl"
group :development do
gem "guard"
View
@@ -34,6 +34,7 @@ GEM
guard (>= 0.2.2)
guard-rspec (0.6.0)
guard (>= 0.10.0)
+ highline (1.6.11)
httpclient (2.2.4)
json (1.6.5)
json_pure (1.6.4)
@@ -116,6 +117,7 @@ DEPENDENCIES
guard
guard-bundler
guard-rspec
+ highline
monit_api
nats (= 0.4.22.beta.4)
netaddr
@@ -124,6 +126,7 @@ DEPENDENCIES
rake
rcov
rspec
+ ruby-atmos-pure
ruby-debug
ruby-debug19
ruby_gntp
View
@@ -15,8 +15,12 @@ begin
rescue LoadError
end
+require "atmos"
require "bundler_task"
require "ci_task"
+require "highline/import"
+require "json"
+require "yaml"
BundlerTask.new
View
@@ -442,4 +442,246 @@ namespace "stemcell" do
build_vm_image(:part_in => "misc/#{component}/part.in")
end
+ namespace "public" do
+
+ # If the user is trying to upload a new file to the public repository then
+ # this function will determine if the file is already uploaded. If it is
+ # and the file to upload is an exact duplicate, the program will exit. If
+ # it exists but the user is trying to upload a new version it will prompt
+ # the user to overwrite the existing one.
+ # @param [Hash] index_yaml The index file as a Hash.
+ # @param [String] stemcell_path The path to the stemcell the user wants to
+ # upload.
+ # @return [Boolean] Returns whether this upload is an update operation.
+ def is_update?(index_yaml, stemcell_path)
+ if index_yaml.has_key?(File.basename(stemcell_path))
+ entry = index_yaml[File.basename(stemcell_path)]
+ if entry["sha"] == Digest::SHA1.file(stemcell_path).hexdigest
+ puts("No action taken, files are identical.")
+ exit(0)
+ end
+ if agree("Stemcell already uploaded. Do you want to overwrite it? " +
+ "[yn]")
+ return true
+ else
+ exit(0)
+ end
+ end
+ return false
+ end
+
+ # Loads the public stemcell uploader configuration file.
+ # @return [Array] An array of the pertinent configuration file parameters:
+ # stemcells_index_id, atmos_url, expiration, uid, secret
+ def load_stemcell_config
+ unless File.exists?("#{INDEX_FILE_DIR}/public_stemcell_config.yml")
+ raise "#{INDEX_FILE_DIR}/public_stemcell_config.yml does not exist."
+ end
+ cfg = YAML.load_file("#{INDEX_FILE_DIR}/public_stemcell_config.yml")
+ [cfg["stemcells_index_id"], cfg["atmos_url"], cfg["expiration"],
+ cfg["uid"], cfg["secret"]]
+ end
+
+ # Gets the public stemcell index file from the blobstore.
+ # @param [Atmos::Store] store The atmos store.
+ # @param [String] stemcells_index_id The object ID of the index file.
+ # @return [Array] An array of the index file and it's YAML as a Hash.
+ def get_index_file(store, stemcells_index_id)
+ index_file = store.get(:id => decode_object_id(stemcells_index_id)["oid"])
+ index_yaml = YAML.load(index_file.data)
+ index_yaml = index_yaml.is_a?(Hash) ? index_yaml : {}
+ [index_file, index_yaml]
+ end
+
+ # Changes all of the base shareable URLs in the index file. E.x. if the
+ # index file has www.vmware.com and the configuration file has 172.168.1.1
+ # then it will all be changed to 172.168.1.1 urls.
+ # @param [Hash] yaml The index file as a hash.
+ # @param [String] url The new URL.
+ # @return [Hash] The new YAML as a Hash.
+ def change_all_urls(yaml, url)
+ yaml.each do |filename, file_info|
+ file_info["url"] = file_info["url"].sub(/(https?:\/\/[^\/]*)/, url)
+ end
+ yaml
+ end
+
+ # Updates the index file in the blobstore and locally with the most recent
+ # changes.
+ # @param [Hash] yaml The index file as a hash.
+ # @param [String] url The new URL.
+ def update_index_file(stemcell_index, yaml, url)
+ yaml = change_all_urls(yaml, url)
+ yaml_dump = YAML.dump(yaml)
+
+ File.open("#{INDEX_FILE_DIR}/#{INDEX_FILE_NAME}", "w") do |f|
+ f.write(yaml_dump)
+ end
+
+ stemcell_index.update(yaml_dump)
+ puts("***Commit #{INDEX_FILE_DIR}/#{INDEX_FILE_NAME} to git repository " +
+ "immediately.***")
+ end
+
+ # A helper function to get the shareable URL for an entry in the blobstore.
+ # @param [String] oid The object ID.
+ # @param [String] sig The signature.
+ # @param [String] url The base url (e.g. www.vmware.com).
+ # @param [String] exp The expiration as an epoch time stamp.
+ # @param [String] uid The user id.
+ # @return [String] The shareable URL.
+ def get_shareable_url(oid, sig, url, exp, uid)
+ return url + "/rest/objects/#{oid}?uid=#{uid}&expires=#{exp}&signature=" +
+ "#{URI::escape(sig)}"
+ end
+
+ # Decodes the object ID.
+ # @param [String] object_id The object ID.
+ # @return [Hash] A hash with the oid and sig for the object_id.
+ def decode_object_id(object_id)
+ begin
+ object_info = JSON.load(Base64.decode64(URI::unescape(object_id)))
+ rescue JSON::ParserError => e
+ raise "Failed to parse object_id '#{object_id}'"
+ end
+
+ if !object_info.kind_of?(Hash) || object_info["oid"].nil? ||
+ object_info["sig"].nil?
+ raise "Failed to parse object_id '#{object_id}'"
+ end
+ object_info
+ end
+
+ # Encodes an object ID with an expiration, uid and secret.
+ # @param [String] object_id The object ID.
+ # @param [String] exp The expiration as an epoch time stamp.
+ # @param [String] uid The user id.
+ # @param [String] secret The secret.
+ # @return [String] The encoded object_id.
+ def encode_object_id(object_id, exp, uid, secret)
+ hash_string = "GET\n/rest/objects/#{object_id}\n#{uid}\n#{exp.to_s}"
+ sig = HMAC::SHA1.digest(Base64.decode64(secret), hash_string)
+ signature = Base64.encode64(sig.to_s).chomp
+ URI::escape(Base64.encode64(JSON.dump(:oid => object_id,
+ :sig => signature)))
+ end
+
+ INDEX_FILE_NAME = "public_stemcells_index.yml"
+ INDEX_FILE_DIR = ".stemcell_builds"
+
+ desc "Deletes <stemcell_name> from the public repository."
+ task "delete", :stemcell_name do |t, args|
+ stemcell_name = args[:stemcell_name]
+ stemcells_index_id, url, expiration, uid, secret = load_stemcell_config
+
+ store = Atmos::Store.new(:url => url, :uid => uid, :secret => secret)
+
+ (index_file, index_yaml) = get_index_file(store, stemcells_index_id)
+
+ unless index_yaml.has_key?(File.basename(stemcell_name))
+ names = []
+ index_yaml.each do |k, v|
+ names << k
+ end
+ puts("Stemcell '#{stemcell_name}' is not in [#{names.join(', ')}]")
+ return
+ end
+
+ if stemcell_name[INDEX_FILE_NAME]
+ puts("Nice try knucklehead. You can't delete this.'")
+ return
+ end
+
+ encoded_id = index_yaml[stemcell_name]["object_id"]
+ begin
+ output = store.get(:id => decode_object_id(encoded_id)["oid"])
+ output.delete
+ rescue => e
+
+ end
+ index_yaml.delete(stemcell_name)
+ update_index_file(index_file, index_yaml, url)
+ puts("Deleted #{stemcell_name}.")
+ end
+
+ desc "Uploads <stemcell_path> to the public repository."
+ task "upload", :stemcell_path do |t, args|
+ stemcell_path = args[:stemcell_path]
+
+ stemcells_index_id, url, expiration, uid, secret = load_stemcell_config
+
+ store = Atmos::Store.new(:url => url, :uid => uid, :secret => secret)
+
+ index_file, index_yaml = get_index_file(store, stemcells_index_id)
+
+ stemcell = File.open(stemcell_path, "r")
+ begin
+ if is_update?(index_yaml, stemcell_path)
+ entry = index_yaml[File.basename(stemcell_path)]
+ key = entry["object_id"]
+ output = store.get(:id => decode_object_id(key)["oid"])
+ output.update(stemcell)
+ puts("Updated #{stemcell_path}.")
+ else
+ output = store.create(:data => stemcell,
+ :length => File.size(stemcell_path))
+ puts("Uploaded #{stemcell_path}.")
+ end
+ encoded_id = encode_object_id(output.aoid, expiration, uid, secret)
+ object_info = decode_object_id(encoded_id)
+ oid = object_info["oid"]
+ sig = object_info["sig"]
+
+ index_yaml[File.basename(stemcell_path)] = {
+ "object_id" => encoded_id,
+ "url" => get_shareable_url(oid, sig, url, expiration, uid),
+ "sha" => Digest::SHA1.file(stemcell_path).hexdigest,
+ "size" => File.size(stemcell_path)
+ }
+
+ update_index_file(index_file, index_yaml, url)
+ ensure
+ stemcell.close
+ end
+ end
+
+ desc "Updates all stemcell's base URL with whatever is in " +
+ "public_stemcell_config.yml."
+ task "update_urls" do
+ stemcells_index_id, url, expiration, uid, secret = load_stemcell_config
+
+ store = Atmos::Store.new(:url => url, :uid => uid, :secret => secret)
+
+ index_file, index_yaml = get_index_file(store, stemcells_index_id)
+
+ update_index_file(index_file, index_yaml, url)
+ end
+
+ desc "Downloads the index file for debugging."
+ task "download_index_file" do
+ stemcells_index_id, url, expiration, uid, secret = load_stemcell_config
+
+ store = Atmos::Store.new(:url => url, :uid => uid, :secret => secret)
+
+ index_file, index_yaml = get_index_file(store, stemcells_index_id)
+
+ File.open("#{INDEX_FILE_DIR}/#{INDEX_FILE_NAME}", "w") do |f|
+ f.write(YAML.dump(index_yaml))
+ end
+
+ puts("Downloaded to #{INDEX_FILE_DIR}/#{INDEX_FILE_NAME}.")
+ end
+
+ desc "Uploads your local index file in case of emergency."
+ task "upload_index_file" do
+ if agree("Are you sure you want to upload your " +
+ "public_stemcell_config.yml over the existing one?")
+ yaml = YAML.load_file("#{INDEX_FILE_DIR}/#{INDEX_FILE_NAME}")
+ stemcells_index_id, url, expiration, uid, secret = load_stemcell_config
+ store = Atmos::Store.new(:url => url, :uid => uid, :secret => secret)
+ index_file, index_yaml = get_index_file(store, stemcells_index_id)
+ update_index_file(index_file, yaml, url)
+ end
+ end
+ end
end
Binary file not shown.

0 comments on commit 7369cf0

Please sign in to comment.