Permalink
Browse files

Merge "post data via REST, instead of vmc tunnel" into next

  • Loading branch information...
pxie authored and Gerrit Code Review committed May 18, 2012
2 parents 79fccd0 + 97bd9c5 commit b5e539da416008e12a807f6824c5bf737af5711a
Showing with 80 additions and 23 deletions.
  1. +1 −1 lib/harness.rb
  2. +79 −22 tools/scripts/assets_helper.rb
View
@@ -23,7 +23,7 @@ module Harness
"../.assets-binaries")
VCAP_BVT_ASSETS_PACKAGES_MANIFEST = File.join(VCAP_BVT_ASSETS_PACKAGES_HOME,
"packages.yml")
- VCAP_BVT_ASSETS_STORE_URL = "http://bolbs.cloudfoundry.com"
+ VCAP_BVT_ASSETS_STORE_URL = "http://blobs-next.cloudfoundry.com"
end
end
@@ -11,7 +11,7 @@
module Tools
module AssetsHelper
- include Interactive, BVT::Harness::ColorHelpers
+ include Interactive, BVT::Harness::ColorHelpers, BVT::Harness::HTTP_RESPONSE_CODE
def update_local_hash
if Dir.exist?(VCAP_BVT_ASSETS_PACKAGES_HOME)
@@ -34,31 +34,24 @@ def update_local_hash
VCAP_BVT_GRIDFS_COLLECTION = 'fs.files'
def upload_assets
datastore_config = YAML.load_file(VCAP_BVT_ASSETS_DATASTORE_CONFIG)
- conn = Mongo::Connection.new(datastore_config['host'], datastore_config['port'])
- db = conn.db(datastore_config['dbname'])
- auth = db.authenticate(datastore_config['username'], datastore_config['password'])
+ assets = list_binaries(datastore_config[:list])
puts "check assets storage server"
uploads = YAML.load_file(VCAP_BVT_ASSETS_PACKAGES_MANIFEST)['packages']
- grid = Mongo::Grid.new(db)
- if db.collection_names.include?(VCAP_BVT_GRIDFS_COLLECTION)
- coll = db[VCAP_BVT_GRIDFS_COLLECTION]
- total = coll.find.to_a.length
- coll.find.each_with_index do |row, index|
- uploads_index = uploads.index {|item| item['filename'] == row['filename']}
- index_str = "[#{(index + 1).to_s}/#{total.to_s}]"
- if uploads_index
- if uploads[uploads_index]['md5'] == row['md5']
- puts green("#{index_str}Skipped\t\t#{row['filename']}")
- uploads.delete_at(uploads_index)
- else
- puts yellow("#{index_str}Need to update\t#{row['filename']}")
- grid.delete(row['_id'])
- end
+ total = assets.length
+ assets.each_with_index do |row, index|
+ uploads_index = uploads.index {|item| item['filename'] == row['filename']}
+ index_str = "[#{(index + 1).to_s}/#{total.to_s}]"
+ if uploads_index
+ if uploads[uploads_index]['md5'] == row['md5']
+ puts green("#{index_str}Skipped\t\t#{row['filename']}")
+ uploads.delete_at(uploads_index)
else
- puts red("#{index_str}Removed\t\t#{row['filename']}")
- grid.delete(row['_id'])
+ puts yellow("#{index_str}Need to update\t#{row['filename']}")
end
+ else
+ puts red("#{index_str}Removed\t\t#{row['filename']}")
+ delete_binary(datastore_config[:delete], row['filename'])
end
end
@@ -70,7 +63,7 @@ def upload_assets
file = File.open(filepath)
index_str = "[#{(index + 1).to_s}/#{total.to_s}]"
puts yellow("#{index_str}Uploading\t#{item['filename']}")
- grid.put(file, :filename => item['filename'], :safe => true)
+ post_binary(datastore_config[:upload], filepath, item['md5'])
end
end
puts green("uploading assets binaries finished")
@@ -80,6 +73,70 @@ def check_md5(filepath)
Digest::MD5.hexdigest(File.read(filepath))
end
+ private
+
+ def list_binaries(url)
+ easy = Curl::Easy.new
+ easy.url = url
+ easy.resolve_mode = :ipv4
+ easy.timeout = 10
+ begin
+ easy.http_get
+ rescue Curl::Err::CurlError
+ raise RuntimeError,
+ red("Cannot connect to yeti blobs storage server, #{easy.url}\n" +
+ "Please check your network connection.")
+ end
+
+ if easy.response_code == OK
+ parser = Yajl::Parser.new
+ return parser.parse(easy.body_str)
+ end
+ end
+
+ def delete_binary(url, filename)
+ easy = Curl::Easy.new
+ easy.url = "#{url}/#{filename}"
+ easy.resolve_mode = :ipv4
+ easy.timeout = 10
+ begin
+ easy.http_delete
+ rescue Curl::Err::CurlError
+ raise RuntimeError,
+ red("Cannot connect to yeti blobs storage server, #{easy.url}\n" +
+ "Please check your network connection.")
+ end
+
+ unless easy.response_code == OK
+ raise RuntimeError, "Fail to delete file #{filename}\nPlease rerun " +
+ "'#{yellow("rake upload_assets")}' command."
+ end
+ end
+
+ def post_binary(url, filepath, md5)
+ filename = File.basename(filepath)
+ easy = Curl::Easy.new
+ easy.url = "#{url}?md5=#{md5}"
+ easy.resolve_mode = :ipv4
+ easy.timeout = 60 * 10
+
+ post_data = Curl::PostField.file('file', filepath)
+ easy.multipart_form_post = true
+
+ begin
+ easy.http_post(post_data)
+ rescue Curl::Err::CurlError
+ raise RuntimeError,
+ red("Cannot connect to yeti blobs storage server, #{easy.url}\n" +
+ "Please check your network connection.")
+ end
+
+ unless easy.response_code == OK
+ raise RuntimeError, "Fail to post file #{filename}\nPlease rerun " +
+ "'#{yellow("rake upload_assets")}' command."
+ end
+ end
+
extend self
end
end

0 comments on commit b5e539d

Please sign in to comment.