Permalink
Browse files

rake tasks update, sync blobs related

1. remove admin cases from rake tests (using tags)
2. add rake build in tools
3. add update.sh in the root dir to update code and blob
4. add naming conversion for war files in .assets_binarys/
5. update path for java apps in config/assets.yml
6. add sync_assets in BVT rake task
7. add upload_assets in tools

Change-Id: Ib9c314611b81c8fe7e4775454ab3f96e9a210e52
Signed-off-by: michael zhang <zhangcheng@rbcon.com>
  • Loading branch information...
1 parent 02ccb5a commit d2a472154200ba7a979fd53397468759ba2ee79d michael zhang committed May 9, 2012
Showing with 368 additions and 22 deletions.
  1. +2 −0 .gitignore
  2. +3 −0 Gemfile
  3. +9 −0 Gemfile.lock
  4. +7 −2 Rakefile
  5. +19 −14 config/assets.yml
  6. +7 −1 lib/harness.rb
  7. +98 −4 lib/harness/rake_helper.rb
  8. +1 −1 spec/users/admin_user_spec.rb
  9. +98 −0 tools/Rakefile
  10. +85 −0 tools/scripts/assets_helper.rb
  11. +18 −0 tools/scripts/color_helper.rb
  12. +15 −0 tools/scripts/tools.rb
  13. +6 −0 update.sh
View
@@ -0,0 +1,2 @@
+.assets-binaries/
+tools/.build
View
@@ -6,3 +6,6 @@ gem "cfoundry"
gem "vcap_logging", ">= 1.0"
gem "interact"
gem "curb"
+gem "mongo"
+gem "bson_ext"
+gem "yajl-ruby"
View
@@ -1,6 +1,9 @@
GEM
remote: http://rubygems.org/
specs:
+ bson (1.5.2)
+ bson_ext (1.5.2)
+ bson (= 1.5.2)
cfoundry (0.1.0)
json_pure
rest-client
@@ -10,6 +13,8 @@ GEM
interact (0.4.2)
json_pure (1.6.6)
mime-types (1.18)
+ mongo (1.5.2)
+ bson (= 1.5.2)
rake (0.9.2.2)
rest-client (1.6.7)
mime-types (>= 1.16)
@@ -24,14 +29,18 @@ GEM
rubyzip (0.9.7)
vcap_logging (1.0.0)
rake
+ yajl-ruby (1.1.0)
PLATFORMS
ruby
DEPENDENCIES
+ bson_ext
cfoundry
curb
interact
+ mongo
rake
rspec
vcap_logging (>= 1.0)
+ yajl-ruby
View
@@ -22,15 +22,15 @@ desc "Run the Basic Viability Tests"
task :tests do
BVT::Harness::RakeHelper.generate_config_file
BVT::Harness::RakeHelper.check_environment
- sh "bundle exec rspec spec/ --format p -c | " +
+ sh "bundle exec rspec spec/ --tag ~admin --format p -c | " +
"tee #{File.join(BVT::Harness::VCAP_BVT_HOME, "error.log")}"
end
desc "Run admin test cases"
task :admin do
BVT::Harness::RakeHelper.generate_config_file
BVT::Harness::RakeHelper.check_environment
- sh "bundle exec rspec spec/users/admin_user_spec.rb --format p -c | " +
+ sh "bundle exec rspec spec/users/ --tag admin --format p -c | " +
"tee #{File.join(BVT::Harness::VCAP_BVT_HOME, "error.log")}"
end
@@ -58,3 +58,8 @@ desc "Clean up test environment"
task :clean do
BVT::Harness::RakeHelper.cleanup!
end
+
+desc "sync yeti assets binaries"
+task :sync_assets do
+ BVT::Harness::RakeHelper.sync_assets
+end
View
@@ -28,7 +28,7 @@ tiny_java_app:
instances: 1
runtime: "java"
memory: 64
- path: "assets/java_web/java_tiny_app/target"
+ path: ".assets-binaries/java_tiny_app.war"
simple_db_app:
framework: "sinatra"
@@ -59,14 +59,14 @@ rails3_app:
jpa_app:
framework: "spring"
memory: 512
- path: "assets/spring/jpa-guestbook/target"
+ path: ".assets-binaries/jpa-guestbook.war"
hibernate_app:
framework: "spring"
instances: 1
runtime: "java"
memory: 320
- path: "assets/spring/hibernate-guestbook/target"
+ path: ".assets-binaries/hibernate-guestbook.war"
dbrails_app:
framework: "rails3"
@@ -81,14 +81,16 @@ dbrails_broken_app:
path: "assets/rails3/dbrails_broken_app"
grails_app:
+ runtime: java
framework: "grails3"
memory: 512
- path: "assets/grails/guestbook/target"
+ path: ".assets-binaries/guestbook.war"
roo_app:
framework: "spring"
+ runtime: java
memory: 256
- path: "assets/spring/roo-guestbook/target"
+ path: ".assets-binaries/roo-guestbook.war"
mochiweb_test:
framework: "otp_rebar"
@@ -98,18 +100,21 @@ mochiweb_test:
simple-lift-app:
framework: "lift/1.0"
memory: 256
- path: "assets/lift/hello_lift/target"
+ runtime: java
+ path: ".assets-binaries/hello_lift.war"
lift-db-app:
framework: "lift/1.0"
memory: 256
- path: "assets/lift/lift-db-app/target"
+ runtime: java
+ path: ".assets-binaries/lift-db-app.war"
tomcat-version-check-app:
framework: "spring"
memory: 128
+ runtime: java
tomcat_version: "7.0.26"
- path: "assets/java_web/tomcat-version-check-app/target"
+ path: ".assets-binaries/tomcat-version-check-app.war"
app_rails_version18:
framework: "rails3"
@@ -259,7 +264,7 @@ app_spring_service:
instances: 1
runtime: "java"
memory: 256
- path: "assets/spring/app_spring_service/target"
+ path: ".assets-binaries/app_spring_service.war"
my_test_app_env_test_app:
framework: "sinatra"
@@ -294,18 +299,18 @@ simple_php_app:
spring-env-app:
framework: "spring"
memory: 256
- path: "assets/spring/spring-env/target"
+ path: ".assets-binaries/spring-env.war"
auto-reconfig-test-app:
framework: "spring"
runtime: "java"
memory: 128
- path: "assets/spring/auto-reconfig-test-app/target"
+ path: ".assets-binaries/auto-reconfig-test-app.war"
auto-reconfig-missing-deps-test-app:
framework: "spring"
memory: 256
- path: "assets/spring/auto-reconfig-missing-deps-test-app/target"
+ path: ".assets-binaries/auto-reconfig-missing-deps-test-app.war"
atmos_app:
framework: "sinatra"
@@ -327,7 +332,7 @@ java_app_with_startup_delay:
instances: 1
runtime: "java"
memory: 64
- path: "assets/java_web/app_with_startup_delay/target"
+ path: ".assets-binaries/java_app_with_startup_delay.war"
rails_console_test_app:
framework: "rails3"
@@ -382,7 +387,7 @@ standalone_java_app:
command: "bin/hello-cloud"
runtime: java
memory: 256
- path: "assets/standalone/java_app/target/appassembler"
+ path: ".assets-binaries/java_app.jar"
no_url: true
standalone_node_app:
View
@@ -17,7 +17,13 @@ module Harness
Dir.mkdir(VCAP_BVT_HOME) unless Dir.exist?(VCAP_BVT_HOME)
VCAP::Logging.setup_from_config(config)
- APP_CHECK_LIMIT = 60
+ # Assets Data Store Config
+ VCAP_BVT_ASSETS_DATASTORE_CONFIG = File.join(VCAP_BVT_HOME, "datastore.yml")
+ VCAP_BVT_ASSETS_PACKAGES_HOME = File.join(File.dirname(__FILE__),
+ "../.assets-binaries")
+ VCAP_BVT_ASSETS_PACKAGES_MANIFEST = File.join(VCAP_BVT_ASSETS_PACKAGES_HOME,
+ "packages.yml")
+ VCAP_BVT_ASSETS_STORE_URL = "http://bolbs.cloudfoundry.com"
end
end
View
@@ -2,6 +2,9 @@
require "interact"
require "harness"
require "curb"
+require "mongo"
+require "yajl"
+require "digest/md5"
module BVT::Harness
module RakeHelper
@@ -36,8 +39,6 @@ def check_environment
File.open(VCAP_BVT_PROFILE_FILE, "w") { |f| f.write YAML.dump(profile) }
end
- HTTP_RESPONSE_OK = 200
-
def check_network_connection
get_config unless @config
@@ -52,10 +53,10 @@ def check_network_connection
red("Cannot connect to target environment, #{easy.url}\n" +
"Please check your network connection to target environment.")
end
- unless easy.response_code == HTTP_RESPONSE_OK
+ unless easy.response_code == HTTP_RESPONSE_CODE::OK
raise RuntimeError,
red("URL: #{easy.url} response code does not equal to " +
- "#{HTTP_RESPONSE_OK}\nPlease check your target environment first.")
+ "#{HTTP_RESPONSE_CODE::OK}\nPlease check your target environment first.")
end
end
@@ -65,6 +66,56 @@ def cleanup!
cleanup_test_accounts
end
+ def sync_assets
+ downloads = get_assets_info
+ if File.exist?(VCAP_BVT_ASSETS_PACKAGES_MANIFEST)
+ locals = YAML.load_file(VCAP_BVT_ASSETS_PACKAGES_MANIFEST)['packages']
+ else
+ locals = []
+ end
+ puts "check local assets binaries"
+ skipped = []
+ unless locals.empty?
+ total = locals.length
+ locals.each_with_index do |item, index|
+ downloads_index = downloads.index {|e| e['filename'] == item['filename']}
+ index_str = "[#{(index + 1).to_s}/#{total.to_s}]"
+ if downloads_index
+ if downloads[downloads_index]['md5'] == item['md5']
+ puts green("#{index_str}Skipped\t\t#{item['filename']}")
+ downloads.delete_at(downloads_index)
+ skipped << Hash['filename' => item['filename'], 'md5' => item['md5']]
+ else
+ puts yellow("#{index_str}Need to update\t#{item['filename']}")
+ end
+ else
+ puts red("#{index_str}Remove\t\t#{item['filename']}")
+ File.delete(File.join(VCAP_BVT_ASSETS_PACKAGES_HOME, item['filename']))
+ end
+ end
+ end
+
+ unless downloads.empty?
+ puts "\ndownloading assets binaries"
+ Dir.mkdir(VCAP_BVT_ASSETS_PACKAGES_HOME) unless Dir.exist?(VCAP_BVT_ASSETS_PACKAGES_HOME)
+ total = downloads.length
+ downloads.each_with_index do |item, index|
+ index_str = "[#{(index + 1).to_s}/#{total.to_s}]"
+ filepath = File.join(VCAP_BVT_ASSETS_PACKAGES_HOME, item['filename'])
+ puts yellow("#{index_str}downloading\t#{item['filename']}")
+ download_binary(filepath)
+ unless check_md5(filepath) == item['md5']
+ puts red("#{index_str}fail to download\t\t#{item['filename']}")
+ end
+ skipped << Hash['filename' => item['filename'], 'md5' => item['md5']]
+ File.open(VCAP_BVT_ASSETS_PACKAGES_MANIFEST, "w") do |f|
+ f.write YAML.dump(Hash['packages' => skipped])
+ end
+ end
+ end
+ puts green("sync assets binaries finished")
+ end
+
private
def get_config
@@ -213,6 +264,49 @@ def cleanup_test_accounts()
puts yellow("Clean up test accounts has been done.\n")
end
+ def get_assets_info
+ easy = Curl::Easy.new
+ easy.url = "#{VCAP_BVT_ASSETS_STORE_URL}/list"
+ easy.resolve_mode = :ipv4
+ easy.timeout = 10
+ begin
+ easy.http_get
+ rescue Curl::Err::CurlError
+ raise RuntimeError,
+ red("Cannot connect to yeti assets storage server, #{easy.url}\n" +
+ "Please check your network connection.")
+ end
+
+ if easy.response_code == HTTP_RESPONSE_CODE::OK
+ parser = Yajl::Parser.new
+ return parser.parse(easy.body_str)
+ end
+ end
+
+ def check_md5(filepath)
+ Digest::MD5.hexdigest(File.read(filepath))
+ end
+
+ def download_binary(filepath)
+ filename = File.basename(filepath)
+ easy = Curl::Easy.new
+ easy.url = "#{VCAP_BVT_ASSETS_STORE_URL}/files/#{filename}"
+ easy.resolve_mode = :ipv4
+ easy.timeout = 60 * 5
+ easy.http_get
+ # retry once
+ unless easy.response_code == HTTP_RESPONSE_CODE::OK
+ sleep(1) # waiting for 1 second and try again
+ easy.http_get
+ end
+
+ if easy.response_code == HTTP_RESPONSE_CODE::OK
+ File.open(filepath, 'wb') { |f| f.write(easy.body_str) }
+ else
+ raise RuntimeError, "Fail to download binary #{filename}"
+ end
+ end
+
extend self
end
end
@@ -13,7 +13,7 @@
test_user.delete
end
- it "test add-user/users/delete-user/passwd command" do
+ it "test add-user/users/delete-user/passwd command", :admin => true do
# create user
test_user = @admin_session.user(@test_email)
test_pwd = "test-pwd"
Oops, something went wrong.

0 comments on commit d2a4721

Please sign in to comment.