Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

rake tasks update, sync blobs related

1. remove admin cases from rake tests (using tags)
2. add rake build in tools
3. add update.sh in the root dir to update code and blob
4. add naming conversion for war files in .assets_binarys/
5. update path for java apps in config/assets.yml
6. add sync_assets in BVT rake task
7. add upload_assets in tools

Change-Id: Ib9c314611b81c8fe7e4775454ab3f96e9a210e52
Signed-off-by: michael zhang <zhangcheng@rbcon.com>
  • Loading branch information...
commit d2a472154200ba7a979fd53397468759ba2ee79d 1 parent 02ccb5a
michael zhang authored
View
2  .gitignore
@@ -0,0 +1,2 @@
+.assets-binaries/
+tools/.build
View
3  Gemfile
@@ -6,3 +6,6 @@ gem "cfoundry"
gem "vcap_logging", ">= 1.0"
gem "interact"
gem "curb"
+gem "mongo"
+gem "bson_ext"
+gem "yajl-ruby"
View
9 Gemfile.lock
@@ -1,6 +1,9 @@
GEM
remote: http://rubygems.org/
specs:
+ bson (1.5.2)
+ bson_ext (1.5.2)
+ bson (= 1.5.2)
cfoundry (0.1.0)
json_pure
rest-client
@@ -10,6 +13,8 @@ GEM
interact (0.4.2)
json_pure (1.6.6)
mime-types (1.18)
+ mongo (1.5.2)
+ bson (= 1.5.2)
rake (0.9.2.2)
rest-client (1.6.7)
mime-types (>= 1.16)
@@ -24,14 +29,18 @@ GEM
rubyzip (0.9.7)
vcap_logging (1.0.0)
rake
+ yajl-ruby (1.1.0)
PLATFORMS
ruby
DEPENDENCIES
+ bson_ext
cfoundry
curb
interact
+ mongo
rake
rspec
vcap_logging (>= 1.0)
+ yajl-ruby
View
9 Rakefile
@@ -22,7 +22,7 @@ desc "Run the Basic Viability Tests"
task :tests do
BVT::Harness::RakeHelper.generate_config_file
BVT::Harness::RakeHelper.check_environment
- sh "bundle exec rspec spec/ --format p -c | " +
+ sh "bundle exec rspec spec/ --tag ~admin --format p -c | " +
"tee #{File.join(BVT::Harness::VCAP_BVT_HOME, "error.log")}"
end
@@ -30,7 +30,7 @@ desc "Run admin test cases"
task :admin do
BVT::Harness::RakeHelper.generate_config_file
BVT::Harness::RakeHelper.check_environment
- sh "bundle exec rspec spec/users/admin_user_spec.rb --format p -c | " +
+ sh "bundle exec rspec spec/users/ --tag admin --format p -c | " +
"tee #{File.join(BVT::Harness::VCAP_BVT_HOME, "error.log")}"
end
@@ -58,3 +58,8 @@ desc "Clean up test environment"
task :clean do
BVT::Harness::RakeHelper.cleanup!
end
+
+desc "sync yeti assets binaries"
+task :sync_assets do
+ BVT::Harness::RakeHelper.sync_assets
+end
View
33 config/assets.yml
@@ -28,7 +28,7 @@ tiny_java_app:
instances: 1
runtime: "java"
memory: 64
- path: "assets/java_web/java_tiny_app/target"
+ path: ".assets-binaries/java_tiny_app.war"
simple_db_app:
framework: "sinatra"
@@ -59,14 +59,14 @@ rails3_app:
jpa_app:
framework: "spring"
memory: 512
- path: "assets/spring/jpa-guestbook/target"
+ path: ".assets-binaries/jpa-guestbook.war"
hibernate_app:
framework: "spring"
instances: 1
runtime: "java"
memory: 320
- path: "assets/spring/hibernate-guestbook/target"
+ path: ".assets-binaries/hibernate-guestbook.war"
dbrails_app:
framework: "rails3"
@@ -81,14 +81,16 @@ dbrails_broken_app:
path: "assets/rails3/dbrails_broken_app"
grails_app:
+ runtime: java
framework: "grails3"
memory: 512
- path: "assets/grails/guestbook/target"
+ path: ".assets-binaries/guestbook.war"
roo_app:
framework: "spring"
+ runtime: java
memory: 256
- path: "assets/spring/roo-guestbook/target"
+ path: ".assets-binaries/roo-guestbook.war"
mochiweb_test:
framework: "otp_rebar"
@@ -98,18 +100,21 @@ mochiweb_test:
simple-lift-app:
framework: "lift/1.0"
memory: 256
- path: "assets/lift/hello_lift/target"
+ runtime: java
+ path: ".assets-binaries/hello_lift.war"
lift-db-app:
framework: "lift/1.0"
memory: 256
- path: "assets/lift/lift-db-app/target"
+ runtime: java
+ path: ".assets-binaries/lift-db-app.war"
tomcat-version-check-app:
framework: "spring"
memory: 128
+ runtime: java
tomcat_version: "7.0.26"
- path: "assets/java_web/tomcat-version-check-app/target"
+ path: ".assets-binaries/tomcat-version-check-app.war"
app_rails_version18:
framework: "rails3"
@@ -259,7 +264,7 @@ app_spring_service:
instances: 1
runtime: "java"
memory: 256
- path: "assets/spring/app_spring_service/target"
+ path: ".assets-binaries/app_spring_service.war"
my_test_app_env_test_app:
framework: "sinatra"
@@ -294,18 +299,18 @@ simple_php_app:
spring-env-app:
framework: "spring"
memory: 256
- path: "assets/spring/spring-env/target"
+ path: ".assets-binaries/spring-env.war"
auto-reconfig-test-app:
framework: "spring"
runtime: "java"
memory: 128
- path: "assets/spring/auto-reconfig-test-app/target"
+ path: ".assets-binaries/auto-reconfig-test-app.war"
auto-reconfig-missing-deps-test-app:
framework: "spring"
memory: 256
- path: "assets/spring/auto-reconfig-missing-deps-test-app/target"
+ path: ".assets-binaries/auto-reconfig-missing-deps-test-app.war"
atmos_app:
framework: "sinatra"
@@ -327,7 +332,7 @@ java_app_with_startup_delay:
instances: 1
runtime: "java"
memory: 64
- path: "assets/java_web/app_with_startup_delay/target"
+ path: ".assets-binaries/java_app_with_startup_delay.war"
rails_console_test_app:
framework: "rails3"
@@ -382,7 +387,7 @@ standalone_java_app:
command: "bin/hello-cloud"
runtime: java
memory: 256
- path: "assets/standalone/java_app/target/appassembler"
+ path: ".assets-binaries/java_app.jar"
no_url: true
standalone_node_app:
View
8 lib/harness.rb
@@ -17,7 +17,13 @@ module Harness
Dir.mkdir(VCAP_BVT_HOME) unless Dir.exist?(VCAP_BVT_HOME)
VCAP::Logging.setup_from_config(config)
- APP_CHECK_LIMIT = 60
+ # Assets Data Store Config
+ VCAP_BVT_ASSETS_DATASTORE_CONFIG = File.join(VCAP_BVT_HOME, "datastore.yml")
+ VCAP_BVT_ASSETS_PACKAGES_HOME = File.join(File.dirname(__FILE__),
+ "../.assets-binaries")
+ VCAP_BVT_ASSETS_PACKAGES_MANIFEST = File.join(VCAP_BVT_ASSETS_PACKAGES_HOME,
+ "packages.yml")
+ VCAP_BVT_ASSETS_STORE_URL = "http://bolbs.cloudfoundry.com"
end
end
View
102 lib/harness/rake_helper.rb
@@ -2,6 +2,9 @@
require "interact"
require "harness"
require "curb"
+require "mongo"
+require "yajl"
+require "digest/md5"
module BVT::Harness
module RakeHelper
@@ -36,8 +39,6 @@ def check_environment
File.open(VCAP_BVT_PROFILE_FILE, "w") { |f| f.write YAML.dump(profile) }
end
- HTTP_RESPONSE_OK = 200
-
def check_network_connection
get_config unless @config
@@ -52,10 +53,10 @@ def check_network_connection
red("Cannot connect to target environment, #{easy.url}\n" +
"Please check your network connection to target environment.")
end
- unless easy.response_code == HTTP_RESPONSE_OK
+ unless easy.response_code == HTTP_RESPONSE_CODE::OK
raise RuntimeError,
red("URL: #{easy.url} response code does not equal to " +
- "#{HTTP_RESPONSE_OK}\nPlease check your target environment first.")
+ "#{HTTP_RESPONSE_CODE::OK}\nPlease check your target environment first.")
end
end
@@ -65,6 +66,56 @@ def cleanup!
cleanup_test_accounts
end
+ def sync_assets
+ downloads = get_assets_info
+ if File.exist?(VCAP_BVT_ASSETS_PACKAGES_MANIFEST)
+ locals = YAML.load_file(VCAP_BVT_ASSETS_PACKAGES_MANIFEST)['packages']
+ else
+ locals = []
+ end
+ puts "check local assets binaries"
+ skipped = []
+ unless locals.empty?
+ total = locals.length
+ locals.each_with_index do |item, index|
+ downloads_index = downloads.index {|e| e['filename'] == item['filename']}
+ index_str = "[#{(index + 1).to_s}/#{total.to_s}]"
+ if downloads_index
+ if downloads[downloads_index]['md5'] == item['md5']
+ puts green("#{index_str}Skipped\t\t#{item['filename']}")
+ downloads.delete_at(downloads_index)
+ skipped << Hash['filename' => item['filename'], 'md5' => item['md5']]
+ else
+ puts yellow("#{index_str}Need to update\t#{item['filename']}")
+ end
+ else
+ puts red("#{index_str}Remove\t\t#{item['filename']}")
+ File.delete(File.join(VCAP_BVT_ASSETS_PACKAGES_HOME, item['filename']))
+ end
+ end
+ end
+
+ unless downloads.empty?
+ puts "\ndownloading assets binaries"
+ Dir.mkdir(VCAP_BVT_ASSETS_PACKAGES_HOME) unless Dir.exist?(VCAP_BVT_ASSETS_PACKAGES_HOME)
+ total = downloads.length
+ downloads.each_with_index do |item, index|
+ index_str = "[#{(index + 1).to_s}/#{total.to_s}]"
+ filepath = File.join(VCAP_BVT_ASSETS_PACKAGES_HOME, item['filename'])
+ puts yellow("#{index_str}downloading\t#{item['filename']}")
+ download_binary(filepath)
+ unless check_md5(filepath) == item['md5']
+ puts red("#{index_str}fail to download\t\t#{item['filename']}")
+ end
+ skipped << Hash['filename' => item['filename'], 'md5' => item['md5']]
+ File.open(VCAP_BVT_ASSETS_PACKAGES_MANIFEST, "w") do |f|
+ f.write YAML.dump(Hash['packages' => skipped])
+ end
+ end
+ end
+ puts green("sync assets binaries finished")
+ end
+
private
def get_config
@@ -213,6 +264,49 @@ def cleanup_test_accounts()
puts yellow("Clean up test accounts has been done.\n")
end
+ def get_assets_info
+ easy = Curl::Easy.new
+ easy.url = "#{VCAP_BVT_ASSETS_STORE_URL}/list"
+ easy.resolve_mode = :ipv4
+ easy.timeout = 10
+ begin
+ easy.http_get
+ rescue Curl::Err::CurlError
+ raise RuntimeError,
+ red("Cannot connect to yeti assets storage server, #{easy.url}\n" +
+ "Please check your network connection.")
+ end
+
+ if easy.response_code == HTTP_RESPONSE_CODE::OK
+ parser = Yajl::Parser.new
+ return parser.parse(easy.body_str)
+ end
+ end
+
+ def check_md5(filepath)
+ Digest::MD5.hexdigest(File.read(filepath))
+ end
+
+ def download_binary(filepath)
+ filename = File.basename(filepath)
+ easy = Curl::Easy.new
+ easy.url = "#{VCAP_BVT_ASSETS_STORE_URL}/files/#{filename}"
+ easy.resolve_mode = :ipv4
+ easy.timeout = 60 * 5
+ easy.http_get
+ # retry once
+ unless easy.response_code == HTTP_RESPONSE_CODE::OK
+ sleep(1) # waiting for 1 second and try again
+ easy.http_get
+ end
+
+ if easy.response_code == HTTP_RESPONSE_CODE::OK
+ File.open(filepath, 'wb') { |f| f.write(easy.body_str) }
+ else
+ raise RuntimeError, "Fail to download binary #{filename}"
+ end
+ end
+
extend self
end
end
View
2  spec/users/admin_user_spec.rb
@@ -13,7 +13,7 @@
test_user.delete
end
- it "test add-user/users/delete-user/passwd command" do
+ it "test add-user/users/delete-user/passwd command", :admin => true do
# create user
test_user = @admin_session.user(@test_email)
test_pwd = "test-pwd"
View
98 tools/Rakefile
@@ -0,0 +1,98 @@
+$:.unshift(File.join(File.dirname(__FILE__), "scripts"))
+require "tools"
+
+task :default => [:help]
+
+desc "List help commands"
+task :help do
+ puts "Usage: rake [command]"
+ puts " build\t\t\t\tbuild the tests"
+ puts " upload_assets\t\t\tupload the pre-compiled java packages"
+ puts " help\t\t\t\tlist help commands"
+end
+
+desc "upload the pre-compiled java packages"
+task :upload_assets do
+ Tools::AssetsHelper.upload_assets
+end
+
+BUILD_ARTIFACT = File.join(Dir.pwd, ".build")
+TESTS_PATH = File.join(Dir.pwd, "../assets")
+VCAP_BVT_ASSETS_PACKAGES_HOME = File.join(File.dirname(__FILE__), "../.assets-binaries")
+
+TESTS_TO_BUILD = ["#{TESTS_PATH}/spring/auto-reconfig-test-app",
+ "#{TESTS_PATH}/spring/auto-reconfig-missing-deps-test-app",
+ "#{TESTS_PATH}/spring/app_spring_service",
+ "#{TESTS_PATH}/java_web/app_with_startup_delay",
+ "#{TESTS_PATH}/java_web/tomcat-version-check-app",
+ "#{TESTS_PATH}/spring/roo-guestbook",
+ "#{TESTS_PATH}/spring/jpa-guestbook",
+ "#{TESTS_PATH}/spring/hibernate-guestbook",
+ "#{TESTS_PATH}/spring/spring-env",
+ "#{TESTS_PATH}/grails/guestbook",
+ "#{TESTS_PATH}/java_web/java_tiny_app",
+ "#{TESTS_PATH}/lift/hello_lift",
+ "#{TESTS_PATH}/lift/lift-db-app",
+ "#{TESTS_PATH}/standalone/java_app"
+]
+
+desc "Build the tests. If the git hash hasn't change, nothing is built. To force a build, invoke 'rake build[--force]'"
+task :build, [:force] do |t, args|
+ puts "\nBuilding tests"
+ sh('cd .. && git submodule update --init')
+ if build_required? args.force
+ prompt_message = "\nBVT need java development environment to build java-base apps.\n"+
+ "Please run 'sudo aptitude install maven2 default-jdk' on your Linux box"
+ `mvn -v 2>&1`
+ raise prompt_message if $?.exitstatus != 0
+ ENV['MAVEN_OPTS']="-XX:MaxPermSize=256M"
+ TESTS_TO_BUILD.each do |test|
+ puts "\tBuilding '#{test}'"
+ Dir.chdir test do
+ sh('mvn package -DskipTests') do |success, exit_code|
+ if success
+ binaryname = File.join("{target,target/appassembler}", "*.{war,jar}")
+ binary_file = Dir.glob(binaryname).first
+ app_name = test.split('/')[-1]
+ file_type = '.'+binary_file.split('.')[-1]
+ target_file = File.join(VCAP_BVT_ASSETS_PACKAGES_HOME, app_name+file_type)
+ sh("mkdir #{VCAP_BVT_ASSETS_PACKAGES_HOME}") unless Dir.exist?(VCAP_BVT_ASSETS_PACKAGES_HOME)
+ sh("cp #{binary_file} #{target_file}")
+ else
+ clear_build_artifact
+ sh("mvn clean -q")
+ fail "\tFailed to build #{test} - aborting build"
+ end
+ end
+ end
+ puts "\tCompleted building '#{test}'"
+ end
+ save_git_hash
+ Tools::AssetsHelper.update_local_hash
+ else
+ puts "Built artifacts in sync with test assets - no build required"
+ end
+end
+
+def build_required? (force_build=nil)
+ if File.exists?(BUILD_ARTIFACT) == false or (force_build and force_build == "--force")
+ return true
+ end
+ Dir.chdir(TESTS_PATH) do
+ saved_git_hash = IO.readlines(BUILD_ARTIFACT)[0].split[0]
+ git_hash = `git rev-parse --short=8 --verify HEAD`
+ saved_git_hash.to_s.strip != git_hash.to_s.strip
+ end
+end
+
+def save_git_hash
+ Dir.chdir(TESTS_PATH) do
+ git_hash = `git rev-parse --short=8 --verify HEAD`
+ File.open(BUILD_ARTIFACT, 'w') {|f| f.puts("#{git_hash}")}
+ end
+end
+
+def clear_build_artifact
+ puts "\tClearing build artifact #{BUILD_ARTIFACT}"
+ File.unlink BUILD_ARTIFACT if File.exists? BUILD_ARTIFACT
+end
View
85 tools/scripts/assets_helper.rb
@@ -0,0 +1,85 @@
+$:.unshift(File.join(File.dirname(__FILE__), "../../lib"))
+
+require "yaml"
+require "interact"
+require "tools"
+require "curb"
+require "mongo"
+require "yajl"
+require "digest/md5"
+require "harness"
+
+module Tools
+ module AssetsHelper
+ include Interactive, BVT::Harness::ColorHelpers
+
+ def update_local_hash
+ if Dir.exist?(VCAP_BVT_ASSETS_PACKAGES_HOME)
+ skipped = []
+ Dir.new(VCAP_BVT_ASSETS_PACKAGES_HOME).each {|d|
+ if d.end_with?('.war') or d.end_with?('.jar')
+ file_path = File.join(VCAP_BVT_ASSETS_PACKAGES_HOME, d)
+ md5 = check_md5(file_path)
+ skipped << Hash['filename' => d, 'md5' => md5]
+ end
+ }
+ if skipped != []
+ File.open(VCAP_BVT_ASSETS_PACKAGES_MANIFEST, "w") do |f|
+ f.write YAML.dump(Hash['packages' => skipped])
+ end
+ end
+ end
+ end
+
+ VCAP_BVT_GRIDFS_COLLECTION = 'fs.files'
+ def upload_assets
+ datastore_config = YAML.load_file(VCAP_BVT_ASSETS_DATASTORE_CONFIG)
+ conn = Mongo::Connection.new(datastore_config['host'], datastore_config['port'])
+ db = conn.db(datastore_config['dbname'])
+ auth = db.authenticate(datastore_config['username'], datastore_config['password'])
+
+ puts "check assets storage server"
+ uploads = YAML.load_file(VCAP_BVT_ASSETS_PACKAGES_MANIFEST)['packages']
+ grid = Mongo::Grid.new(db)
+ if db.collection_names.include?(VCAP_BVT_GRIDFS_COLLECTION)
+ coll = db[VCAP_BVT_GRIDFS_COLLECTION]
+ total = coll.find.to_a.length
+ coll.find.each_with_index do |row, index|
+ uploads_index = uploads.index {|item| item['filename'] == row['filename']}
+ index_str = "[#{(index + 1).to_s}/#{total.to_s}]"
+ if uploads_index
+ if uploads[uploads_index]['md5'] == row['md5']
+ puts green("#{index_str}Skipped\t\t#{row['filename']}")
+ uploads.delete_at(uploads_index)
+ else
+ puts yellow("#{index_str}Need to update\t#{row['filename']}")
+ grid.delete(row['_id'])
+ end
+ else
+ puts red("#{index_str}Removed\t\t#{row['filename']}")
+ grid.delete(row['_id'])
+ end
+ end
+ end
+
+ puts "\nUploading assets binaries"
+ unless uploads.empty?
+ total = uploads.length
+ uploads.each_with_index do |item, index|
+ filepath = File.join(VCAP_BVT_ASSETS_PACKAGES_HOME, item['filename'])
+ file = File.open(filepath)
+ index_str = "[#{(index + 1).to_s}/#{total.to_s}]"
+ puts yellow("#{index_str}Uploading\t#{item['filename']}")
+ grid.put(file, :filename => item['filename'], :safe => true)
+ end
+ end
+ puts green("uploading assets binaries finished")
+ end
+
+ def check_md5(filepath)
+ Digest::MD5.hexdigest(File.read(filepath))
+ end
+
+ extend self
+ end
+end
View
18 tools/scripts/color_helper.rb
@@ -0,0 +1,18 @@
+# Displaying text in color
+module Tools
+ module ColorHelpers
+
+ def red(text)
+ "\e[31m#{text}\e[0m"
+ end
+
+ def green(text)
+ "\e[32m#{text}\e[0m"
+ end
+
+ def yellow(text)
+ "\e[33m#{text}\e[0m"
+ end
+
+ end
+end
View
15 tools/scripts/tools.rb
@@ -0,0 +1,15 @@
+require "vcap/logging"
+require "yaml"
+
+module Tools
+ VCAP_BVT_HOME = File.join(ENV['HOME'], '.bvt')
+ # Assets Data Store Config
+ VCAP_BVT_ASSETS_DATASTORE_CONFIG = File.join(VCAP_BVT_HOME, "datastore.yml")
+ VCAP_BVT_ASSETS_PACKAGES_HOME = File.join(File.dirname(__FILE__),
+ "../../.assets-binaries")
+ VCAP_BVT_ASSETS_PACKAGES_MANIFEST = File.join(VCAP_BVT_ASSETS_PACKAGES_HOME,
+ "packages.yml")
+ VCAP_BVT_ASSETS_STORE_URL = "http://bolbs.cloudfoundry.com"
+end
+
+require "assets_helper"
View
6 update.sh
@@ -0,0 +1,6 @@
+echo -e "\n### update yeti code"
+git pull
+echo -e "\n### update yeti submodule, i.e. assets"
+git submodule update --init --recursive
+echo -e "\n### sync pre-compiled java apps"
+rake sync_assets
Please sign in to comment.
Something went wrong with that request. Please try again.