Permalink
Browse files

Mysql lifecycle API implementation.

Add following new features:
  - Snapshot APIs
  - Serialization APIs
  - Service async job base on resque
  - Serialization data downloader

Change-Id: I60c22022bcf49f7be0b90d6cb65406600e353552
  • Loading branch information...
andl committed Nov 28, 2011
1 parent 0dc345a commit 166e263b7bf2c820c706057b88618a609c7563f9
Showing with 1,559 additions and 44 deletions.
  1. +134 −1 base/lib/base/asynchronous_service_gateway.rb
  2. +2 −1 base/lib/base/gateway.rb
  3. +4 −0 base/lib/base/provisioner.rb
  4. +3 −2 mysql/Gemfile
  5. +34 −5 mysql/Gemfile.lock
  6. +28 −0 mysql/Rakefile
  7. +7 −0 mysql/bin/mysql_gateway
  8. +6 −0 mysql/bin/mysql_worker
  9. +11 −0 mysql/config/mysql_gateway.yml
  10. +25 −0 mysql/config/mysql_worker.yml
  11. +14 −0 mysql/lib/mysql_service/job.rb
  12. +82 −0 mysql/lib/mysql_service/job/async_job.rb
  13. +233 −0 mysql/lib/mysql_service/job/serialization.rb
  14. +188 −0 mysql/lib/mysql_service/job/snapshot.rb
  15. +1 −0 mysql/lib/mysql_service/mysql_error.rb
  16. +157 −0 mysql/lib/mysql_service/provisioner.rb
  17. +104 −31 mysql/lib/mysql_service/util.rb
  18. +21 −4 mysql/spec/spec_helper.rb
  19. BIN mysql/vendor/cache/curb-0.7.16.gem
  20. BIN mysql/vendor/cache/macaddr-1.5.0.gem
  21. BIN mysql/vendor/cache/multi_json-1.0.3.gem
  22. BIN mysql/vendor/cache/rack-1.2.2.gem
  23. BIN mysql/vendor/cache/rack-1.3.5.gem
  24. BIN mysql/vendor/cache/rack-protection-1.1.4.gem
  25. BIN mysql/vendor/cache/redis-2.2.2.gem
  26. BIN mysql/vendor/cache/redis-namespace-1.0.3.gem
  27. BIN mysql/vendor/cache/redisk-0.2.2.gem
  28. BIN mysql/vendor/cache/resque-1.19.0.gem
  29. BIN mysql/vendor/cache/resque-status-0.2.4.gem
  30. BIN mysql/vendor/cache/sinatra-1.2.1.gem
  31. BIN mysql/vendor/cache/sinatra-1.3.1.gem
  32. BIN mysql/vendor/cache/systemu-2.4.1.gem
  33. BIN mysql/vendor/cache/tilt-1.2.2.gem
  34. BIN mysql/vendor/cache/tilt-1.3.3.gem
  35. BIN mysql/vendor/cache/uuid-2.3.4.gem
  36. BIN mysql/vendor/cache/vegas-0.1.8.gem
  37. +19 −0 serialization_data_server/Gemfile
  38. +70 −0 serialization_data_server/Gemfile.lock
  39. +1 −0 serialization_data_server/README
  40. +42 −0 serialization_data_server/Rakefile
  41. +99 −0 serialization_data_server/bin/serialization_data_server
  42. +58 −0 serialization_data_server/config/example.nginx.config
  43. +19 −0 serialization_data_server/config/serialization_data_server.yml
  44. +131 −0 serialization_data_server/lib/server.rb
  45. +41 −0 serialization_data_server/spec/Rakefile
  46. +11 −0 serialization_data_server/spec/server_spec.rb
  47. +14 −0 serialization_data_server/spec/spec_helper.rb
  48. BIN serialization_data_server/vendor/cache/builder-3.0.0.gem
  49. BIN serialization_data_server/vendor/cache/ci_reporter-1.6.5.gem
  50. BIN serialization_data_server/vendor/cache/daemons-1.1.4.gem
  51. BIN serialization_data_server/vendor/cache/diff-lcs-1.1.3.gem
  52. BIN serialization_data_server/vendor/cache/eventmachine-0.12.10.gem
  53. BIN serialization_data_server/vendor/cache/json_pure-1.6.3.gem
  54. BIN serialization_data_server/vendor/cache/little-plugger-1.1.3.gem
  55. BIN serialization_data_server/vendor/cache/logging-1.6.1.gem
  56. BIN serialization_data_server/vendor/cache/nats-0.4.10.gem
  57. BIN serialization_data_server/vendor/cache/posix-spawn-0.3.6.gem
  58. BIN serialization_data_server/vendor/cache/rack-1.3.5.gem
  59. BIN serialization_data_server/vendor/cache/rack-protection-1.1.4.gem
  60. BIN serialization_data_server/vendor/cache/rack-test-0.6.1.gem
  61. BIN serialization_data_server/vendor/cache/rake-0.8.7.gem
  62. BIN serialization_data_server/vendor/cache/rcov-0.9.11.gem
  63. BIN serialization_data_server/vendor/cache/redis-2.2.2.gem
  64. BIN serialization_data_server/vendor/cache/rspec-2.7.0.gem
  65. BIN serialization_data_server/vendor/cache/rspec-core-2.7.1.gem
  66. BIN serialization_data_server/vendor/cache/rspec-expectations-2.7.0.gem
  67. BIN serialization_data_server/vendor/cache/rspec-mocks-2.7.0.gem
  68. BIN serialization_data_server/vendor/cache/sinatra-1.3.1.gem
  69. BIN serialization_data_server/vendor/cache/thin-1.3.1.gem
  70. BIN serialization_data_server/vendor/cache/tilt-1.3.3.gem
  71. BIN serialization_data_server/vendor/cache/uuidtools-2.1.2.gem
  72. BIN serialization_data_server/vendor/cache/vcap_common-0.99.gem
  73. BIN serialization_data_server/vendor/cache/vcap_logging-0.1.3.gem
  74. BIN serialization_data_server/vendor/cache/yajl-ruby-0.8.2.gem
@@ -58,6 +58,7 @@ def setup(opts)
@double_check_orphan_interval = opts[:double_check_orphan_interval] || 300
@handle_fetched = false
@fetching_handles = false
+ @api_extensions = opts[:api_extensions] || []
@svc_json = {
:label => @service[:label],
:url => @service[:url],
@@ -67,8 +68,9 @@ def setup(opts)
:description => @service[:description],
:plan_options => @service[:plan_options],
:acls => @service[:acls],
- :timeout => @service[:timeout]
+ :timeout => @service[:timeout],
}.to_json
+
@deact_json = {
:label => @service[:label],
:url => @service[:url],
@@ -224,6 +226,132 @@ def check_orphan(handles, callback, errback)
async_mode
end
+ # create a snapshot
+ post "/gateway/v1/configurations/:service_id/snapshots" do
+ not_impl unless @api_extensions.include? "snapshots"
+ service_id = params["service_id"]
+ @logger.info("Create snapshot request for service_id=#{service_id}")
+ @provisioner.create_snapshot(service_id) do |msg|
+ if msg['success']
+ async_reply(VCAP::Services::Api::Job.new(msg['response']).encode)
+ else
+ async_reply_error(msg['response'])
+ end
+ end
+ async_mode
+ end
+
+ # Get snapshot details
+ get "/gateway/v1/configurations/:service_id/snapshots/:snapshot_id" do
+ not_impl unless @api_extensions.include? "snapshots"
+ service_id = params["service_id"]
+ snapshot_id = params["snapshot_id"]
+ @logger.info("Get snapshot_id=#{snapshot_id} request for service_id=#{service_id}")
+ @provisioner.get_snapshot(service_id, snapshot_id) do |msg|
+ if msg['success']
+ async_reply(VCAP::Services::Api::Snapshot.new(msg['response']).encode)
+ else
+ async_reply_error(msg['response'])
+ end
+ end
+ async_mode
+ end
+
+ # Enumreate snapshot
+ get "/gateway/v1/configurations/:service_id/snapshots" do
+ not_impl unless @api_extensions.include? "snapshots"
+ service_id = params["service_id"]
+ @logger.info("Enumerate snapshots request for service_id=#{service_id}")
+ @provisioner.enumerate_snapshots(service_id) do |msg|
+ if msg['success']
+ async_reply(VCAP::Services::Api::SnapshotList.new(msg['response']).encode)
+ else
+ async_reply_error(msg['response'])
+ end
+ end
+ async_mode
+ end
+
+ # Rollback to a snapshot
+ put "/gateway/v1/configurations/:service_id/snapshots/:snapshot_id" do
+ not_impl unless @api_extensions.include? "snapshots"
+ service_id = params["service_id"]
+ snapshot_id = params["snapshot_id"]
+ @logger.info("Rollback service_id=#{service_id} to snapshot_id=#{snapshot_id}")
+ @provisioner.rollback_snapshot(service_id, snapshot_id) do |msg|
+ if msg['success']
+ async_reply(VCAP::Services::Api::Job.new(msg['response']).encode)
+ else
+ async_reply_error(msg['response'])
+ end
+ end
+ async_mode
+ end
+
+ # Get serialized url
+ get "/gateway/v1/configurations/:service_id/serialized/url" do
+ not_impl unless @api_extensions.include? "serialization"
+ service_id = params["service_id"]
+ @logger.info("Get serialized url for service_id=#{service_id}")
+ @provisioner.get_serialized_url(service_id) do |msg|
+ if msg['success']
+ async_reply(VCAP::Services::Api::Job.new(msg['response']).encode)
+ else
+ async_reply_error(msg['response'])
+ end
+ end
+ async_mode
+ end
+
+
+ # Import serialized data from url
+ put "/gateway/v1/configurations/:service_id/serialized/url" do
+ not_impl unless @api_extensions.include? "serialization"
+ req = VCAP::Services::Api::SerializedURL.decode(request_body)
+ service_id = params["service_id"]
+ @logger.info("Import serialized data from url:#{req.url} for service_id=#{service_id}")
+ @provisioner.import_from_url(service_id, req.url) do |msg|
+ if msg['success']
+ async_reply(VCAP::Services::Api::Job.new(msg['response']).encode)
+ else
+ async_reply_error(msg['response'])
+ end
+ end
+ async_mode
+ end
+
+ # Import serialized data from request
+ put "/gateway/v1/configurations/:service_id/serialized/data" do
+ not_impl unless @api_extensions.include? "serialization"
+ req = VCAP::Services::Api::SerializedData.decode(request_body)
+ service_id = params["service_id"]
+ @logger.info("Import data from request for service_id=#{service_id}")
+ @provisioner.import_from_data(service_id, req) do |msg|
+ if msg['success']
+ async_reply(VCAP::Services::Api::Job.new(msg['response']).encode)
+ else
+ async_reply_error(msg['response'])
+ end
+ end
+ async_mode
+ end
+
+ # Get Job details
+ get "/gateway/v1/configurations/:service_id/jobs/:job_id" do
+ not_impl unless @api_extensions.include? "jobs"
+ service_id = params["service_id"]
+ job_id = params["job_id"]
+ @logger.info("Get job=#{job_id} for service_id=#{service_id}")
+ @provisioner.job_details(service_id, job_id) do |msg|
+ if msg['success']
+ async_reply(VCAP::Services::Api::Job.new(msg['response']).encode)
+ else
+ async_reply_error(msg['response'])
+ end
+ end
+ async_mode
+ end
+
# Restore an instance of the service
#
post '/service/internal/v1/restore' do
@@ -285,6 +413,7 @@ def check_orphan(handles, callback, errback)
async_mode
end
+
#################### Helpers ####################
helpers do
@@ -376,6 +505,10 @@ def async_reply_error(error_msg)
err_body = error_msg['msg'].to_json()
async_reply_raw(error_msg['status'], {'Content-Type' => Rack::Mime.mime_type('.json')}, err_body)
end
+
+ def not_impl
+ halt 501, {'Content-Type' => Rack::Mime.mime_type('.json') }
+ end
end
private
View
@@ -105,7 +105,8 @@ def start
:node_timeout => node_timeout,
:cloud_controller_uri => cloud_controller_uri,
:check_orphan_interval => @config[:check_orphan_interval],
- :double_check_orphan_interval => @config[:double_check_orphan_interval]
+ :double_check_orphan_interval => @config[:double_check_orphan_interval],
+ :api_extensions => @config[:api_extensions],
)
Thin::Server.start(@config[:host], @config[:port], sg)
end
@@ -77,6 +77,9 @@ def process_nodes
@nodes.delete_if {|_, timestamp| Time.now.to_i - timestamp > 300}
end
+ def pre_send_announcement
+ end
+
def on_connect_node
@logger.debug("[#{service_description}] Connected to node mbus..")
@node_nats.subscribe("#{service_name}.announce") { |msg|
@@ -85,6 +88,7 @@ def on_connect_node
@node_nats.subscribe("#{service_name}.node_handles") { |msg| on_node_handles(msg) }
@node_nats.subscribe("#{service_name}.handles") {|msg, reply| on_query_handles(msg, reply) }
@node_nats.subscribe("#{service_name}.update_service_handle") {|msg, reply| on_update_service_handle(msg, reply) }
+ pre_send_announcement()
@node_nats.publish("#{service_name}.discover")
end
View
@@ -12,12 +12,13 @@ gem "uuidtools"
gem "ruby-hmac", :require => "hmac-sha1"
gem "thin"
gem "sinatra"
+gem "resque-status", "~> 0.2.4"
+gem "rake"
+gem "curb"
gem 'vcap_common', :require => ['vcap/common', 'vcap/component']
gem 'vcap_logging', '>=0.1.3', :require => ['vcap/logging']
-
group :test do
- gem "rake"
gem "rspec"
gem "rcov"
gem "ci_reporter"
View
@@ -6,6 +6,7 @@ GEM
builder (3.0.0)
ci_reporter (1.6.4)
builder (>= 2.1.2)
+ curb (0.7.16)
daemons (1.1.2)
data_objects (0.10.3)
addressable (~> 2.1)
@@ -65,15 +66,35 @@ GEM
little-plugger (1.1.2)
logging (1.5.0)
little-plugger (>= 1.1.2)
+ macaddr (1.5.0)
+ systemu (>= 2.4.0)
+ multi_json (1.0.3)
mysql (2.8.1)
nats (0.4.10)
daemons (>= 1.1.0)
eventmachine (>= 0.12.10)
json_pure (>= 1.5.1)
posix-spawn (0.3.6)
- rack (1.2.2)
+ rack (1.3.5)
+ rack-protection (1.1.4)
+ rack
rake (0.8.7)
rcov (0.9.9)
+ redis (2.2.2)
+ redis-namespace (1.0.3)
+ redis (< 3.0.0)
+ redisk (0.2.2)
+ redis (>= 0.1.1)
+ redis-namespace (>= 0.1.0)
+ resque (1.19.0)
+ multi_json (~> 1.0)
+ redis-namespace (~> 1.0.2)
+ sinatra (>= 0.9.2)
+ vegas (~> 0.1.2)
+ resque-status (0.2.4)
+ redisk (>= 0.2.1)
+ resque (>= 1.3.1)
+ uuid (>= 2.0.2)
rspec (2.5.0)
rspec-core (~> 2.5.0)
rspec-expectations (~> 2.5.0)
@@ -83,15 +104,19 @@ GEM
diff-lcs (~> 1.1.2)
rspec-mocks (2.5.0)
ruby-hmac (0.4.0)
- sinatra (1.2.1)
- rack (~> 1.1)
- tilt (>= 1.2.2, < 2.0)
+ sinatra (1.3.1)
+ rack (>= 1.3.4, ~> 1.3)
+ rack-protection (>= 1.1.2, ~> 1.1)
+ tilt (>= 1.3.3, ~> 1.3)
stringex (1.2.1)
+ systemu (2.4.1)
thin (1.2.11)
daemons (>= 1.0.9)
eventmachine (>= 0.12.6)
rack (>= 1.0.0)
- tilt (1.2.2)
+ tilt (1.3.3)
+ uuid (2.3.4)
+ macaddr (~> 1.0)
uuidtools (2.1.2)
vcap_common (0.99)
eventmachine (~> 0.12.10)
@@ -101,13 +126,16 @@ GEM
thin
yajl-ruby
vcap_logging (0.1.3)
+ vegas (0.1.8)
+ rack (>= 1.0.0)
yajl-ruby (0.8.2)
PLATFORMS
ruby
DEPENDENCIES
ci_reporter
+ curb
datamapper (>= 0.10.2)
dm-sqlite-adapter
do_sqlite3
@@ -118,6 +146,7 @@ DEPENDENCIES
nats (>= 0.4.10)
rake
rcov
+ resque-status (~> 0.2.4)
rspec
ruby-hmac
sinatra
View
@@ -1,4 +1,7 @@
require 'rake'
+require 'resque/tasks'
+require 'yaml'
+require 'yajl'
desc "Run specs"
task "spec" => ["bundler:install:test", "test:spec"]
@@ -37,3 +40,28 @@ namespace "test" do
sh("cd spec && rake spec:rcov")
end
end
+
+# resque tasks
+task "resque:setup" do |t|
+ config_base_dir = ENV["CLOUD_FOUNDRY_CONFIG_PATH"] || File.join(File.dirname(__FILE__), "config")
+ config_file = File.join(config_base_dir, "mysql_worker.yml")
+
+ begin
+ config = YAML.load_file(config_file)
+ rescue => e
+ puts "Could not read configuration file: #{e}"
+ exit
+ end
+
+ REQ_OPTS = %w(resque node_id pid)
+ missing_opts = REQ_OPTS.select {|o| !config.has_key? o}
+ raise ArgumentError, "Missing options: #{missing_opts.join(', ')}" unless missing_opts.empty?
+
+ ENV['QUEUE'] = config["node_id"]
+ ENV['PIDFILE'] = config["pid"]
+ ENV['COUNT'] = config["worker_count"].to_s if config["worker_count"]
+ ENV['WORKER_CONFIG'] = Yajl::Encoder.encode(config)
+
+ resque = %w(host port password).inject({}){|res, o| res[o.to_sym] = config["resque"][o]; res}
+ Resque.redis = Redis.new(resque)
+end
View
@@ -22,6 +22,13 @@ class VCAP::Services::Mysql::Gateway < VCAP::Services::Base::Gateway
File.join(config_base_dir, 'mysql_gateway.yml')
end
+ def additional_options
+ {
+ :resque => @config[:resque],
+ :upload_temp_dir => @config[:upload_temp_dir]
+ }
+ end
end
VCAP::Services::Mysql::Gateway.new.start
+
View
@@ -0,0 +1,6 @@
+#!/bin/bash
+path=`dirname $0`
+
+cd $path
+cd ..
+bundle exec rake -r ./lib/mysql_service/job.rb resque:work
@@ -18,6 +18,17 @@ logging:
mbus: nats://localhost:4222
pid: /var/vcap/sys/run/mysql_service.pid
node_timeout: 2
+api_extensions:
+ - snapshots
+ - serialization
+ - jobs
+resque:
+ host: localhost
+ port: 6379
+ passwd: redis
+ expire: 86400 # 24 hours, in seconds
+
+upload_temp_dir: /tmp
# allow_over_provisioning: false
# z_interval: 30
# check_orphan_interval: 3600
Oops, something went wrong.

0 comments on commit 166e263

Please sign in to comment.