Permalink
Browse files

move lifecycle framework into base

including async job, snapshot and serilized

Change-Id: I35fa7726c4f237b42edf346716c5c29e46dd440c
  • Loading branch information...
1 parent 44823ca commit 31105b71cab17feabeca4cc8a706bccd442a10b1 @andl andl committed Dec 6, 2011
View
@@ -1,14 +1,16 @@
PATH
remote: .
specs:
- vcap_services_base (0.1.0)
+ vcap_services_base (0.1.1)
+ curb (~> 0.7.16)
datamapper (~> 1.1.0)
do_sqlite3 (~> 0.10.3)
em-http-request (~> 0.3.0)
eventmachine (~> 0.12.11.cloudfoundry.3)
eventmachine_httpserver (~> 0.2.1)
json (~> 1.4.6)
nats (~> 0.4.10)
+ resque-status (~> 0.2.4)
ruby-hmac (~> 0.4.0)
sinatra (~> 1.2.3)
thin (~> 1.2.11)
@@ -24,7 +26,8 @@ GEM
builder (3.0.0)
ci_reporter (1.6.4)
builder (>= 2.1.2)
- daemons (1.1.4)
+ curb (0.7.16)
+ daemons (1.1.5)
data_objects (0.10.7)
addressable (~> 2.1)
datamapper (1.1.0)
@@ -78,14 +81,32 @@ GEM
little-plugger (1.1.3)
logging (1.6.1)
little-plugger (>= 1.1.2)
- nats (0.4.12)
+ macaddr (1.5.0)
+ systemu (>= 2.4.0)
+ multi_json (1.0.4)
+ nats (0.4.10)
daemons (>= 1.1.0)
eventmachine (>= 0.12.10)
json_pure (>= 1.5.1)
posix-spawn (0.3.6)
rack (1.2.2)
rake (0.8.7)
rcov (0.9.9)
+ redis (2.2.2)
+ redis-namespace (1.0.3)
+ redis (< 3.0.0)
+ redisk (0.2.2)
+ redis (>= 0.1.1)
+ redis-namespace (>= 0.1.0)
+ resque (1.19.0)
+ multi_json (~> 1.0)
+ redis-namespace (~> 1.0.2)
+ sinatra (>= 0.9.2)
+ vegas (~> 0.1.2)
+ resque-status (0.2.4)
+ redisk (>= 0.2.1)
+ resque (>= 1.3.1)
+ uuid (>= 2.0.2)
rspec (2.5.0)
rspec-core (~> 2.5.0)
rspec-expectations (~> 2.5.0)
@@ -99,11 +120,14 @@ GEM
rack (~> 1.1)
tilt (>= 1.2.2, < 2.0)
stringex (1.2.2)
+ systemu (2.4.2)
thin (1.2.11)
daemons (>= 1.0.9)
eventmachine (>= 0.12.6)
rack (>= 1.0.0)
tilt (1.2.2)
+ uuid (2.3.4)
+ macaddr (~> 1.0)
uuidtools (2.1.2)
vcap_common (1.0.2)
eventmachine (~> 0.12.11.cloudfoundry.3)
@@ -113,6 +137,8 @@ GEM
thin (~> 1.2.11)
yajl-ruby (~> 0.8.3)
vcap_logging (0.1.3)
+ vegas (0.1.8)
+ rack (>= 1.0.0)
yajl-ruby (0.8.3)
PLATFORMS
View
@@ -0,0 +1,87 @@
+# Copyright (c) 2009-2011 VMware, Inc.
+require "resque/job_with_status"
+
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..')
+require "service_error"
+
+module Resque
+ extend self
+ # Patch Resque so we can determine queue by input args.
+ # Job class can define select_queue method and the result will be the queue name.
+ def enqueue(klass, *args)
+ queue = (klass.respond_to?(:select_queue) && klass.select_queue(*args)) || queue_from_class(klass)
+ enqueue_to(queue, klass, *args)
+ end
+
+ # Backport from resque master branch, we can remove this method when gem is updated.
+ def enqueue_to(queue, klass, *args)
+ # Perform before_enqueue hooks. Don't perform enqueue if any hook returns false
+ before_hooks = Plugin.before_enqueue_hooks(klass).collect do |hook|
+ klass.send(hook, *args)
+ end
+ return nil if before_hooks.any? { |result| result == false }
+
+ Job.create(queue, klass, *args)
+
+ Plugin.after_enqueue_hooks(klass).each do |hook|
+ klass.send(hook, *args)
+ end
+
+ return true
+ end
+
+ class Status
+ # new attributes
+ hash_accessor :complete_time
+ end
+end
+
+module VCAP
+ module Services
+ end
+end
+
+# A thin layer wraps resque-status
+module VCAP::Services::AsyncJob
+ include VCAP::Services::Base::Error
+
+ def self.logger=(logger)
+ @logger = logger
+ end
+
+ def job_repo_setup(options={})
+ raise "AsyncJob requires redis configuration." unless options[:redis]
+ @logger.debug("Initialize Resque using #{options}")
+ Resque.redis = options[:redis]
+ Resque::Status.expire_in = options[:expire] if options[:expire]
+ end
+
+ def get_job(jobid)
+ res = Resque::Status.get(jobid)
+ job_to_json(res)
+ end
+
+ def get_all_jobs()
+ Resque::Status.status_ids
+ end
+
+ def job_to_json(job)
+ return nil unless job
+ res = {
+ :job_id => job.uuid,
+ :status => job.status,
+ :start_time => job.time.to_s,
+ :description => job.options[:description] || "None"
+ }
+ res[:complete_time] = job.complete_time if job.complete_time
+ res[:result] = validate_message(job.message) if job.message
+ res
+ end
+
+ def validate_message(msg)
+ Yajl::Parser.parse(msg)
+ rescue => e
+ # generate internal error if we can't parse err msg
+ ServiceError.new(ServiceError::INTERNAL_ERROR).to_hash
+ end
+end
@@ -0,0 +1,112 @@
+# Copyright (c) 2009-2011 VMware, Inc.
+require "resque/job_with_status"
+require "fileutils"
+require "curb"
+require "vcap/logging"
+
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..')
+require "base/service_error"
+
+module VCAP
+ module Services
+ end
+end
+
+module VCAP::Services::Serialization
+ SERIALIZATION_KEY_PREFIX = "vcap:serialization".freeze
+
+ class << self
+ attr_reader :redis, :logger
+
+ # Config the redis using options
+ def redis_connect(opts)
+ resque = %w(host port password).inject({}){|res, o| res[o.to_sym] = opts[o]; res}
+ @redis = Redis.new(resque)
+ end
+
+ def redis=(redis)
+ raise "Serialization requires redis configuration." unless redis
+ @redis = redis
+ end
+
+ def logger=(logger)
+ @logger = logger
+ end
+ end
+
+ def redis_key(key)
+ "#{SERIALIZATION_KEY_PREFIX}:#{key}"
+ end
+
+ class SerializationJob < Resque::JobWithStatus
+ include VCAP::Services::Serialization
+ include VCAP::Services::Base::Error
+
+ class << self
+ attr_reader :logger
+
+ def queue_lookup_key
+ :node_id
+ end
+
+ def logger=(logger)
+ @logger = logger
+ end
+
+ def select_queue(*args)
+ result = nil
+ args.each do |arg|
+ result = arg[queue_lookup_key]if (arg.is_a? Hash )&& (arg.has_key?(queue_lookup_key))
+ end
+ @logger.info("Select queue #{result} for job #{self.class} with args:#{args.inspect}") if @logger
+ result
+ end
+ end
+
+ def initialize(*args)
+ super(*args)
+ parse_config
+ init_worker_logger()
+ end
+
+ def client
+ VCAP::Services::Serialization.redis
+ end
+
+ def init_worker_logger
+ VCAP::Logging.setup_from_config(@config["logging"])
+ @logger = VCAP::Logging.logger("#{@config["service_name"]}_worker")
+ end
+
+ # the serialize path structure looks like <base-dir>\serialize\<service-name>\<aa>\<bb>\<cc>\
+ # <aabbcc-rest-of-instance-guid>\<serialization data>
+ def get_serialized_data_path(name)
+ File.join(@config["serialization_base_dir"], "serialize", @config["service_name"] , name[0,2],name[2,2], name[4,2], name)
+ end
+
+ # Update the download token and save it in redis
+ def update_download_token(service, name, token)
+ client.set(redis_key("#{service}:#{name}:token"), token)
+ end
+
+ def parse_config
+ @config = Yajl::Parser.parse(ENV['WORKER_CONFIG'])
+ raise "Need environment variable: WORKER_CONFIG" unless @config
+ end
+
+ def cleanup(name)
+ return unless name
+ FileUtils.rm_rf(get_serialized_data_path(name))
+ end
+
+ # Fetch remote uri and stream content to file.
+ def fetch_url(url, file_path)
+ # TODO check the file size before download?
+ File.open(file_path, "wb+") do |f|
+ c = Curl::Easy.new(url)
+ c.on_body{|data| f.write(data)}
+ c.perform
+ end
+ end
+ end
+end
Oops, something went wrong.

0 comments on commit 31105b7

Please sign in to comment.