Merge "add ttl for all lifecycle jobs" into services-r12

base/lib/base/job/lock.rb

@@ -12,10 +12,17 @@ class Lock
     attr_reader :expiration, :timeout, :name
     include VCAP::Services::Base::Error
 
+    # Options for lock
+    # name       - The uuid of the lock
+    # timeout    - The time that waits to acquire the lock, default 10 seconds
+    # expiration - Lock expires in given seconds if not refreshed, default 10 seconds
+    # logger     - The logger..
+    # ttl        - The max time that a thread can acquire the lock, default 600 seconds. Lock raise +JOB_TIMEOUT+ error once the ttl is exceeded.
     def initialize(name, opts={})
       @name = name
       @timeout = opts[:timeout] || 10 #seconds
       @expiration = opts[:expiration] || 10  # seconds
+      @ttl = opts[:ttl] || 600 # seconds
       @logger = opts[:logger] || make_logger
       config = Config.redis_config
       raise "Can't find configuration of redis." unless config
@@ -52,7 +59,11 @@ def lock
       @logger.debug("Lock #{@name} is acquired, will expire at #{@lock_expiration}")
 
       begin
-        yield if block_given?
+        Timeout::timeout(@ttl) do
+          yield if block_given?
+        end
+      rescue Timeout::Error =>e
+        raise ServiceError.new(ServiceError::JOB_TIMEOUT, @ttl)
       ensure
         release_thread(refresh_thread) if refresh_thread
         delete

base/lib/base/job/serialization.rb

@@ -67,7 +67,8 @@ def initialize(*args)
 
       def create_lock
         lock_name = "lock:lifecycle:#{name}"
-        lock = Lock.new(lock_name, :logger => @logger)
+        ttl = @config[:job_ttl] || 600
+        lock = Lock.new(lock_name, :logger => @logger, :ttl => ttl)
         lock
       end
 

base/lib/base/job/snapshot.rb

@@ -144,7 +144,8 @@ def required_options(*args)
 
       def create_lock
         lock_name = "lock:lifecycle:#{name}"
-        lock = Lock.new(lock_name, :logger => @logger)
+        ttl = @config[:job_ttl] || 600
+        lock = Lock.new(lock_name, :logger => @logger, :ttl => ttl)
         lock
       end
 

base/lib/base/service_error.rb

@@ -53,7 +53,8 @@ class ServiceError < StandardError
           # 30800 - 30899 500 Lifecycle error
           OVER_QUOTA = [30800, HTTP_INTERNAL, "Instance %s has %s snapshots. Quota is %s "]
           JOB_QUEUE_TIMEOUT = [30801, HTTP_INTERNAL, "Job timeout after waiting for %s seconds."]
-          BAD_SERIALIZED_DATAFILE = [30801, HTTP_INTERNAL, "Invalid serialized data file from: %s"]
+          JOB_TIMEOUT = [30802, HTTP_INTERNAL, "Job is killed since it runs longer than ttl: %s seconds."]
+          BAD_SERIALIZED_DATAFILE = [30803, HTTP_INTERNAL, "Invalid serialized data file from: %s"]
 
           # 31000 - 32000  Service-specific Error
           # Defined in services directory, for example mongodb/lib/mongodb_service/

base/spec/job_spec.rb

@@ -101,5 +101,18 @@
     current_value = @stored_value
     sleep expiration * 2
     current_value.should == @stored_value
+    ran_once.should be_true
+  end
+
+  it "should raise error if lock exceed ttl" do
+    @redis.should_receive(:watch).with(@name).any_number_of_times
+    @redis.should_receive(:multi).any_number_of_times.and_yield
+
+    ttl = 1
+    lock = VCAP::Services::Base::AsyncJob::Lock.new(@name, :logger => @logger, :ttl => ttl)
+
+    ran_once = false
+    expect { lock.lock{ ran_once = true; sleep ttl * 2} }.should raise_error(VCAP::Services::Base::Error::ServiceError, /ttl: #{ttl} seconds/)
+    ran_once.should be_true
   end
 end