diff --git a/lib/temporal/activity/context.rb b/lib/temporal/activity/context.rb
index 623d9566..5754056d 100644
--- a/lib/temporal/activity/context.rb
+++ b/lib/temporal/activity/context.rb
@@ -11,7 +11,6 @@ class Context
       def initialize(connection, metadata, is_shutting_down)
         @connection = connection
         @metadata = metadata
-        @start_to_close_deadline = metadata.start_to_close_timeout > 0 ? Time.now + metadata.start_to_close_timeout : nil
         @async = false
         @is_shutting_down = is_shutting_down
       end
@@ -96,11 +95,19 @@ def name
 
       private
 
-      attr_reader :connection, :metadata, :start_to_close_deadline
+      attr_reader :connection, :metadata
 
       def task_token
         metadata.task_token
       end
+
+      def start_to_close_deadline
+        if metadata.start_to_close_timeout.positive?
+          metadata.started_at + metadata.start_to_close_timeout
+        else
+          nil
+        end
+      end
     end
   end
 end
diff --git a/spec/unit/lib/temporal/activity/context_spec.rb b/spec/unit/lib/temporal/activity/context_spec.rb
index ab3ce0cf..6e9aed35 100644
--- a/spec/unit/lib/temporal/activity/context_spec.rb
+++ b/spec/unit/lib/temporal/activity/context_spec.rb
@@ -47,29 +47,23 @@
     end
 
     describe 'timed out' do
-      let(:metadata_hash) { Fabricate(:activity_metadata, start_to_close_timeout: 0.1).to_h }
-
-      it 'interrupted, raise flag true' do
-        subject.heartbeat_interrupted
-        expect(client)
-          .to have_received(:record_activity_task_heartbeat)
-          .with(namespace: metadata.namespace, task_token: metadata.task_token, details: nil)
-
-        sleep 0.1
+      let(:metadata_hash) { Fabricate(:activity_metadata, start_to_close_timeout: 10, started_at: Time.now - 15).to_h }
+      it 'raise flag true' do
         expect do
           subject.heartbeat_interrupted
         end.to raise_error(Temporal::ActivityExecutionTimedOut)
+
+        expect(client)
+          .to_not have_received(:record_activity_task_heartbeat)
+          .with(namespace: metadata.namespace, task_token: metadata.task_token, details: nil)
       end
 
-      it 'not interrupted, raise flag false' do
-        subject.heartbeat
-        sleep 0.1
+      it 'raise flag false' do
         subject.heartbeat
 
         expect(client)
           .to have_received(:record_activity_task_heartbeat)
-          .with(namespace: metadata.namespace, task_token: metadata.task_token, details: nil)
-          .twice
+          .once
       end
     end
 
@@ -205,13 +199,20 @@
   end
 
   describe '#timed_out?' do
-    let(:metadata_hash) { Fabricate(:activity_metadata, start_to_close_timeout: 0.1).to_h }
+    context 'is timed out' do
+      let(:metadata_hash) { Fabricate(:activity_metadata, start_to_close_timeout: 10, started_at: Time.now - 15).to_h }
+
+      it 'true when start to close exceeded' do
+        expect(subject.timed_out?).to be(true)
+      end
+    end
 
-    it 'becomes true when start to close exceeded' do
-      # Starts out as false
-      expect(subject.timed_out?).to be(false)
-      sleep 0.1
-      expect(subject.timed_out?).to be(true)
+    context 'is not timed out' do
+      let(:metadata_hash) { Fabricate(:activity_metadata, start_to_close_timeout: 10, started_at: Time.now - 5).to_h }
+
+      it 'false when start to close not exceeded' do
+        expect(subject.timed_out?).to be(false)
+      end
     end
   end