diff --git a/src/torchcodec/decoders/_core/VideoDecoder.cpp b/src/torchcodec/decoders/_core/VideoDecoder.cpp
index 58a605fd3..63abee3a3 100644
--- a/src/torchcodec/decoders/_core/VideoDecoder.cpp
+++ b/src/torchcodec/decoders/_core/VideoDecoder.cpp
@@ -1168,7 +1168,7 @@ VideoDecoder::RawDecodedOutput VideoDecoder::getNextRawDecodedOutputNoDemux() {
       getDecodedOutputWithFilter([this](int frameStreamIndex, AVFrame* frame) {
         StreamInfo& activeStream = streams_[frameStreamIndex];
         return frame->pts >=
-            activeStream.discardFramesBeforePts.value_or(INT64_MIN);
+            activeStream.discardFramesBeforePts;
       });
   return rawOutput;
 }
diff --git a/src/torchcodec/decoders/_core/VideoDecoder.h b/src/torchcodec/decoders/_core/VideoDecoder.h
index 8535a61ea..c4da3c614 100644
--- a/src/torchcodec/decoders/_core/VideoDecoder.h
+++ b/src/torchcodec/decoders/_core/VideoDecoder.h
@@ -307,8 +307,8 @@ class VideoDecoder {
     int64_t currentDuration = 0;
     // The desired position of the cursor in the stream. We send frames >=
     // this pts to the user when they request a frame.
-    // We set this field if the user requested a seek.
-    std::optional<int64_t> discardFramesBeforePts = 0;
+    // We update this field if the user requested a seek.
+    int64_t discardFramesBeforePts = INT64_MIN;
     VideoStreamDecoderOptions options;
     // The filter state associated with this stream (for video streams). The
     // actual graph will be nullptr for inactive streams.