Allows the authoring engine to skip frame.

This is 1st part of the work to allow audio and video resync if we found out that audio and video are out of sync during authoring - also fixed a problem in AACEncoder::read() where the buffer acquired from the buffer group does not release when error out at reading from source. Change-Id: I8a2740097fcfdf85e6178869afeb9f3687a99118
joneschrisg · Jul 22, 2010 · 53d4e0d · 53d4e0d
1 parent db3eb35
commit 53d4e0d
Show file tree

Hide file tree

Showing 7 changed files with 138 additions and 54 deletions.
diff --git a/include/media/stagefright/MediaSource.h b/include/media/stagefright/MediaSource.h
@@ -78,18 +78,31 @@ struct MediaSource : public RefBase {
         void clearSeekTo();
         bool getSeekTo(int64_t *time_us, SeekMode *mode) const;
 
+        // Option allows encoder to skip some frames until the specified
+        // time stamp.
+        // To prevent from being abused, when the skipFrame timestamp is
+        // found to be more than 1 second later than the current timestamp,
+        // an error will be returned from read().
+        void clearSkipFrame();
+        bool getSkipFrame(int64_t *timeUs) const;
+        void setSkipFrame(int64_t timeUs);
+
         void setLateBy(int64_t lateness_us);
         int64_t getLateBy() const;
 
     private:
         enum Options {
+            // Bit map
             kSeekTo_Option      = 1,
+            kSkipFrame_Option   = 2,
         };
 
         uint32_t mOptions;
         int64_t mSeekTimeUs;
         SeekMode mSeekMode;
         int64_t mLatenessUs;
+
+        int64_t mSkipFrameUntilTimeUs;
     };
 
     // Causes this source to suspend pulling data from its upstream source

diff --git a/include/media/stagefright/OMXCodec.h b/include/media/stagefright/OMXCodec.h
@@ -143,6 +143,7 @@ struct OMXCodec : public MediaSource,
     int64_t mSeekTimeUs;
     ReadOptions::SeekMode mSeekMode;
     int64_t mTargetTimeUs;
+    int64_t mSkipTimeUs;
 
     MediaBuffer *mLeftOverBuffer;
 

diff --git a/media/libstagefright/AudioSource.cpp b/media/libstagefright/AudioSource.cpp
@@ -137,52 +137,75 @@ status_t AudioSource::read(
     MediaBuffer *buffer;
     CHECK_EQ(mGroup->acquire_buffer(&buffer), OK);
 
-    uint32_t numFramesRecorded;
-    mRecord->getPosition(&numFramesRecorded);
-    int64_t latency = mRecord->latency() * 1000;
-
-    int64_t readTime = systemTime() / 1000;
-    if (numFramesRecorded == 0) {
-        // Initial delay
-        if (mStartTimeUs > 0) {
-            mStartTimeUs = readTime - mStartTimeUs;
+    while (mStarted) {
+        uint32_t numFramesRecorded;
+        mRecord->getPosition(&numFramesRecorded);
+        int64_t latency = mRecord->latency() * 1000;
+
+        int64_t readTime = systemTime() / 1000;
+
+        if (numFramesRecorded == 0) {
+            // Initial delay
+            if (mStartTimeUs > 0) {
+                mStartTimeUs = readTime - mStartTimeUs;
+            } else {
+                mStartTimeUs += latency;
+            }
+        }
+
+        ssize_t n = 0;
+        if (mCollectStats) {
+            n = mRecord->read(buffer->data(), buffer->size());
+            int64_t endTime = systemTime() / 1000;
+            mTotalReadTimeUs += (endTime - readTime);
+            if (n >= 0) {
+                mTotalReadBytes += n;
+            }
         } else {
-            mStartTimeUs += latency;
+            n = mRecord->read(buffer->data(), buffer->size());
         }
-    }
 
-    ssize_t n = 0;
-    if (mCollectStats) {
-        n = mRecord->read(buffer->data(), buffer->size());
-        int64_t endTime = systemTime() / 1000;
-        mTotalReadTimeUs += (endTime - readTime);
-        if (n >= 0) {
-            mTotalReadBytes += n;
+        if (n < 0) {
+            buffer->release();
+            buffer = NULL;
+
+            return (status_t)n;
         }
-    } else {
-        n = mRecord->read(buffer->data(), buffer->size());
-    }
 
-    if (n < 0) {
-        buffer->release();
-        buffer = NULL;
+        uint32_t sampleRate = mRecord->getSampleRate();
+        int64_t timestampUs = (1000000LL * numFramesRecorded) / sampleRate +
+                                 mStartTimeUs;
+        int64_t skipFrameUs;
+        if (!options || !options->getSkipFrame(&skipFrameUs)) {
+            skipFrameUs = timestampUs;  // Don't skip frame
+        }
 
-        return (status_t)n;
-    }
+        if (skipFrameUs > timestampUs) {
+            // Safe guard against the abuse of the kSkipFrame_Option.
+            if (skipFrameUs - timestampUs >= 1E6) {
+                LOGE("Frame skipping requested is way too long: %lld us",
+                    skipFrameUs - timestampUs);
+                buffer->release();
+                return UNKNOWN_ERROR;
+            }
+            LOGV("skipFrame: %lld us > timestamp: %lld us, samples %d",
+                skipFrameUs, timestampUs, numFramesRecorded);
+            continue;
+        }
 
-    if (mTrackMaxAmplitude) {
-        trackMaxAmplitude((int16_t *) buffer->data(), n >> 1);
-    }
+        if (mTrackMaxAmplitude) {
+            trackMaxAmplitude((int16_t *) buffer->data(), n >> 1);
+        }
 
-    uint32_t sampleRate = mRecord->getSampleRate();
-    int64_t timestampUs = (1000000LL * numFramesRecorded) / sampleRate + mStartTimeUs;
-    buffer->meta_data()->setInt64(kKeyTime, timestampUs);
-    LOGV("initial delay: %lld, sample rate: %d, timestamp: %lld",
-            mStartTimeUs, sampleRate, timestampUs);
+        buffer->meta_data()->setInt64(kKeyTime, timestampUs);
+        LOGV("initial delay: %lld, sample rate: %d, timestamp: %lld",
+                mStartTimeUs, sampleRate, timestampUs);
 
-    buffer->set_range(0, n);
+        buffer->set_range(0, n);
 
-    *out = buffer;
+        *out = buffer;
+        return OK;
+    }
 
     return OK;
 }

diff --git a/media/libstagefright/CameraSource.cpp b/media/libstagefright/CameraSource.cpp
@@ -277,23 +277,44 @@ status_t CameraSource::read(
 
     {
         Mutex::Autolock autoLock(mLock);
-        while (mStarted && mFramesReceived.empty()) {
-            mFrameAvailableCondition.wait(mLock);
-        }
-        if (!mStarted) {
-            return OK;
-        }
-        frame = *mFramesReceived.begin();
-        mFramesReceived.erase(mFramesReceived.begin());
+        while (mStarted) {
+            while(mFramesReceived.empty()) {
+                mFrameAvailableCondition.wait(mLock);
+            }
+
+            if (!mStarted) {
+                return OK;
+            }
 
-        frameTime = *mFrameTimes.begin();
-        mFrameTimes.erase(mFrameTimes.begin());
+            frame = *mFramesReceived.begin();
+            mFramesReceived.erase(mFramesReceived.begin());
 
-        mFramesBeingEncoded.push_back(frame);
-        *buffer = new MediaBuffer(frame->pointer(), frame->size());
-        (*buffer)->setObserver(this);
-        (*buffer)->add_ref();
-        (*buffer)->meta_data()->setInt64(kKeyTime, frameTime);
+            frameTime = *mFrameTimes.begin();
+            mFrameTimes.erase(mFrameTimes.begin());
+            int64_t skipTimeUs;
+            if (!options || !options->getSkipFrame(&skipTimeUs)) {
+                skipTimeUs = frameTime;
+            }
+            if (skipTimeUs > frameTime) {
+                LOGV("skipTimeUs: %lld us > frameTime: %lld us",
+                    skipTimeUs, frameTime);
+                releaseOneRecordingFrame(frame);
+                ++mNumFramesDropped;
+                // Safeguard against the abuse of the kSkipFrame_Option.
+                if (skipTimeUs - frameTime >= 1E6) {
+                    LOGE("Frame skipping requested is way too long: %lld us",
+                        skipTimeUs - frameTime);
+                    return UNKNOWN_ERROR;
+                }
+            } else {
+                mFramesBeingEncoded.push_back(frame);
+                *buffer = new MediaBuffer(frame->pointer(), frame->size());
+                (*buffer)->setObserver(this);
+                (*buffer)->add_ref();
+                (*buffer)->meta_data()->setInt64(kKeyTime, frameTime);
+                return OK;
+            }
+        }
     }
     return OK;
 }

diff --git a/media/libstagefright/MediaSource.cpp b/media/libstagefright/MediaSource.cpp
@@ -32,6 +32,7 @@ void MediaSource::ReadOptions::reset() {
     mOptions = 0;
     mSeekTimeUs = 0;
     mLatenessUs = 0;
+    mSkipFrameUntilTimeUs = 0;
 }
 
 void MediaSource::ReadOptions::setSeekTo(int64_t time_us, SeekMode mode) {
@@ -53,6 +54,21 @@ bool MediaSource::ReadOptions::getSeekTo(
     return (mOptions & kSeekTo_Option) != 0;
 }
 
+void MediaSource::ReadOptions::clearSkipFrame() {
+    mOptions &= ~kSkipFrame_Option;
+    mSkipFrameUntilTimeUs = 0;
+}
+
+void MediaSource::ReadOptions::setSkipFrame(int64_t timeUs) {
+    mOptions |= kSkipFrame_Option;
+    mSkipFrameUntilTimeUs = timeUs;
+}
+
+bool MediaSource::ReadOptions::getSkipFrame(int64_t *timeUs) const {
+    *timeUs = mSkipFrameUntilTimeUs;
+    return (mOptions & kSkipFrame_Option) != 0;
+}
+
 void MediaSource::ReadOptions::setLateBy(int64_t lateness_us) {
     mLatenessUs = lateness_us;
 }

diff --git a/media/libstagefright/OMXCodec.cpp b/media/libstagefright/OMXCodec.cpp
@@ -1274,6 +1274,7 @@ OMXCodec::OMXCodec(
       mSeekTimeUs(-1),
       mSeekMode(ReadOptions::SEEK_CLOSEST_SYNC),
       mTargetTimeUs(-1),
+      mSkipTimeUs(-1),
       mLeftOverBuffer(NULL),
       mPaused(false) {
     mPortStatus[kPortIndexInput] = ENABLED;
@@ -2200,13 +2201,15 @@ void OMXCodec::drainInputBuffer(BufferInfo *info) {
     int32_t n = 0;
     for (;;) {
         MediaBuffer *srcBuffer;
+        MediaSource::ReadOptions options;
+        if (mSkipTimeUs >= 0) {
+            options.setSkipFrame(mSkipTimeUs);
+        }
         if (mSeekTimeUs >= 0) {
             if (mLeftOverBuffer) {
                 mLeftOverBuffer->release();
                 mLeftOverBuffer = NULL;
             }
-
-            MediaSource::ReadOptions options;
             options.setSeekTo(mSeekTimeUs, mSeekMode);
 
             mSeekTimeUs = -1;
@@ -2231,7 +2234,7 @@ void OMXCodec::drainInputBuffer(BufferInfo *info) {
 
             err = OK;
         } else {
-            err = mSource->read(&srcBuffer);
+            err = mSource->read(&srcBuffer, &options);
         }
 
         if (err != OK) {
@@ -2830,6 +2833,12 @@ status_t OMXCodec::read(
     if (options && options->getSeekTo(&seekTimeUs, &seekMode)) {
         seeking = true;
     }
+    int64_t skipTimeUs;
+    if (options && options->getSkipFrame(&skipTimeUs)) {
+        mSkipTimeUs = skipTimeUs;
+    } else {
+        mSkipTimeUs = -1;
+    }
 
     if (mInitialBufferSubmit) {
         mInitialBufferSubmit = false;

diff --git a/media/libstagefright/codecs/aacenc/AACEncoder.cpp b/media/libstagefright/codecs/aacenc/AACEncoder.cpp
@@ -224,6 +224,7 @@ status_t AACEncoder::read(
         if (mInputBuffer == NULL) {
             if (mSource->read(&mInputBuffer, options) != OK) {
                 if (mNumInputSamples == 0) {
+                    buffer->release();
                     return ERROR_END_OF_STREAM;
                 }
                 memset(&mInputFrame[mNumInputSamples],