Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Issue 2071329: audio track is shorter than video track for video capt…

…ure on sholes

Add API to retrieve number of frames dropped by audio input kernel driver.

Submitted on behalf of Masaki Sato <masaki.sato@motorola.com>
  • Loading branch information...
commit 8dea89784955d6a02ab971f67e2a8315f1e88aa4 1 parent b7cd675
Eric Laurent authored
14 android/author/android_audio_input.cpp
View
@@ -23,6 +23,7 @@
#include "android_audio_input.h"
#include "pv_mime_string_utils.h"
#include "oscl_dll.h"
+#include "pvmf_duration_infomessage.h"
#include <media/AudioRecord.h>
#include <sys/prctl.h>
@@ -1173,6 +1174,19 @@ int AndroidAudioInput::audin_thread_func() {
iOSSRequestQueue.erase(&iOSSRequestQueue[0]);
iOSSRequestQueueLock.Unlock();
+ uint32 inputFramesLost = record->getInputFramesLost();
+ if (inputFramesLost > 0) {
+ uint32 dataDurationLost = inputFramesLost * 1000 / iAudioSamplingRate;
+ LOGW("Notifying %d observers of %u ms of lost audio", iObservers.size(), dataDurationLost);
+
+ for(uint32 i = 0; i < iObservers.size(); i++) {
+ int32 leavecode = 0;
+ PVMFDurationInfoMessage* eventMsg = NULL;
+ OSCL_TRY(leavecode, eventMsg = OSCL_NEW(PVMFDurationInfoMessage, (dataDurationLost)));
+ iObservers[i]->ReportInfoEvent(PVMFInfoOverflow, OSCL_STATIC_CAST(PVInterface*, eventMsg));
+ }
+ }
+
int numOfBytes = record->read(data, kBufferSize);
//LOGV("read %d bytes", numOfBytes);
if (numOfBytes <= 0)
33 android/author/android_camera_input.cpp
View
@@ -53,6 +53,7 @@ AndroidCameraInput::AndroidCameraInput()
iAuthorClock(NULL),
iClockNotificationsInf(NULL),
iAudioFirstFrameTs(0),
+ iAudioLossDuration(0),
pPmemInfo(NULL)
{
LOGV("constructor(%p)", this);
@@ -72,6 +73,7 @@ AndroidCameraInput::AndroidCameraInput()
mFlags = 0;
iFrameQueue.reserve(5);
iFrameQueueMutex.Create();
+ iAudioLossMutex.Create();
// setup callback listener
mListener = new AndroidCameraInputListener(this);
@@ -121,6 +123,7 @@ AndroidCameraInput::~AndroidCameraInput()
mCamera.clear();
}
iFrameQueueMutex.Close();
+ iAudioLossMutex.Close();
mListener.clear();
}
@@ -423,7 +426,7 @@ void AndroidCameraInput::writeComplete(PVMFStatus aStatus,
#endif
// View finder freeze detection
// Note for low frame rate, we don't bother to log view finder freezes
- int processingTimeInMs = (systemTime()/1000000L - iAudioFirstFrameTs) - data.iXferHeader.timestamp;
+ int processingTimeInMs = (systemTime()/1000000L - (iAudioFirstFrameTs + iAudioLossDuration)) - data.iXferHeader.timestamp;
if (processingTimeInMs >= VIEW_FINDER_FREEZE_DETECTION_TOLERANCE && mFrameRate >= 10.0) {
LOGW("Frame %p takes too long (%d ms) to process, staring at %d", data.iFrameBuffer.get(), processingTimeInMs, iAudioFirstFrameTs);
}
@@ -992,6 +995,7 @@ PVMFStatus AndroidCameraInput::DoReset()
// Remove and destroy the clock state observer
RemoveDestroyClockObs();
iDataEventCounter = 0;
+ iAudioLossDuration = 0;
iWriteState = EWriteOK;
if ( (iState == STATE_STARTED) || (iState == STATE_PAUSED) ) {
if (mCamera != NULL) {
@@ -1184,13 +1188,24 @@ PVMFStatus AndroidCameraInput::postWriteAsync(nsecs_t timestamp, const sp<IMemor
if (iAudioFirstFrameTs == 0)
iAudioFirstFrameTs = ts;
- if (ts < iAudioFirstFrameTs) {
+ iAudioLossMutex.Lock();
+ if (ts < iAudioFirstFrameTs + iAudioLossDuration) {
// Drop the frame
+ iAudioLossMutex.Unlock();
mCamera->releaseRecordingFrame(frame);
return PVMFSuccess;
} else {
- // calculate timestamp as offset from start time
- ts -= iAudioFirstFrameTs;
+ // calculate timestamp as offset from start time plus lost audio
+ ts -= (iAudioFirstFrameTs + iAudioLossDuration);
+ }
+ iAudioLossMutex.Unlock();
+
+ // Determine how much video to drop so when we resync to the audio
+ // time we don't go into old video (we need increasing timestamps)
+ if (ts <= iTimeStamp) {
+ LOGD("Dropping video frame to catch up for audio ts %lu, dropUntil %lu", ts, iTimeStamp);
+ mCamera->releaseRecordingFrame(frame);
+ return PVMFSuccess;
}
// Make sure that no two samples have the same timestamp
@@ -1249,6 +1264,15 @@ PVMFStatus AndroidCameraInput::postWriteAsync(nsecs_t timestamp, const sp<IMemor
return PVMFSuccess;
}
+// Value is expected to be in ms
+void AndroidCameraInput::setAudioLossDuration(uint32 duration)
+{
+ iAudioLossMutex.Lock();
+ LOGD("Update for lost audio for %lu for existing duration %lu", duration, iAudioLossDuration);
+ iAudioLossDuration += duration;
+ iAudioLossMutex.Unlock();
+}
+
// camera callback interface
void AndroidCameraInputListener::postData(int32_t msgType, const sp<IMemory>& dataPtr)
{
@@ -1290,4 +1314,3 @@ void AndroidCameraInput::RemoveDestroyClockObs()
}
-
3  android/author/android_camera_input.h
View
@@ -410,6 +410,7 @@ class AndroidCameraInput
// add for Camcorder
PVMFStatus postWriteAsync(nsecs_t timestamp, const sp<IMemory>& frame);
+ void setAudioLossDuration(uint32 duration);
bool isRecorderStarting() { return iState==STATE_STARTED?true:false; }
@@ -531,6 +532,8 @@ class AndroidCameraInput
PVMFMediaClockNotificationsInterface *iClockNotificationsInf;
uint32 iAudioFirstFrameTs;
+ OsclMutex iAudioLossMutex;
+ uint32 iAudioLossDuration;
PVRefBufferAlloc mbufferAlloc;
// data structures for tunneling buffers
27 android/author/authordriver.cpp
View
@@ -27,6 +27,7 @@
#include "pv_omxcore.h"
#include <sys/prctl.h>
#include "pvmf_composer_size_and_duration.h"
+#include "pvmf_duration_infomessage.h"
#include "android_camera_input.h"
using namespace android;
@@ -1407,10 +1408,28 @@ void AuthorDriver::HandleInformationalEvent(const PVAsyncInformationalEvent& aEv
LOGV("HandleInformationalEvent(%d)", event_type);
}
- mListener->notify(
- MEDIA_RECORDER_EVENT_INFO,
- GetMediaRecorderInfoCode(aEvent),
- aEvent.GetEventType());
+ if (PVMFInfoOverflow == event_type && mVideoInputMIO) {
+ PVUuid infomsguuid = PVMFDurationInfoMessageInterfaceUUID;
+ PVMFDurationInfoMessageInterface* eventMsg = NULL;
+ PVInterface* infoExtInterface = aEvent.GetEventExtensionInterface();
+ if (infoExtInterface &&
+ infoExtInterface->queryInterface(infomsguuid, (PVInterface*&)eventMsg) && eventMsg) {
+ PVUuid eventuuid;
+ int32 infoCode;
+ eventMsg->GetCodeUUID(infoCode, eventuuid);
+ if (eventuuid == infomsguuid) {
+ uint32 duration = eventMsg->GetDuration();
+ ((AndroidCameraInput *)mVideoInputMIO)->setAudioLossDuration(duration);
+ }
+ eventMsg->removeRef();
+ eventMsg = NULL;
+ }
+ } else {
+ mListener->notify(
+ MEDIA_RECORDER_EVENT_INFO,
+ GetMediaRecorderInfoCode(aEvent),
+ aEvent.GetEventType());
+ }
}
status_t AuthorDriver::setListener(const sp<IMediaPlayerClient>& listener) {
8 engines/author/src/pvauthorengine.cpp
View
@@ -522,6 +522,14 @@ void PVAuthorEngine::HandleNodeInformationalEvent(const PVMFAsyncEvent& aEvent)
PushCmdInFront(cmd);
}
break;
+
+ case PVMFInfoOverflow:
+ {
+ PVAsyncInformationalEvent event(aEvent.GetEventType(), NULL, aEvent.GetEventExtensionInterface());
+ iInfoEventObserver->HandleInformationalEvent(event);
+ }
+ break;
+
case PVMF_COMPOSER_FILESIZE_PROGRESS:
case PVMF_COMPOSER_DURATION_PROGRESS:
{
8 nodes/pvmediainputnode/src/pvmf_media_input_node.cpp
View
@@ -560,8 +560,12 @@ OSCL_EXPORT_REF void PvmfMediaInputNode::ReportErrorEvent(PVMFEventType aEventTy
////////////////////////////////////////////////////////////////////////////
OSCL_EXPORT_REF void PvmfMediaInputNode::ReportInfoEvent(PVMFEventType aEventType, PVInterface* aExtMsg)
{
- OSCL_UNUSED_ARG(aEventType);
- OSCL_UNUSED_ARG(aExtMsg);
+ if (aEventType == PVMFInfoOverflow) {
+ PVMFNodeInterface::ReportInfoEvent(aEventType, NULL, aExtMsg);
+ } else {
+ OSCL_UNUSED_ARG(aEventType);
+ OSCL_UNUSED_ARG(aExtMsg);
+ }
}
////////////////////////////////////////////////////////////////////////////
Please sign in to comment.
Something went wrong with that request. Please try again.