Doing the downsampling on many small buffers prevent an optimal appro…

…ximation and causes a lot of noise.

This commit fixes it by processing fewer bigger buffers:
- The decoder buffers are accumulated into one big buffer (mToProcessBuffer) until it is full or the end of the stream has been reached and then it is processed.
- The encoder output buffer has been made bigger (1mb) to accomodate the larger input buffers. The concept of "output buffer overflow" has been removed because they output buffer should now always be big enough.
- mLastEncoderUs is now stored in the mPresentationtime to prevent the first buffer from each data source to be dropped

lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java

@@ -247,7 +247,7 @@ private TimeInterpolator createStepTimeInterpolator(@NonNull TrackType type, int
 
             private long mLastInterpolatedTime;
             private long mFirstInputTime = Long.MAX_VALUE;
-            private long mTimeBase = timebase + 10;
+            private long mTimeBase = timebase;
 
             @Override
             public long interpolate(@NonNull TrackType type, long time) {

lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultAudioStrategy.java

@@ -147,6 +147,7 @@ public TrackStatus createOutputFormat(@NonNull List<MediaFormat> inputFormats,
             outputFormat.setInteger(MediaFormat.KEY_AAC_PROFILE,
                     MediaCodecInfo.CodecProfileLevel.AACObjectLC);
         }
+        outputFormat.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 1024 * 1024);
         return TrackStatus.COMPRESSING;
     }
 

lib/src/main/java/com/otaliastudios/transcoder/time/PresentationTime.java

@@ -2,6 +2,7 @@
 
 public class PresentationTime {
     private long mTotalEncoderDurationUs = 0;
+    public long mLastEncoderUs = 0;
     public void increaseEncoderDuration(long encoderDurationUs) {
         mTotalEncoderDurationUs += encoderDurationUs;
     }

lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java

@@ -42,6 +42,8 @@ public class AudioEngine {
     private final MediaCodec mEncoder;
     private final int mDecoderSampleRate;
     private final int mEncoderSampleRate;
+    private ShortBuffer mToProcessBuffer;
+    private long mDecoderTimestampUs = 0;
     private final int mDecoderChannels;
     @SuppressWarnings("FieldCanBeLocal") private final int mEncoderChannels;
     private final AudioRemixer mRemixer;
@@ -50,8 +52,8 @@ public class AudioEngine {
     private final AudioStretcher mStretcher;
     private final TimeInterpolator mTimeInterpolator;
     private final PresentationTime mPresentationTime;
-    private long mLastDecoderUs = Long.MIN_VALUE;
-    private long mLastEncoderUs = Long.MIN_VALUE;
+    private long mLastDecoderUs = 0;
+    private boolean mFirstFrameProcessed = false; //TODO: investigate why the first buffer is always "ignored" and so need to be small
     private ShortBuffer mTempBuffer1;
     private ShortBuffer mTempBuffer2;
 
@@ -82,6 +84,9 @@ public AudioEngine(@NonNull MediaCodec decoder,
         mEncoderSampleRate = encoderOutputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE);
         mDecoderSampleRate = decoderOutputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE);
 
+        int mEncoderBufferSize = encoderOutputFormat.getInteger(MediaFormat.KEY_MAX_INPUT_SIZE);
+        mToProcessBuffer = ByteBuffer.allocateDirect(mEncoderBufferSize / 2).asShortBuffer();
+
         // Check channel count.
         mEncoderChannels = encoderOutputFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
         mDecoderChannels = decoderOutputFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
@@ -155,35 +160,54 @@ public boolean feedEncoder(@NonNull MediaCodecBuffers encoderBuffers, long timeo
         ShortBuffer encoderBuffer = encoderBuffers.getInputBuffer(encoderBufferIndex).asShortBuffer();
         encoderBuffer.clear();
 
-        // Get the latest raw buffer to be processed.
-        AudioBuffer buffer = mPendingBuffers.peek();
+        // accumulate the raw buffer into mToProcessBufferData to be processed.
+        boolean isEndOfStream = false;
+        boolean processBufferOverflow = false;
+        while (!mPendingBuffers.isEmpty()){
+            AudioBuffer pendingBuffer = mPendingBuffers.element();
+            if (pendingBuffer.isEndOfStream) {
+                isEndOfStream = true;
+                break;
+            } else if (pendingBuffer.decoderData.remaining() > mToProcessBuffer.remaining()) {
+                processBufferOverflow = true;
+                break;
+            } else {
+                mDecoderTimestampUs = pendingBuffer.decoderTimestampUs;
+                mToProcessBuffer.put(pendingBuffer.decoderData);
+                mDecoder.releaseOutputBuffer(pendingBuffer.decoderBufferIndex, false);
+                mPendingBuffers.remove();
+            }
+        }
 
-        // When endOfStream, just signal EOS and return false.
-        //noinspection ConstantConditions
-        if (buffer.isEndOfStream) {
+        if (mToProcessBuffer.position() > 0 && (!mFirstFrameProcessed || processBufferOverflow || isEndOfStream)) {
+            // Process the buffer.
+            mFirstFrameProcessed = true;
+            mToProcessBuffer.limit(mToProcessBuffer.position());
+            mToProcessBuffer.rewind();
+            AudioBuffer buffer = new AudioBuffer();
+            buffer.decoderData = mToProcessBuffer;
+            buffer.decoderTimestampUs = mDecoderTimestampUs;
+            process(buffer, encoderBuffer, encoderBufferIndex);
+            mToProcessBuffer.clear();
+        }
+        else if (isEndOfStream) {
             mEncoder.queueInputBuffer(encoderBufferIndex,
                     0,
                     0,
                     0,
                     MediaCodec.BUFFER_FLAG_END_OF_STREAM);
-            return false;
-        }
-
-        // Process the buffer.
-        boolean overflows = process(buffer, encoderBuffer, encoderBufferIndex);
-        if (overflows) {
-            // If this buffer does overflow, we will keep it in the queue and do
-            // not release. It will be used at the next cycle (presumably soon,
-            // since we return true).
-            return true;
-        } else {
-            // If this buffer does not overflow, it can be removed from our queue,
-            // re-added to empty, and also released from the decoder buffers.
+            mDecoder.releaseOutputBuffer(mPendingBuffers.element().decoderBufferIndex, false);
             mPendingBuffers.remove();
-            mEmptyBuffers.add(buffer);
-            mDecoder.releaseOutputBuffer(buffer.decoderBufferIndex, false);
-            return true;
+        } else {
+            mEncoder.queueInputBuffer(encoderBufferIndex,
+                    0,
+                    0,
+                    0,
+                    0
+            );
         }
+
+        return processBufferOverflow;
     }
 
     /**
@@ -212,23 +236,20 @@ public boolean feedEncoder(@NonNull MediaCodecBuffers encoderBuffers, long timeo
      * @param encoderBuffer coming from encoder. At this point this is in a cleared state
      * @param encoderBufferIndex the index of encoderBuffer so we can release it
      */
-    private boolean process(@NonNull AudioBuffer buffer, @NonNull ShortBuffer encoderBuffer, int encoderBufferIndex) {
+    private void process(@NonNull AudioBuffer buffer, @NonNull ShortBuffer encoderBuffer, int encoderBufferIndex) {
         // Only process the amount of data that can fill in the encoderBuffer.
         final int outputSize = encoderBuffer.remaining();
         final int totalInputSize = buffer.decoderData.remaining();
+
         int processedTotalInputSize = totalInputSize;
 
         // 1. Perform TimeInterpolator computation
         // TODO we should compare the NEXT timestamp with this, instead of comparing this with previous!
         long encoderUs = mTimeInterpolator.interpolate(TrackType.AUDIO, buffer.decoderTimestampUs);
-        if (mLastDecoderUs == Long.MIN_VALUE) {
-            mLastDecoderUs = buffer.decoderTimestampUs;
-            mLastEncoderUs = encoderUs;
-        }
         long decoderDurationUs = buffer.decoderTimestampUs - mLastDecoderUs;
-        long encoderDurationUs = encoderUs - mLastEncoderUs;
+        long encoderDurationUs = encoderUs - mPresentationTime.mLastEncoderUs;
         mLastDecoderUs = buffer.decoderTimestampUs;
-        mLastEncoderUs = encoderUs;
+        mPresentationTime.mLastEncoderUs = encoderUs;
         double stretchFactor = (double) encoderDurationUs / decoderDurationUs;
         LOG.i("process - time stretching -" +
                 " decoderDurationUs:" + decoderDurationUs +
@@ -244,22 +265,17 @@ private boolean process(@NonNull AudioBuffer buffer, @NonNull ShortBuffer encode
         processedTotalInputSize = (int) Math.ceil((double) processedTotalInputSize
                 * mEncoderSampleRate / mDecoderSampleRate);
 
-        // 4. Compare processedInputSize and outputSize. If processedInputSize > outputSize,
-        // we overflow. In this case, isolate the valid data.
-        boolean overflow = processedTotalInputSize > outputSize;
-        int overflowReduction = 0;
-        if (overflow) {
-            // Compute the input size that matches this output size.
-            double ratio = (double) processedTotalInputSize / totalInputSize; // > 1
-            overflowReduction = totalInputSize - (int) Math.floor((double) outputSize / ratio);
-            LOG.w("process - overflowing! Reduction:" + overflowReduction);
-            buffer.decoderData.limit(buffer.decoderData.limit() - overflowReduction);
-        }
+        // 4. Compare processedInputSize and outputSize. If processedInputSize > outputSize, throw
         final int inputSize = buffer.decoderData.remaining();
-        LOG.i("process - totalInputSize:" + totalInputSize +
+        String log = "process - totalInputSize:" + totalInputSize +
                 " processedTotalInputSize:" + processedTotalInputSize +
                 " outputSize:" + outputSize +
-                " inputSize:" + inputSize);
+                " inputSize:" + inputSize;
+        if (processedTotalInputSize > outputSize) {
+            throw new RuntimeException("Output buffer is too small for input: " + log);
+        } else {
+            LOG.i(log);
+        }
 
         // 5. Do the stretching. We need a bridge buffer for its output.
         ensureTempBuffer1((int) Math.ceil(inputSize * stretchFactor));
@@ -282,27 +298,17 @@ private boolean process(@NonNull AudioBuffer buffer, @NonNull ShortBuffer encode
         //8. Do the post processing
         encoderDurationUs = mPostProcessor.postProcess(mTempBuffer1, encoderBuffer, encoderDurationUs);
 
-        // 8. Add the bytes we have processed to the decoderTimestampUs, and restore the limit.
-        // We need an updated timestamp for the next cycle, since we will cycle on the same input
-        // buffer that has overflown.
-        if (overflow) {
-            buffer.decoderTimestampUs += AudioConversions.shortsToUs(inputSize, mDecoderSampleRate,
-                    mDecoderChannels);
-            buffer.decoderData.limit(buffer.decoderData.limit() + overflowReduction);
-        }
-
         // 9. Write the buffer.
         // This is the encoder buffer: we have likely written it all, but let's use
         // encoderBuffer.position() to know how much anyway.
-        mPresentationTime.increaseEncoderDuration(encoderDurationUs);
         mEncoder.queueInputBuffer(encoderBufferIndex,
                 0,
                 encoderBuffer.position() * BYTES_PER_SHORT,
                 mPresentationTime.getEncoderPresentationTimeUs(),
                 0
         );
+        mPresentationTime.increaseEncoderDuration(encoderDurationUs);
 
-        return overflow;
     }
 
     private void ensureTempBuffer1(int desiredSize) {