From 492996dbb4794a9e2c544d9bc3d56b8afb6edb6b Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Mon, 22 Jul 2019 19:59:34 -0300 Subject: [PATCH 01/22] Add TimeInterpolator interface and option --- .../transcoder/TranscoderOptions.java | 29 +++++++++++++++++++ .../transcoder/engine/TranscoderEngine.java | 2 +- .../transcoder/engine/TranscoderMuxer.java | 6 +++- .../time/DefaultTimeInterpolator.java | 17 +++++++++++ .../transcoder/time/TimeInterpolator.java | 21 ++++++++++++++ 5 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 lib/src/main/java/com/otaliastudios/transcoder/time/DefaultTimeInterpolator.java create mode 100644 lib/src/main/java/com/otaliastudios/transcoder/time/TimeInterpolator.java diff --git a/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java b/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java index a10c7863..ffce01fc 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java @@ -12,6 +12,8 @@ import com.otaliastudios.transcoder.strategy.DefaultAudioStrategy; import com.otaliastudios.transcoder.strategy.DefaultVideoStrategies; import com.otaliastudios.transcoder.strategy.OutputStrategy; +import com.otaliastudios.transcoder.time.DefaultTimeInterpolator; +import com.otaliastudios.transcoder.time.TimeInterpolator; import com.otaliastudios.transcoder.validator.DefaultValidator; import com.otaliastudios.transcoder.validator.Validator; @@ -34,6 +36,7 @@ private TranscoderOptions() {} private OutputStrategy videoOutputStrategy; private Validator validator; private int rotation; + private TimeInterpolator timeInterpolator; TranscoderListener listener; Handler listenerHandler; @@ -68,6 +71,11 @@ public int getRotation() { return rotation; } + @NonNull + public TimeInterpolator getTimeInterpolator() { + return timeInterpolator; + } + public static class Builder { private String outPath; private DataSource dataSource; @@ -77,6 +85,7 @@ public static class Builder { private OutputStrategy videoOutputStrategy; private Validator validator; private int rotation; + private TimeInterpolator timeInterpolator; Builder(@NonNull String outPath) { this.outPath = outPath; @@ -176,6 +185,8 @@ public Builder setValidator(@Nullable Validator validator) { /** * The clockwise rotation to be applied to the input video frames. + * Defaults to 0, which leaves the input rotation unchanged. + * * @param rotation either 0, 90, 180 or 270 * @return this for chaining */ @@ -186,6 +197,20 @@ public Builder setRotation(int rotation) { return this; } + /** + * Sets a {@link TimeInterpolator} to change the frames timestamps - either video or + * audio or both - before they are written into the output file. + * Defaults to {@link com.otaliastudios.transcoder.time.DefaultTimeInterpolator}. + * + * @param timeInterpolator a time interpolator + * @return this for chaining + */ + @NonNull + public Builder setTimeInterpolator(@NonNull TimeInterpolator timeInterpolator) { + this.timeInterpolator = timeInterpolator; + return this; + } + @NonNull public TranscoderOptions build() { if (listener == null) { @@ -214,6 +239,9 @@ public TranscoderOptions build() { if (validator == null) { validator = new DefaultValidator(); } + if (timeInterpolator == null) { + timeInterpolator = new DefaultTimeInterpolator(); + } TranscoderOptions options = new TranscoderOptions(); options.listener = listener; options.dataSource = dataSource; @@ -223,6 +251,7 @@ public TranscoderOptions build() { options.videoOutputStrategy = videoOutputStrategy; options.validator = validator; options.rotation = rotation; + options.timeInterpolator = timeInterpolator; return options; } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderEngine.java b/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderEngine.java index 2f7cd020..30fa2d1b 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderEngine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderEngine.java @@ -226,7 +226,7 @@ private void setUpTrackTranscoder(@NonNull TranscoderOptions options, private void setupTrackTranscoders(@NonNull TranscoderOptions options) { mTracks = Tracks.create(mExtractor); - TranscoderMuxer muxer = new TranscoderMuxer(mMuxer, mTracks); + TranscoderMuxer muxer = new TranscoderMuxer(mMuxer, mTracks, options.getTimeInterpolator()); setUpTrackTranscoder(options, muxer, TrackType.VIDEO); setUpTrackTranscoder(options, muxer, TrackType.AUDIO); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderMuxer.java b/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderMuxer.java index bffec0bb..03c0c15d 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderMuxer.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderMuxer.java @@ -20,6 +20,7 @@ import android.media.MediaMuxer; import com.otaliastudios.transcoder.internal.Logger; +import com.otaliastudios.transcoder.time.TimeInterpolator; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -61,10 +62,12 @@ private void toBufferInfo(@NonNull MediaCodec.BufferInfo bufferInfo, int offset) private boolean mMuxerStarted; private final List mQueue = new ArrayList<>(); private ByteBuffer mQueueBuffer; + private TimeInterpolator mTimeInterpolator; - TranscoderMuxer(@NonNull MediaMuxer muxer, @NonNull Tracks info) { + TranscoderMuxer(@NonNull MediaMuxer muxer, @NonNull Tracks info, @NonNull TimeInterpolator timeInterpolator) { mMuxer = muxer; mTracks = info; + mTimeInterpolator = timeInterpolator; } /** @@ -110,6 +113,7 @@ public void setOutputFormat(@NonNull TrackType trackType, @NonNull MediaFormat f public void write(@NonNull TrackType type, @NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) { if (mMuxerStarted) { + bufferInfo.presentationTimeUs = mTimeInterpolator.interpolate(type, bufferInfo.presentationTimeUs); mMuxer.writeSampleData(mTracks.outputIndex(type), byteBuf, bufferInfo); } else { enqueue(type, byteBuf, bufferInfo); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/time/DefaultTimeInterpolator.java b/lib/src/main/java/com/otaliastudios/transcoder/time/DefaultTimeInterpolator.java new file mode 100644 index 00000000..d4936afc --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/time/DefaultTimeInterpolator.java @@ -0,0 +1,17 @@ +package com.otaliastudios.transcoder.time; + +import androidx.annotation.NonNull; + +import com.otaliastudios.transcoder.engine.TrackType; + +/** + * A {@link TimeInterpolator} that does no time interpolation or correction - + * it just returns the input time. + */ +public class DefaultTimeInterpolator implements TimeInterpolator { + + @Override + public long interpolate(@NonNull TrackType type, long time) { + return time; + } +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/time/TimeInterpolator.java b/lib/src/main/java/com/otaliastudios/transcoder/time/TimeInterpolator.java new file mode 100644 index 00000000..515b4471 --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/time/TimeInterpolator.java @@ -0,0 +1,21 @@ +package com.otaliastudios.transcoder.time; + +import androidx.annotation.NonNull; + +import com.otaliastudios.transcoder.engine.TrackType; + +/** + * An interface to redefine the time between video or audio frames. + */ +public interface TimeInterpolator { + + /** + * Given the track type (audio or video) and the frame timestamp in microseconds, + * should return the corrected timestamp. + * + * @param type track type + * @param time frame timestamp in microseconds + * @return the new frame timestamp + */ + long interpolate(@NonNull TrackType type, long time); +} From 2c27825b0c93a20a84123e55680f9ea2a0272f9c Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Mon, 22 Jul 2019 20:04:13 -0300 Subject: [PATCH 02/22] Create non functional SpeedTimeInterpolator plus shorthand methods --- .../transcoder/TranscoderOptions.java | 16 ++++++++++ .../time/SpeedTimeInterpolator.java | 32 +++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 lib/src/main/java/com/otaliastudios/transcoder/time/SpeedTimeInterpolator.java diff --git a/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java b/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java index ffce01fc..32f964b4 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java @@ -13,6 +13,7 @@ import com.otaliastudios.transcoder.strategy.DefaultVideoStrategies; import com.otaliastudios.transcoder.strategy.OutputStrategy; import com.otaliastudios.transcoder.time.DefaultTimeInterpolator; +import com.otaliastudios.transcoder.time.SpeedTimeInterpolator; import com.otaliastudios.transcoder.time.TimeInterpolator; import com.otaliastudios.transcoder.validator.DefaultValidator; import com.otaliastudios.transcoder.validator.Validator; @@ -206,11 +207,26 @@ public Builder setRotation(int rotation) { * @return this for chaining */ @NonNull + @SuppressWarnings("WeakerAccess") public Builder setTimeInterpolator(@NonNull TimeInterpolator timeInterpolator) { this.timeInterpolator = timeInterpolator; return this; } + /** + * Shorthand for calling {@link #setTimeInterpolator(TimeInterpolator)} + * and passing a {@link com.otaliastudios.transcoder.time.SpeedTimeInterpolator}. + * This interpolator can modify the video speed by the given factor. + * + * @param speedFactor a factor, greather than 0 + * @return this for chaining + */ + @NonNull + @SuppressWarnings("unused") + public Builder setSpeed(float speedFactor) { + return setTimeInterpolator(new SpeedTimeInterpolator(speedFactor)); + } + @NonNull public TranscoderOptions build() { if (listener == null) { diff --git a/lib/src/main/java/com/otaliastudios/transcoder/time/SpeedTimeInterpolator.java b/lib/src/main/java/com/otaliastudios/transcoder/time/SpeedTimeInterpolator.java new file mode 100644 index 00000000..8fdd3c18 --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/time/SpeedTimeInterpolator.java @@ -0,0 +1,32 @@ +package com.otaliastudios.transcoder.time; + +import androidx.annotation.NonNull; + +import com.otaliastudios.transcoder.engine.TrackType; + +/** + * A {@link TimeInterpolator} that modifies the playback speed by the given + * float factor. A factor less than 1 will slow down, while a bigger factor will + * accelerate. + */ +public class SpeedTimeInterpolator implements TimeInterpolator { + + private float mFactor; + + /** + * Creates a new speed interpolator for the given factor. + * Throws if factor is less than 0 or equal to 0. + * @param factor a factor + */ + public SpeedTimeInterpolator(float factor) { + if (factor <= 0) { + throw new IllegalArgumentException("Invalid speed factor: " + factor); + } + mFactor = factor; + } + + @Override + public long interpolate(@NonNull TrackType type, long time) { + return time; + } +} From a04c8efb6d5ee9ee888f0b5186840664219c01d2 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Mon, 22 Jul 2019 20:13:00 -0300 Subject: [PATCH 03/22] Add SpeedTimeInterpolator documentation --- README.md | 47 +++++++++++++++++++ .../transcoder/engine/TrackType.java | 1 - 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1193be90..a9711642 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ It features a lot of improvements over the original project, including:* - *Multithreading support* - *Crop to any aspect ratio* - *Set output video rotation* +- *Change output video speed (0.5x, 2x or any float)* - *Various bugs fixed* - *[Input](#data-sources): Accept content Uris and other types* - *[Real error handling](#listening-for-events) instead of errors being thrown* @@ -316,6 +317,52 @@ Transcoder.into(filePath) // ... ``` +#### Time interpolation + +We offer APIs to change the timestamp of each video and audio frame. You can pass a `TimeInterpolator` +to the transcoder builder to be able to receive the frame timestamp as input, and return a new one +as output. + +```java +Transcoder.into(filePath) + .setTimeInterpolator(timeInterpolator) + // ... +``` + +As an example, this is the implementation of the default interpolator, called `DefaultTimeInterpolator`, +that will just return the input time unchanged: + +```java +@Override +public long interpolate(@NonNull TrackType type, long time) { + // Receive input time in microseconds and return a possibly different one. + return time; +} +``` + +It should be obvious that returning invalid times can make the process crash at any point, or at least +the transcoding operation fail. + +#### Video speed + +We also offer a special time interpolator called `SpeedTimeInterpolator` that accepts a `float` parameter +and will modify the video speed. + +- A speed factor equal to 1 will leave speed unchanged +- A speed factor < 1 will slow the video down +- A speed factor > 1 will accelerate the video + +This interpolator can be set using `setTimeInterpolator(TimeInterpolator)`, or, as a shorthand, +using `setSpeed(float)`: + +```java +Transcoder.into(filePath) + .setSpeed(0.5F) // 0.5x + .setSpeed(1F) // Unchanged + .setSpeed(2F) // Twice as fast + // ... +``` + ## Compatibility As stated pretty much everywhere, **not all codecs/devices/manufacturers support all sizes/options**. diff --git a/lib/src/main/java/com/otaliastudios/transcoder/engine/TrackType.java b/lib/src/main/java/com/otaliastudios/transcoder/engine/TrackType.java index 4beb0a8c..e21b094c 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/engine/TrackType.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/engine/TrackType.java @@ -2,7 +2,6 @@ /** * Represent the types of a track (either video or audio). - * Used internally. */ public enum TrackType { VIDEO, AUDIO From 6b08bca3e57d7443f6e754efe7d78807c09f04ae Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Mon, 22 Jul 2019 20:15:43 -0300 Subject: [PATCH 04/22] Add speed control to demo app --- .../transcoder/demo/TranscoderActivity.java | 12 ++++++ .../main/res/layout/activity_transcoder.xml | 37 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java b/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java index 1ec9f17d..2f4ed6cf 100644 --- a/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java +++ b/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java @@ -44,6 +44,8 @@ public class TranscoderActivity extends AppCompatActivity implements private RadioGroup mVideoResolutionGroup; private RadioGroup mVideoAspectGroup; private RadioGroup mVideoRotationGroup; + private RadioGroup mSpeedGroup; + private ProgressBar mProgressView; private TextView mButtonView; @@ -79,6 +81,8 @@ protected void onCreate(Bundle savedInstanceState) { mVideoResolutionGroup = findViewById(R.id.resolution); mVideoAspectGroup = findViewById(R.id.aspect); mVideoRotationGroup = findViewById(R.id.rotation); + mSpeedGroup = findViewById(R.id.speed); + mAudioChannelsGroup.setOnCheckedChangeListener(this); mVideoFramesGroup.setOnCheckedChangeListener(this); mVideoResolutionGroup.setOnCheckedChangeListener(this); @@ -166,6 +170,13 @@ private void transcode() { default: rotation = 0; } + float speed; + switch (mSpeedGroup.getCheckedRadioButtonId()) { + case R.id.speed_05x: speed = 0.5F; break; + case R.id.speed_2x: speed = 2F; break; + default: speed = 1F; + } + // Launch the transcoding operation. mTranscodeStartTime = SystemClock.uptimeMillis(); setIsTranscoding(true); @@ -175,6 +186,7 @@ private void transcode() { .setAudioOutputStrategy(mTranscodeAudioStrategy) .setVideoOutputStrategy(mTranscodeVideoStrategy) .setRotation(rotation) + .setSpeed(speed) .transcode(); } diff --git a/demo/src/main/res/layout/activity_transcoder.xml b/demo/src/main/res/layout/activity_transcoder.xml index 154c800e..af6da133 100644 --- a/demo/src/main/res/layout/activity_transcoder.xml +++ b/demo/src/main/res/layout/activity_transcoder.xml @@ -212,6 +212,43 @@ android:layout_height="wrap_content" /> + + + + + + + + + Date: Mon, 22 Jul 2019 21:18:22 -0300 Subject: [PATCH 05/22] Add SpeedTimeInterpolator implementation --- .../time/SpeedTimeInterpolator.java | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/time/SpeedTimeInterpolator.java b/lib/src/main/java/com/otaliastudios/transcoder/time/SpeedTimeInterpolator.java index 8fdd3c18..b69c4462 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/time/SpeedTimeInterpolator.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/time/SpeedTimeInterpolator.java @@ -4,6 +4,9 @@ import com.otaliastudios.transcoder.engine.TrackType; +import java.util.HashMap; +import java.util.Map; + /** * A {@link TimeInterpolator} that modifies the playback speed by the given * float factor. A factor less than 1 will slow down, while a bigger factor will @@ -11,7 +14,8 @@ */ public class SpeedTimeInterpolator implements TimeInterpolator { - private float mFactor; + private double mFactor; + private final Map mTrackData = new HashMap<>(); /** * Creates a new speed interpolator for the given factor. @@ -25,8 +29,36 @@ public SpeedTimeInterpolator(float factor) { mFactor = factor; } + /** + * Returns the factor passed to the constructor. + * @return the factor + */ + @SuppressWarnings("unused") + public float getFactor() { + return (float) mFactor; + } + @Override public long interpolate(@NonNull TrackType type, long time) { - return time; + if (!mTrackData.containsKey(type)) { + mTrackData.put(type, new TrackData()); + } + TrackData data = mTrackData.get(type); + //noinspection ConstantConditions + if (data.lastRealTime == Long.MIN_VALUE) { + data.lastRealTime = time; + data.lastCorrectedTime = time; + } else { + long realDelta = time - data.lastRealTime; + long correctedDelta = (long) ((double) realDelta * mFactor); + data.lastRealTime = time; + data.lastCorrectedTime += correctedDelta; + } + return data.lastCorrectedTime; + } + + private static class TrackData { + private long lastRealTime = Long.MIN_VALUE; + private long lastCorrectedTime = Long.MIN_VALUE; } } From 608ba3ce5383e42442d07804be09990b1c819ff3 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Mon, 22 Jul 2019 21:43:45 -0300 Subject: [PATCH 06/22] Remove audio when speed is used, fix bugs --- README.md | 4 ++++ .../transcoder/demo/TranscoderActivity.java | 9 +++++++-- demo/src/main/res/layout/activity_transcoder.xml | 2 +- .../otaliastudios/transcoder/engine/TranscoderMuxer.java | 9 ++++++++- .../transcoder/time/SpeedTimeInterpolator.java | 5 ++++- 5 files changed, 24 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a9711642..dfddd313 100644 --- a/README.md +++ b/README.md @@ -319,6 +319,8 @@ Transcoder.into(filePath) #### Time interpolation +**Note: when time is changed, you must make sure to remove the audio track or the process breaks.** + We offer APIs to change the timestamp of each video and audio frame. You can pass a `TimeInterpolator` to the transcoder builder to be able to receive the frame timestamp as input, and return a new one as output. @@ -345,6 +347,8 @@ the transcoding operation fail. #### Video speed +**Note: when time is changed, you must make sure to remove the audio track or the process breaks.** + We also offer a special time interpolator called `SpeedTimeInterpolator` that accepts a `float` parameter and will modify the video speed. diff --git a/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java b/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java index 2f4ed6cf..dc1cdfd3 100644 --- a/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java +++ b/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java @@ -11,13 +11,16 @@ import com.otaliastudios.transcoder.Transcoder; import com.otaliastudios.transcoder.TranscoderListener; +import com.otaliastudios.transcoder.engine.TrackStatus; import com.otaliastudios.transcoder.internal.Logger; import com.otaliastudios.transcoder.strategy.DefaultAudioStrategy; import com.otaliastudios.transcoder.strategy.DefaultVideoStrategy; import com.otaliastudios.transcoder.strategy.OutputStrategy; +import com.otaliastudios.transcoder.strategy.RemoveTrackStrategy; import com.otaliastudios.transcoder.strategy.size.AspectRatioResizer; import com.otaliastudios.transcoder.strategy.size.FractionResizer; import com.otaliastudios.transcoder.strategy.size.PassThroughResizer; +import com.otaliastudios.transcoder.validator.DefaultValidator; import java.io.File; import java.io.IOException; @@ -176,6 +179,8 @@ private void transcode() { case R.id.speed_2x: speed = 2F; break; default: speed = 1F; } + OutputStrategy audioStrategy = speed == 1F ? mTranscodeAudioStrategy : new RemoveTrackStrategy(); + OutputStrategy videoStrategy = mTranscodeVideoStrategy; // Launch the transcoding operation. mTranscodeStartTime = SystemClock.uptimeMillis(); @@ -183,8 +188,8 @@ private void transcode() { mTranscodeFuture = Transcoder.into(mTranscodeOutputFile.getAbsolutePath()) .setDataSource(this, mTranscodeInputUri) .setListener(this) - .setAudioOutputStrategy(mTranscodeAudioStrategy) - .setVideoOutputStrategy(mTranscodeVideoStrategy) + .setAudioOutputStrategy(audioStrategy) + .setVideoOutputStrategy(videoStrategy) .setRotation(rotation) .setSpeed(speed) .transcode(); diff --git a/demo/src/main/res/layout/activity_transcoder.xml b/demo/src/main/res/layout/activity_transcoder.xml index af6da133..3ddd3312 100644 --- a/demo/src/main/res/layout/activity_transcoder.xml +++ b/demo/src/main/res/layout/activity_transcoder.xml @@ -218,7 +218,7 @@ android:padding="16dp" android:layout_width="match_parent" android:layout_height="wrap_content" - android:text="Speed" /> + android:text="Playback speed" /> Date: Tue, 23 Jul 2019 12:23:48 +0100 Subject: [PATCH 07/22] Create BaseTrackTranscoder for both audio and video --- .../transcode/AudioTrackTranscoder.java | 191 +--------- .../transcode/BaseTrackTranscoder.java | 330 ++++++++++++++++++ .../transcode/VideoTrackTranscoder.java | 255 ++++---------- .../internal/VideoDecoderOutput.java | 19 +- 4 files changed, 426 insertions(+), 369 deletions(-) create mode 100644 lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java index ac9aaeed..59fed50c 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java @@ -5,205 +5,44 @@ import android.media.MediaFormat; import androidx.annotation.NonNull; -import androidx.annotation.Nullable; import com.otaliastudios.transcoder.engine.TrackType; import com.otaliastudios.transcoder.engine.TranscoderMuxer; -import com.otaliastudios.transcoder.internal.MediaCodecBuffers; import com.otaliastudios.transcoder.transcode.internal.AudioChannel; -import java.io.IOException; - -public class AudioTrackTranscoder implements TrackTranscoder { - - private static final TrackType SAMPLE_TYPE = TrackType.AUDIO; - - private static final int DRAIN_STATE_NONE = 0; - private static final int DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY = 1; - private static final int DRAIN_STATE_CONSUMED = 2; - - private final MediaExtractor mExtractor; - private final TranscoderMuxer mMuxer; - private long mWrittenPresentationTimeUs; - - private final int mTrackIndex; - - private final MediaCodec.BufferInfo mBufferInfo = new MediaCodec.BufferInfo(); - private MediaCodec mDecoder; - private MediaCodec mEncoder; - private MediaFormat mActualOutputFormat; - - private MediaCodecBuffers mDecoderBuffers; - private MediaCodecBuffers mEncoderBuffers; - - private boolean mIsExtractorEOS; - private boolean mIsDecoderEOS; - private boolean mIsEncoderEOS; - private boolean mDecoderStarted; - private boolean mEncoderStarted; +public class AudioTrackTranscoder extends BaseTrackTranscoder { private AudioChannel mAudioChannel; public AudioTrackTranscoder(@NonNull MediaExtractor extractor, int trackIndex, @NonNull TranscoderMuxer muxer) { - mExtractor = extractor; - mTrackIndex = trackIndex; - mMuxer = muxer; + super(extractor, trackIndex, muxer, TrackType.AUDIO); } @Override - public void setUp(@NonNull MediaFormat desiredOutputFormat) { - mExtractor.selectTrack(mTrackIndex); - try { - mEncoder = MediaCodec.createEncoderByType(desiredOutputFormat.getString(MediaFormat.KEY_MIME)); - } catch (IOException e) { - throw new IllegalStateException(e); - } - mEncoder.configure(desiredOutputFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); - mEncoder.start(); - mEncoderStarted = true; - mEncoderBuffers = new MediaCodecBuffers(mEncoder); - - final MediaFormat inputFormat = mExtractor.getTrackFormat(mTrackIndex); - try { - mDecoder = MediaCodec.createDecoderByType(inputFormat.getString(MediaFormat.KEY_MIME)); - } catch (IOException e) { - throw new IllegalStateException(e); - } - mDecoder.configure(inputFormat, null, null, 0); - mDecoder.start(); - mDecoderStarted = true; - mDecoderBuffers = new MediaCodecBuffers(mDecoder); - - mAudioChannel = new AudioChannel(mDecoder, mEncoder, desiredOutputFormat); + protected void onCodecsStarted(@NonNull MediaFormat inputFormat, @NonNull MediaFormat outputFormat, @NonNull MediaCodec decoder, @NonNull MediaCodec encoder) { + super.onCodecsStarted(inputFormat, outputFormat, decoder, encoder); + mAudioChannel = new AudioChannel(decoder, encoder, outputFormat); } @Override - public boolean stepPipeline() { - boolean busy = false; - - int status; - while (drainEncoder(0) != DRAIN_STATE_NONE) busy = true; - do { - status = drainDecoder(0); - if (status != DRAIN_STATE_NONE) busy = true; - // NOTE: not repeating to keep from deadlock when encoder is full. - } while (status == DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY); - - while (mAudioChannel.feedEncoder(0)) busy = true; - while (drainExtractor(0) != DRAIN_STATE_NONE) busy = true; - - return busy; - } - - @SuppressWarnings("SameParameterValue") - private int drainExtractor(long timeoutUs) { - if (mIsExtractorEOS) return DRAIN_STATE_NONE; - int trackIndex = mExtractor.getSampleTrackIndex(); - if (trackIndex >= 0 && trackIndex != mTrackIndex) { - return DRAIN_STATE_NONE; - } - - final int result = mDecoder.dequeueInputBuffer(timeoutUs); - if (result < 0) return DRAIN_STATE_NONE; - if (trackIndex < 0) { - mIsExtractorEOS = true; - mDecoder.queueInputBuffer(result, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM); - return DRAIN_STATE_NONE; - } - - final int sampleSize = mExtractor.readSampleData(mDecoderBuffers.getInputBuffer(result), 0); - final boolean isKeyFrame = (mExtractor.getSampleFlags() & MediaExtractor.SAMPLE_FLAG_SYNC) != 0; - mDecoder.queueInputBuffer(result, 0, sampleSize, mExtractor.getSampleTime(), isKeyFrame ? MediaCodec.BUFFER_FLAG_SYNC_FRAME : 0); - mExtractor.advance(); - return DRAIN_STATE_CONSUMED; - } - - @SuppressWarnings("SameParameterValue") - private int drainDecoder(long timeoutUs) { - if (mIsDecoderEOS) return DRAIN_STATE_NONE; - - int result = mDecoder.dequeueOutputBuffer(mBufferInfo, timeoutUs); - switch (result) { - case MediaCodec.INFO_TRY_AGAIN_LATER: - return DRAIN_STATE_NONE; - case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: - mAudioChannel.setActualDecodedFormat(mDecoder.getOutputFormat()); - case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: - return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; - } - - if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { - mIsDecoderEOS = true; - mAudioChannel.drainDecoderBufferAndQueue(AudioChannel.BUFFER_INDEX_END_OF_STREAM, 0); - } else if (mBufferInfo.size > 0) { - mAudioChannel.drainDecoderBufferAndQueue(result, mBufferInfo.presentationTimeUs); - } - - return DRAIN_STATE_CONSUMED; - } - - @SuppressWarnings("SameParameterValue") - private int drainEncoder(long timeoutUs) { - if (mIsEncoderEOS) return DRAIN_STATE_NONE; - - int result = mEncoder.dequeueOutputBuffer(mBufferInfo, timeoutUs); - switch (result) { - case MediaCodec.INFO_TRY_AGAIN_LATER: - return DRAIN_STATE_NONE; - case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: - if (mActualOutputFormat != null) { - throw new RuntimeException("Audio output format changed twice."); - } - mActualOutputFormat = mEncoder.getOutputFormat(); - mMuxer.setOutputFormat(SAMPLE_TYPE, mActualOutputFormat); - return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; - case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: - mEncoderBuffers = new MediaCodecBuffers(mEncoder); - return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; - } - - if (mActualOutputFormat == null) { - throw new RuntimeException("Could not determine actual output format."); - } - - if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { - mIsEncoderEOS = true; - mBufferInfo.set(0, 0, 0, mBufferInfo.flags); - } - if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) { - // SPS or PPS, which should be passed by MediaFormat. - mEncoder.releaseOutputBuffer(result, false); - return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; - } - mMuxer.write(SAMPLE_TYPE, mEncoderBuffers.getOutputBuffer(result), mBufferInfo); - mWrittenPresentationTimeUs = mBufferInfo.presentationTimeUs; - mEncoder.releaseOutputBuffer(result, false); - return DRAIN_STATE_CONSUMED; + protected boolean onFeedEncoder(@NonNull MediaCodec encoder, long timeoutUs) { + return mAudioChannel.feedEncoder(timeoutUs); } @Override - public long getLastWrittenPresentationTime() { - return mWrittenPresentationTimeUs; + protected void onDecoderOutputFormatChanged(@NonNull MediaFormat format) { + super.onDecoderOutputFormatChanged(format); + mAudioChannel.setActualDecodedFormat(format); } @Override - public boolean isFinished() { - return mIsEncoderEOS; - } - - @Override - public void release() { - if (mDecoder != null) { - if (mDecoderStarted) mDecoder.stop(); - mDecoder.release(); - mDecoder = null; - } - if (mEncoder != null) { - if (mEncoderStarted) mEncoder.stop(); - mEncoder.release(); - mEncoder = null; + protected void onDrainDecoder(@NonNull MediaCodec decoder, int bufferIndex, long presentationTimeUs, boolean endOfStream) { + if (endOfStream) { + mAudioChannel.drainDecoderBufferAndQueue(AudioChannel.BUFFER_INDEX_END_OF_STREAM, 0); + } else { + mAudioChannel.drainDecoderBufferAndQueue(bufferIndex, presentationTimeUs); } } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java new file mode 100644 index 00000000..d84952a6 --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java @@ -0,0 +1,330 @@ +package com.otaliastudios.transcoder.transcode; + +import android.media.MediaCodec; +import android.media.MediaExtractor; +import android.media.MediaFormat; + +import androidx.annotation.CallSuper; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; + +import com.otaliastudios.transcoder.engine.TrackType; +import com.otaliastudios.transcoder.engine.TranscoderMuxer; +import com.otaliastudios.transcoder.internal.MediaCodecBuffers; + +import java.io.IOException; + +/** + * A base implementation of {@link TrackTranscoder} that reads + * from {@link MediaExtractor} and does feeding and draining job. + */ +public abstract class BaseTrackTranscoder implements TrackTranscoder { + + private static final int DRAIN_STATE_NONE = 0; + private static final int DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY = 1; + private static final int DRAIN_STATE_CONSUMED = 2; + + private final MediaExtractor mExtractor; + private final int mTrackIndex; + private final TranscoderMuxer mMuxer; + private final TrackType mTrackType; + + private long mWrittenPresentationTimeUs; + + private final MediaCodec.BufferInfo mBufferInfo = new MediaCodec.BufferInfo(); + private MediaCodec mDecoder; + private MediaCodec mEncoder; + private MediaCodecBuffers mDecoderBuffers; + private MediaCodecBuffers mEncoderBuffers; + private boolean mDecoderStarted; + private boolean mEncoderStarted; + private MediaFormat mActualOutputFormat; + + private boolean mIsDecoderEOS; + private boolean mIsEncoderEOS; + private boolean mIsExtractorEOS; + + @SuppressWarnings("WeakerAccess") + protected BaseTrackTranscoder(@NonNull MediaExtractor extractor, + int trackIndex, + @NonNull TranscoderMuxer muxer, + @NonNull TrackType trackType) { + mExtractor = extractor; + mTrackIndex = trackIndex; + mMuxer = muxer; + mTrackType = trackType; + } + + /** + * Returns the encoder, if we have one at this point. + * @return encoder or null + */ + @Nullable + @SuppressWarnings("WeakerAccess") + protected MediaCodec getEncoder() { + return mEncoder; + } + + /** + * Returns the decoder, if we have one at this point. + * @return decoder or null + */ + @Nullable + @SuppressWarnings("unused") + protected MediaCodec getDecoder() { + return mDecoder; + } + + @Override + public final void setUp(@NonNull MediaFormat desiredOutputFormat) { + mExtractor.selectTrack(mTrackIndex); + try { + mEncoder = MediaCodec.createEncoderByType(desiredOutputFormat.getString(MediaFormat.KEY_MIME)); + } catch (IOException e) { + throw new IllegalStateException(e); + } + onConfigureEncoder(desiredOutputFormat, mEncoder); + onStartEncoder(desiredOutputFormat, mEncoder); + + final MediaFormat inputFormat = mExtractor.getTrackFormat(mTrackIndex); + try { + mDecoder = MediaCodec.createDecoderByType(inputFormat.getString(MediaFormat.KEY_MIME)); + } catch (IOException e) { + throw new IllegalStateException(e); + } + onConfigureDecoder(inputFormat, mDecoder); + onStartDecoder(inputFormat, mDecoder); + onCodecsStarted(inputFormat, desiredOutputFormat, mDecoder, mEncoder); + } + + /** + * Wraps the configure operation on the encoder. + * @param format output format + * @param encoder encoder + */ + @SuppressWarnings("WeakerAccess") + protected void onConfigureEncoder(@NonNull MediaFormat format, @NonNull MediaCodec encoder) { + encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); + } + + /** + * Wraps the start operation on the encoder. + * @param format output format + * @param encoder encoder + */ + @CallSuper + protected void onStartEncoder(@NonNull MediaFormat format, @NonNull MediaCodec encoder) { + encoder.start(); + mEncoderStarted = true; + mEncoderBuffers = new MediaCodecBuffers(encoder); + } + + /** + * Wraps the configure operation on the decoder. + * @param format input format + * @param decoder decoder + */ + protected void onConfigureDecoder(@NonNull MediaFormat format, @NonNull MediaCodec decoder) { + decoder.configure(format, null, null, 0); + } + + /** + * Wraps the start operation on the decoder. + * @param format input format + * @param decoder decoder + */ + @SuppressWarnings({"WeakerAccess", "unused"}) + @CallSuper + protected void onStartDecoder(@NonNull MediaFormat format, @NonNull MediaCodec decoder) { + decoder.start(); + mDecoderStarted = true; + mDecoderBuffers = new MediaCodecBuffers(decoder); + } + + /** + * Called when both codecs have been started with the given formats. + * @param inputFormat input format + * @param outputFormat output format + * @param decoder decoder + * @param encoder encoder + */ + protected void onCodecsStarted(@NonNull MediaFormat inputFormat, @NonNull MediaFormat outputFormat, + @NonNull MediaCodec decoder, @NonNull MediaCodec encoder) { + } + + @Override + public final long getLastWrittenPresentationTime() { + return mWrittenPresentationTimeUs; + } + + @Override + public final boolean isFinished() { + return mIsEncoderEOS; + } + + @Override + public void release() { + if (mDecoder != null) { + if (mDecoderStarted) { + mDecoder.stop(); + mDecoderStarted = false; + } + mDecoder.release(); + mDecoder = null; + } + if (mEncoder != null) { + if (mEncoderStarted) { + mEncoder.stop(); + mEncoderStarted = false; + } + mEncoder.release(); + mEncoder = null; + } + } + + @Override + public final boolean stepPipeline() { + boolean busy = false; + int status; + while (drainEncoder(0) != DRAIN_STATE_NONE) busy = true; + do { + status = drainDecoder(0); + if (status != DRAIN_STATE_NONE) busy = true; + // NOTE: not repeating to keep from deadlock when encoder is full. + } while (status == DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY); + + while (feedEncoder(0)) busy = true; + while (feedDecoder(0) != DRAIN_STATE_NONE) busy = true; + return busy; + } + + /** + * Called when the decoder has defined its actual output format. + * @param format format + */ + @CallSuper + protected void onDecoderOutputFormatChanged(@NonNull MediaFormat format) {} + + /** + * Called when the encoder has defined its actual output format. + * @param format format + */ + @CallSuper + @SuppressWarnings("WeakerAccess") + protected void onEncoderOutputFormatChanged(@NonNull MediaFormat format) { + if (mActualOutputFormat != null) { + throw new RuntimeException("Audio output format changed twice."); + } + mActualOutputFormat = format; + mMuxer.setOutputFormat(mTrackType, mActualOutputFormat); + } + + @SuppressWarnings("SameParameterValue") + private int feedDecoder(long timeoutUs) { + if (mIsExtractorEOS) return DRAIN_STATE_NONE; + int trackIndex = mExtractor.getSampleTrackIndex(); + if (trackIndex >= 0 && trackIndex != mTrackIndex) { + return DRAIN_STATE_NONE; + } + + final int result = mDecoder.dequeueInputBuffer(timeoutUs); + if (result < 0) return DRAIN_STATE_NONE; + if (trackIndex < 0) { + mIsExtractorEOS = true; + mDecoder.queueInputBuffer(result, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM); + return DRAIN_STATE_NONE; + } + + final int sampleSize = mExtractor.readSampleData(mDecoderBuffers.getInputBuffer(result), 0); + final boolean isKeyFrame = (mExtractor.getSampleFlags() & MediaExtractor.SAMPLE_FLAG_SYNC) != 0; + mDecoder.queueInputBuffer(result, 0, sampleSize, mExtractor.getSampleTime(), isKeyFrame ? MediaCodec.BUFFER_FLAG_SYNC_FRAME : 0); + mExtractor.advance(); + return DRAIN_STATE_CONSUMED; + } + + @SuppressWarnings("SameParameterValue") + private boolean feedEncoder(long timeoutUs) { + return onFeedEncoder(mEncoder, timeoutUs); + } + + @SuppressWarnings("SameParameterValue") + private int drainDecoder(long timeoutUs) { + if (mIsDecoderEOS) return DRAIN_STATE_NONE; + int result = mDecoder.dequeueOutputBuffer(mBufferInfo, timeoutUs); + switch (result) { + case MediaCodec.INFO_TRY_AGAIN_LATER: + return DRAIN_STATE_NONE; + case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: + onDecoderOutputFormatChanged(mDecoder.getOutputFormat()); + return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; + case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: + return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; + } + + boolean isEos = (mBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0; + boolean hasSize = mBufferInfo.size > 0; + if (isEos) mIsDecoderEOS = true; + if (isEos || hasSize) { + onDrainDecoder(mDecoder, result, mBufferInfo.presentationTimeUs, isEos); + } + return DRAIN_STATE_CONSUMED; + } + + @SuppressWarnings("SameParameterValue") + private int drainEncoder(long timeoutUs) { + if (mIsEncoderEOS) return DRAIN_STATE_NONE; + + int result = mEncoder.dequeueOutputBuffer(mBufferInfo, timeoutUs); + switch (result) { + case MediaCodec.INFO_TRY_AGAIN_LATER: + return DRAIN_STATE_NONE; + case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: + onEncoderOutputFormatChanged(mEncoder.getOutputFormat()); + return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; + case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: + mEncoderBuffers.onOutputBuffersChanged(); + return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; + } + + if (mActualOutputFormat == null) { + throw new RuntimeException("Could not determine actual output format."); + } + + if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + mIsEncoderEOS = true; + mBufferInfo.set(0, 0, 0, mBufferInfo.flags); + } + if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) { + // SPS or PPS, which should be passed by MediaFormat. + mEncoder.releaseOutputBuffer(result, false); + return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; + } + mMuxer.write(mTrackType, mEncoderBuffers.getOutputBuffer(result), mBufferInfo); + mWrittenPresentationTimeUs = mBufferInfo.presentationTimeUs; + mEncoder.releaseOutputBuffer(result, false); + return DRAIN_STATE_CONSUMED; + } + + /** + * Called to drain the decoder. Implementors are required to call {@link MediaCodec#releaseOutputBuffer(int, boolean)} + * with the given bufferIndex at some point. + * + * @param decoder the decoder + * @param bufferIndex the buffer index to be released + * @param presentationTimeUs frame timestamp + * @param endOfStream whether we are in end of stream + */ + protected abstract void onDrainDecoder(@NonNull MediaCodec decoder, + int bufferIndex, + long presentationTimeUs, + boolean endOfStream); + + + /** + * Called to feed the encoder with processed data. + * @param encoder the encoder + * @param timeoutUs a timeout for this op + * @return true if we want to keep working + */ + protected abstract boolean onFeedEncoder(@NonNull MediaCodec encoder, long timeoutUs); +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java index a472c721..f1c05e62 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java @@ -23,43 +23,20 @@ import com.otaliastudios.transcoder.engine.TrackType; import com.otaliastudios.transcoder.engine.TranscoderMuxer; -import com.otaliastudios.transcoder.internal.MediaCodecBuffers; import com.otaliastudios.transcoder.transcode.internal.VideoDecoderOutput; import com.otaliastudios.transcoder.transcode.internal.VideoEncoderInput; import com.otaliastudios.transcoder.internal.Logger; import com.otaliastudios.transcoder.internal.MediaFormatConstants; -import java.io.IOException; - // Refer: https://android.googlesource.com/platform/cts/+/lollipop-release/tests/tests/media/src/android/media/cts/ExtractDecodeEditEncodeMuxTest.java -public class VideoTrackTranscoder implements TrackTranscoder { +public class VideoTrackTranscoder extends BaseTrackTranscoder { private static final String TAG = VideoTrackTranscoder.class.getSimpleName(); private static final Logger LOG = new Logger(TAG); - private static final int DRAIN_STATE_NONE = 0; - private static final int DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY = 1; - private static final int DRAIN_STATE_CONSUMED = 2; - - private final MediaExtractor mExtractor; - private final int mTrackIndex; - private final TranscoderMuxer mMuxer; - private final MediaCodec.BufferInfo mBufferInfo = new MediaCodec.BufferInfo(); - private MediaCodec mDecoder; - private MediaCodec mEncoder; - private MediaCodecBuffers mDecoderBuffers; - private MediaCodecBuffers mEncoderBuffers; - - private MediaFormat mActualOutputFormat; - private boolean mIsExtractorEOS; - private boolean mIsDecoderEOS; - private boolean mIsEncoderEOS; - private boolean mDecoderStarted; - private boolean mEncoderStarted; - private long mWrittenPresentationTimeUs; - private VideoDecoderOutput mDecoderOutputSurface; private VideoEncoderInput mEncoderInputSurface; + private MediaCodec mEncoder; // Keep this since we want to signal EOS on it. private double mInFrameRateReciprocal; private double mOutFrameRateReciprocal; @@ -69,48 +46,37 @@ public VideoTrackTranscoder( @NonNull MediaExtractor extractor, int trackIndex, @NonNull TranscoderMuxer muxer) { - mExtractor = extractor; - mTrackIndex = trackIndex; - mMuxer = muxer; + super(extractor, trackIndex, muxer, TrackType.VIDEO); } @Override - public void setUp(@NonNull MediaFormat desiredOutputFormat) { - mExtractor.selectTrack(mTrackIndex); - - MediaFormat trackFormat = mExtractor.getTrackFormat(mTrackIndex); - int inFrameRate = trackFormat.getInteger(MediaFormat.KEY_FRAME_RATE); - int outFrameRate = desiredOutputFormat.getInteger(MediaFormat.KEY_FRAME_RATE); - mInFrameRateReciprocal = 1.0d / inFrameRate; - mOutFrameRateReciprocal = 1.0d / outFrameRate; - LOG.v("mInFrameRateReciprocal: " + mInFrameRateReciprocal + " mOutFrameRateReciprocal: " + mOutFrameRateReciprocal); - - // Configure encoder. - try { - mEncoder = MediaCodec.createEncoderByType(desiredOutputFormat.getString(MediaFormat.KEY_MIME)); - } catch (IOException e) { - throw new IllegalStateException(e); - } - mEncoder.configure(desiredOutputFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); - mEncoderInputSurface = new VideoEncoderInput(mEncoder.createInputSurface()); - mEncoder.start(); - mEncoderStarted = true; - mEncoderBuffers = new MediaCodecBuffers(mEncoder); + protected void onStartEncoder(@NonNull MediaFormat format, @NonNull MediaCodec encoder) { + mEncoderInputSurface = new VideoEncoderInput(encoder.createInputSurface()); + super.onStartEncoder(format, encoder); + } - // Configure decoder. - MediaFormat inputFormat = mExtractor.getTrackFormat(mTrackIndex); - if (inputFormat.containsKey(MediaFormatConstants.KEY_ROTATION_DEGREES)) { - // Decoded video is rotated automatically in Android 5.0 lollipop. - // Turn off here because we don't want to encode rotated one. + @Override + protected void onConfigureDecoder(@NonNull MediaFormat format, @NonNull MediaCodec decoder) { + if (format.containsKey(MediaFormatConstants.KEY_ROTATION_DEGREES)) { + // Decoded video is rotated automatically in Android 5.0 lollipop. Turn off here because we don't want to encode rotated one. // refer: https://android.googlesource.com/platform/frameworks/av/+blame/lollipop-release/media/libstagefright/Utils.cpp - inputFormat.setInteger(MediaFormatConstants.KEY_ROTATION_DEGREES, 0); + format.setInteger(MediaFormatConstants.KEY_ROTATION_DEGREES, 0); } + mDecoderOutputSurface = new VideoDecoderOutput(); + decoder.configure(format, mDecoderOutputSurface.getSurface(), null, 0); + } + + @Override + protected void onCodecsStarted(@NonNull MediaFormat inputFormat, @NonNull MediaFormat outputFormat, @NonNull MediaCodec decoder, @NonNull MediaCodec encoder) { + super.onCodecsStarted(inputFormat, outputFormat, decoder, encoder); + mEncoder = encoder; + // Cropping support. float inputWidth = inputFormat.getInteger(MediaFormat.KEY_WIDTH); float inputHeight = inputFormat.getInteger(MediaFormat.KEY_HEIGHT); float inputRatio = inputWidth / inputHeight; - float outputWidth = desiredOutputFormat.getInteger(MediaFormat.KEY_WIDTH); - float outputHeight = desiredOutputFormat.getInteger(MediaFormat.KEY_HEIGHT); + float outputWidth = outputFormat.getInteger(MediaFormat.KEY_WIDTH); + float outputHeight = outputFormat.getInteger(MediaFormat.KEY_HEIGHT); float outputRatio = outputWidth / outputHeight; float scaleX = 1, scaleY = 1; if (inputRatio > outputRatio) { // Input wider. We have a scaleX. @@ -120,44 +86,13 @@ public void setUp(@NonNull MediaFormat desiredOutputFormat) { } // I don't think we should consider rotation and flip these - we operate on non-rotated // surfaces and pass the input rotation metadata to the output muxer, see TranscoderEngine.setupMetadata. - mDecoderOutputSurface = new VideoDecoderOutput(scaleX, scaleY); - try { - mDecoder = MediaCodec.createDecoderByType(inputFormat.getString(MediaFormat.KEY_MIME)); - } catch (IOException e) { - throw new IllegalStateException(e); - } - mDecoder.configure(inputFormat, mDecoderOutputSurface.getSurface(), null, 0); - mDecoder.start(); - mDecoderStarted = true; - mDecoderBuffers = new MediaCodecBuffers(mDecoder); - } - - @Override - public boolean stepPipeline() { - boolean busy = false; - int status; - while (drainEncoder(0) != DRAIN_STATE_NONE) busy = true; - do { - status = drainDecoder(0); - if (status != DRAIN_STATE_NONE) busy = true; - // NOTE: not repeating to keep from deadlock when encoder is full. - } while (status == DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY); - while (drainExtractor(0) != DRAIN_STATE_NONE) busy = true; + mDecoderOutputSurface.setScale(scaleX, scaleY); - return busy; + // Frame dropping support. + int frameRate = outputFormat.getInteger(MediaFormat.KEY_FRAME_RATE); + mTargetAvgStep = (1F / frameRate) * 1000 * 1000; } - @Override - public long getLastWrittenPresentationTime() { - return mWrittenPresentationTimeUs; - } - - @Override - public boolean isFinished() { - return mIsEncoderEOS; - } - - // TODO: CloseGuard @Override public void release() { if (mDecoderOutputSurface != null) { @@ -168,74 +103,56 @@ public void release() { mEncoderInputSurface.release(); mEncoderInputSurface = null; } - if (mDecoder != null) { - if (mDecoderStarted) mDecoder.stop(); - mDecoder.release(); - mDecoder = null; - } - if (mEncoder != null) { - if (mEncoderStarted) mEncoder.stop(); - mEncoder.release(); - mEncoder = null; - } + super.release(); + mEncoder = null; } - @SuppressWarnings("SameParameterValue") - private int drainExtractor(long timeoutUs) { - if (mIsExtractorEOS) return DRAIN_STATE_NONE; - int trackIndex = mExtractor.getSampleTrackIndex(); - if (trackIndex >= 0 && trackIndex != mTrackIndex) { - return DRAIN_STATE_NONE; - } - int result = mDecoder.dequeueInputBuffer(timeoutUs); - if (result < 0) return DRAIN_STATE_NONE; - if (trackIndex < 0) { - mIsExtractorEOS = true; - mDecoder.queueInputBuffer(result, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM); - return DRAIN_STATE_NONE; - } - int sampleSize = mExtractor.readSampleData(mDecoderBuffers.getInputBuffer(result), 0); - boolean isKeyFrame = (mExtractor.getSampleFlags() & MediaExtractor.SAMPLE_FLAG_SYNC) != 0; - mDecoder.queueInputBuffer(result, 0, sampleSize, mExtractor.getSampleTime(), isKeyFrame ? MediaCodec.BUFFER_FLAG_SYNC_FRAME : 0); - mExtractor.advance(); - return DRAIN_STATE_CONSUMED; + @Override + protected boolean onFeedEncoder(@NonNull MediaCodec encoder, long timeoutUs) { + // We do not feed the encoder, see below. + return false; } - @SuppressWarnings("SameParameterValue") - private int drainDecoder(long timeoutUs) { - if (mIsDecoderEOS) return DRAIN_STATE_NONE; - int result = mDecoder.dequeueOutputBuffer(mBufferInfo, timeoutUs); - switch (result) { - case MediaCodec.INFO_TRY_AGAIN_LATER: - return DRAIN_STATE_NONE; - case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: - case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: - return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; - } - if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + @Override + protected void onDrainDecoder(@NonNull MediaCodec decoder, int bufferIndex, long presentationTimeUs, boolean endOfStream) { + if (endOfStream) { mEncoder.signalEndOfInputStream(); - mIsDecoderEOS = true; - mBufferInfo.size = 0; - } - boolean doRender = shouldRenderFrame(); - // NOTE: doRender will block if buffer (of encoder) is full. - // Refer: http://bigflake.com/mediacodec/CameraToMpegTest.java.txt - mDecoder.releaseOutputBuffer(result, doRender); - if (doRender) { + decoder.releaseOutputBuffer(bufferIndex, false); + } else if (shouldRenderFrame(presentationTimeUs)) { + decoder.releaseOutputBuffer(bufferIndex, true); mDecoderOutputSurface.drawFrame(); - mEncoderInputSurface.onFrame(mBufferInfo.presentationTimeUs); + mEncoderInputSurface.onFrame(presentationTimeUs); } - return DRAIN_STATE_CONSUMED; } - //Refer:https://stackoverflow.com/questions/4223766/dropping-video-frames - private boolean shouldRenderFrame() { - if (mBufferInfo.size <= 0) return false; - boolean firstFrame = Double.valueOf(0d).equals(mFrameRateReciprocalSum); - mFrameRateReciprocalSum += mInFrameRateReciprocal; - if (firstFrame) { - // render frame - LOG.v("render this frame -> mFrameRateReciprocalSum: " + mFrameRateReciprocalSum); + // TODO improve this. as it is now, rendering a frame after dropping many, + // will not decrease avgStep but rather increase it (for this single frame; then it starts decreasing). + // This has the effect that, when a frame is rendered, the following frame is always rendered, + // because the conditions are worse then before. After this second frame things go back to normal, + // but this is terrible logic. + private boolean shouldRenderFrame(long presentationTimeUs) { + if (mRenderedSteps > 0 && mAvgStep < mTargetAvgStep) { + // We are rendering too much. Drop this frame. + // Always render first 2 frames, we need them to compute the avg. + LOG.v("FRAME: Dropping. avg: " + mAvgStep + " target: " + mTargetAvgStep); + long newLastStep = presentationTimeUs - mLastRenderedUs; + float allSteps = (mAvgStep * mRenderedSteps) - mLastStep + newLastStep; + mAvgStep = allSteps / mRenderedSteps; // we didn't add a step, just increased the last + mLastStep = newLastStep; + return false; + } else { + // Render this frame, since our average step is too long or exact. + LOG.v("FRAME: RENDERING. avg: " + mAvgStep + " target: " + mTargetAvgStep + "New stepCount: " + (mRenderedSteps + 1)); + if (mRenderedSteps >= 0) { + // Update the average value, since now we have mLastRenderedUs. + long step = presentationTimeUs - mLastRenderedUs; + float allSteps = (mAvgStep * mRenderedSteps) + step; + mAvgStep = allSteps / (mRenderedSteps + 1); // we added a step, so +1 + mLastStep = step; + } + // Increment both + mRenderedSteps++; + mLastRenderedUs = presentationTimeUs; return true; } if (mFrameRateReciprocalSum > mOutFrameRateReciprocal) { @@ -248,40 +165,4 @@ private boolean shouldRenderFrame() { LOG.v("drop this frame -> mFrameRateReciprocalSum: " + mFrameRateReciprocalSum); return false; } - - @SuppressWarnings("SameParameterValue") - private int drainEncoder(long timeoutUs) { - if (mIsEncoderEOS) return DRAIN_STATE_NONE; - int result = mEncoder.dequeueOutputBuffer(mBufferInfo, timeoutUs); - switch (result) { - case MediaCodec.INFO_TRY_AGAIN_LATER: - return DRAIN_STATE_NONE; - case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: - if (mActualOutputFormat != null) - throw new RuntimeException("Video output format changed twice."); - mActualOutputFormat = mEncoder.getOutputFormat(); - mMuxer.setOutputFormat(TrackType.VIDEO, mActualOutputFormat); - return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; - case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: - mEncoderBuffers.onOutputBuffersChanged(); - return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; - } - if (mActualOutputFormat == null) { - throw new RuntimeException("Could not determine actual output format."); - } - - if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { - mIsEncoderEOS = true; - mBufferInfo.set(0, 0, 0, mBufferInfo.flags); - } - if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) { - // SPS or PPS, which should be passed by MediaFormat. - mEncoder.releaseOutputBuffer(result, false); - return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; - } - mMuxer.write(TrackType.VIDEO, mEncoderBuffers.getOutputBuffer(result), mBufferInfo); - mWrittenPresentationTimeUs = mBufferInfo.presentationTimeUs; - mEncoder.releaseOutputBuffer(result, false); - return DRAIN_STATE_CONSUMED; - } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/VideoDecoderOutput.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/VideoDecoderOutput.java index a8320fc1..da7952b1 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/VideoDecoderOutput.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/VideoDecoderOutput.java @@ -40,8 +40,8 @@ public class VideoDecoderOutput { private EglTextureProgram mProgram; private EglDrawable mDrawable; - private final float mScaleX; - private final float mScaleY; + private float mScaleX = 1F; + private float mScaleY = 1F; private int mTextureId; private float[] mTextureTransform = new float[16]; @@ -54,15 +54,12 @@ public class VideoDecoderOutput { * Creates an VideoDecoderOutput using the current EGL context (rather than establishing a * new one). Creates a Surface that can be passed to MediaCodec.configure(). */ - public VideoDecoderOutput(float scaleX, float scaleY) { + public VideoDecoderOutput() { mScene = new EglScene(); mProgram = new EglTextureProgram(); mDrawable = new EglRect(); mTextureId = mProgram.createTexture(); - mScaleX = scaleX; - mScaleY = scaleY; - // Even if we don't access the SurfaceTexture after the constructor returns, we // still need to keep a reference to it. The Surface doesn't retain a reference // at the Java level, so if we don't either then the object can get GCed, which @@ -84,6 +81,16 @@ public void onFrameAvailable(SurfaceTexture surfaceTexture) { mSurface = new Surface(mSurfaceTexture); } + /** + * Sets the frame scale along the two axes. + * @param scaleX x scale + * @param scaleY y scale + */ + public void setScale(float scaleX, float scaleY) { + mScaleX = scaleX; + mScaleY = scaleY; + } + /** * Returns a Surface to draw onto. * @return the output surface From 586ac9c5486b1a8ceaa30bfef595d874a073a919 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Tue, 23 Jul 2019 12:50:30 +0100 Subject: [PATCH 08/22] Refactor AudioChannel --- .../transcode/AudioTrackTranscoder.java | 17 ++- .../transcode/BaseTrackTranscoder.java | 35 +++--- .../transcode/VideoTrackTranscoder.java | 7 +- .../transcode/internal/AudioChannel.java | 102 +++++++++--------- 4 files changed, 76 insertions(+), 85 deletions(-) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java index 59fed50c..7314f1f4 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java @@ -8,8 +8,11 @@ import com.otaliastudios.transcoder.engine.TrackType; import com.otaliastudios.transcoder.engine.TranscoderMuxer; +import com.otaliastudios.transcoder.internal.MediaCodecBuffers; import com.otaliastudios.transcoder.transcode.internal.AudioChannel; +import java.nio.ByteBuffer; + public class AudioTrackTranscoder extends BaseTrackTranscoder { private AudioChannel mAudioChannel; @@ -27,22 +30,18 @@ protected void onCodecsStarted(@NonNull MediaFormat inputFormat, @NonNull MediaF } @Override - protected boolean onFeedEncoder(@NonNull MediaCodec encoder, long timeoutUs) { - return mAudioChannel.feedEncoder(timeoutUs); + protected boolean onFeedEncoder(@NonNull MediaCodec encoder, @NonNull MediaCodecBuffers encoderBuffers, long timeoutUs) { + return mAudioChannel.feedEncoder(encoderBuffers, timeoutUs); } @Override protected void onDecoderOutputFormatChanged(@NonNull MediaFormat format) { super.onDecoderOutputFormatChanged(format); - mAudioChannel.setActualDecodedFormat(format); + mAudioChannel.onDecoderOutputFormat(format); } @Override - protected void onDrainDecoder(@NonNull MediaCodec decoder, int bufferIndex, long presentationTimeUs, boolean endOfStream) { - if (endOfStream) { - mAudioChannel.drainDecoderBufferAndQueue(AudioChannel.BUFFER_INDEX_END_OF_STREAM, 0); - } else { - mAudioChannel.drainDecoderBufferAndQueue(bufferIndex, presentationTimeUs); - } + protected void onDrainDecoder(@NonNull MediaCodec decoder, int bufferIndex, @NonNull ByteBuffer bufferData, long presentationTimeUs, boolean endOfStream) { + mAudioChannel.drainDecoder(bufferIndex, bufferData, presentationTimeUs, endOfStream); } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java index d84952a6..adf08052 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java @@ -13,6 +13,7 @@ import com.otaliastudios.transcoder.internal.MediaCodecBuffers; import java.io.IOException; +import java.nio.ByteBuffer; /** * A base implementation of {@link TrackTranscoder} that reads @@ -55,26 +56,6 @@ protected BaseTrackTranscoder(@NonNull MediaExtractor extractor, mTrackType = trackType; } - /** - * Returns the encoder, if we have one at this point. - * @return encoder or null - */ - @Nullable - @SuppressWarnings("WeakerAccess") - protected MediaCodec getEncoder() { - return mEncoder; - } - - /** - * Returns the decoder, if we have one at this point. - * @return decoder or null - */ - @Nullable - @SuppressWarnings("unused") - protected MediaCodec getDecoder() { - return mDecoder; - } - @Override public final void setUp(@NonNull MediaFormat desiredOutputFormat) { mExtractor.selectTrack(mTrackIndex); @@ -244,7 +225,7 @@ private int feedDecoder(long timeoutUs) { @SuppressWarnings("SameParameterValue") private boolean feedEncoder(long timeoutUs) { - return onFeedEncoder(mEncoder, timeoutUs); + return onFeedEncoder(mEncoder, mEncoderBuffers, timeoutUs); } @SuppressWarnings("SameParameterValue") @@ -265,7 +246,11 @@ private int drainDecoder(long timeoutUs) { boolean hasSize = mBufferInfo.size > 0; if (isEos) mIsDecoderEOS = true; if (isEos || hasSize) { - onDrainDecoder(mDecoder, result, mBufferInfo.presentationTimeUs, isEos); + onDrainDecoder(mDecoder, + result, + mDecoderBuffers.getOutputBuffer(result), + mBufferInfo.presentationTimeUs, + isEos); } return DRAIN_STATE_CONSUMED; } @@ -311,11 +296,13 @@ private int drainEncoder(long timeoutUs) { * * @param decoder the decoder * @param bufferIndex the buffer index to be released + * @param bufferData the buffer data * @param presentationTimeUs frame timestamp * @param endOfStream whether we are in end of stream */ protected abstract void onDrainDecoder(@NonNull MediaCodec decoder, int bufferIndex, + @NonNull ByteBuffer bufferData, long presentationTimeUs, boolean endOfStream); @@ -326,5 +313,7 @@ protected abstract void onDrainDecoder(@NonNull MediaCodec decoder, * @param timeoutUs a timeout for this op * @return true if we want to keep working */ - protected abstract boolean onFeedEncoder(@NonNull MediaCodec encoder, long timeoutUs); + protected abstract boolean onFeedEncoder(@NonNull MediaCodec encoder, + @NonNull MediaCodecBuffers encoderBuffers, + long timeoutUs); } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java index f1c05e62..b809e101 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java @@ -23,11 +23,14 @@ import com.otaliastudios.transcoder.engine.TrackType; import com.otaliastudios.transcoder.engine.TranscoderMuxer; +import com.otaliastudios.transcoder.internal.MediaCodecBuffers; import com.otaliastudios.transcoder.transcode.internal.VideoDecoderOutput; import com.otaliastudios.transcoder.transcode.internal.VideoEncoderInput; import com.otaliastudios.transcoder.internal.Logger; import com.otaliastudios.transcoder.internal.MediaFormatConstants; +import java.nio.ByteBuffer; + // Refer: https://android.googlesource.com/platform/cts/+/lollipop-release/tests/tests/media/src/android/media/cts/ExtractDecodeEditEncodeMuxTest.java public class VideoTrackTranscoder extends BaseTrackTranscoder { @@ -108,13 +111,13 @@ public void release() { } @Override - protected boolean onFeedEncoder(@NonNull MediaCodec encoder, long timeoutUs) { + protected boolean onFeedEncoder(@NonNull MediaCodec encoder, @NonNull MediaCodecBuffers encoderBuffers, long timeoutUs) { // We do not feed the encoder, see below. return false; } @Override - protected void onDrainDecoder(@NonNull MediaCodec decoder, int bufferIndex, long presentationTimeUs, boolean endOfStream) { + protected void onDrainDecoder(@NonNull MediaCodec decoder, int bufferIndex, @NonNull ByteBuffer bufferData, long presentationTimeUs, boolean endOfStream) { if (endOfStream) { mEncoder.signalEndOfInputStream(); decoder.releaseOutputBuffer(bufferIndex, false); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioChannel.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioChannel.java index 19ce52e9..133d888b 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioChannel.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioChannel.java @@ -27,10 +27,9 @@ private static class AudioBuffer { int bufferIndex; long presentationTimeUs; ShortBuffer data; + boolean endOfStream; } - public static final int BUFFER_INDEX_END_OF_STREAM = -1; - private static final int BYTES_PER_SHORT = 2; private static final long MICROSECS_PER_SEC = 1000000; @@ -39,54 +38,43 @@ private static class AudioBuffer { private final MediaCodec mDecoder; private final MediaCodec mEncoder; - private final MediaFormat mEncodeFormat; private int mInputSampleRate; + private int mOutputSampleRate; private int mInputChannelCount; private int mOutputChannelCount; private AudioRemixer mRemixer; - private final MediaCodecBuffers mDecoderBuffers; - private final MediaCodecBuffers mEncoderBuffers; - private final AudioBuffer mOverflowBuffer = new AudioBuffer(); - private MediaFormat mActualDecodedFormat; - public AudioChannel(@NonNull final MediaCodec decoder, @NonNull final MediaCodec encoder, - @NonNull final MediaFormat encodeFormat) { + @NonNull final MediaFormat outputFormat) { mDecoder = decoder; mEncoder = encoder; - mEncodeFormat = encodeFormat; - mDecoderBuffers = new MediaCodecBuffers(mDecoder); - mEncoderBuffers = new MediaCodecBuffers(mEncoder); + mOutputSampleRate = outputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE); + mOutputChannelCount = outputFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT); + if (mOutputChannelCount != 1 && mOutputChannelCount != 2) { + throw new UnsupportedOperationException("Output channel count (" + mOutputChannelCount + ") not supported."); + } } - public void setActualDecodedFormat(@NonNull final MediaFormat decodedFormat) { - mActualDecodedFormat = decodedFormat; - - // TODO I think these exceptions are either useless or not in the right place. - // We have MediaFormatValidator doing this kind of stuff. - - mInputSampleRate = mActualDecodedFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE); - if (mInputSampleRate != mEncodeFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE)) { + /** + * Should be the first method to be called, when we get the {@link MediaCodec#INFO_OUTPUT_FORMAT_CHANGED} + * event from MediaCodec. + * @param decoderOutputFormat the output format + */ + public void onDecoderOutputFormat(@NonNull final MediaFormat decoderOutputFormat) { + mInputSampleRate = decoderOutputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE); + if (mInputSampleRate != mOutputSampleRate) { throw new UnsupportedOperationException("Audio sample rate conversion not supported yet."); } - - mInputChannelCount = mActualDecodedFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT); - mOutputChannelCount = mEncodeFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT); - + mInputChannelCount = decoderOutputFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT); if (mInputChannelCount != 1 && mInputChannelCount != 2) { throw new UnsupportedOperationException("Input channel count (" + mInputChannelCount + ") not supported."); } - - if (mOutputChannelCount != 1 && mOutputChannelCount != 2) { - throw new UnsupportedOperationException("Output channel count (" + mOutputChannelCount + ") not supported."); - } - if (mInputChannelCount > mOutputChannelCount) { mRemixer = AudioRemixer.DOWNMIX; } else if (mInputChannelCount < mOutputChannelCount) { @@ -94,41 +82,51 @@ public void setActualDecodedFormat(@NonNull final MediaFormat decodedFormat) { } else { mRemixer = AudioRemixer.PASSTHROUGH; } - mOverflowBuffer.presentationTimeUs = 0; } - public void drainDecoderBufferAndQueue(final int bufferIndex, final long presentationTimeUs) { - if (mActualDecodedFormat == null) { - throw new RuntimeException("Buffer received before format!"); - } - - final ByteBuffer data = bufferIndex == BUFFER_INDEX_END_OF_STREAM ? - null : mDecoderBuffers.getOutputBuffer(bufferIndex); - + /** + * Drains the decoder, which means scheduling data for processing. + * We fill a new {@link AudioBuffer} with data (not a copy, just a view of it) + * and add it to the filled buffers queue. + * + * @param bufferIndex the buffer index + * @param bufferData the buffer data + * @param presentationTimeUs the presentation time + * @param endOfStream true if end of stream + */ + public void drainDecoder(final int bufferIndex, + @NonNull ByteBuffer bufferData, + final long presentationTimeUs, + final boolean endOfStream) { + if (mRemixer == null) throw new RuntimeException("Buffer received before format!"); AudioBuffer buffer = mEmptyBuffers.poll(); if (buffer == null) { buffer = new AudioBuffer(); } - buffer.bufferIndex = bufferIndex; - buffer.presentationTimeUs = presentationTimeUs; - buffer.data = data == null ? null : data.asShortBuffer(); + buffer.presentationTimeUs = endOfStream ? 0 : presentationTimeUs; + buffer.data = endOfStream ? null : bufferData.asShortBuffer(); + buffer.endOfStream = endOfStream; if (mOverflowBuffer.data == null) { - // data should be null only on BUFFER_INDEX_END_OF_STREAM - //noinspection ConstantConditions - mOverflowBuffer.data = ByteBuffer - .allocateDirect(data.capacity()) + mOverflowBuffer.data = ByteBuffer.allocateDirect(bufferData.capacity()) .order(ByteOrder.nativeOrder()) .asShortBuffer(); mOverflowBuffer.data.clear().flip(); } - mFilledBuffers.add(buffer); } - public boolean feedEncoder(@SuppressWarnings("SameParameterValue") long timeoutUs) { + /** + * Feeds the encoder, which in our case means processing a filled buffer from {@link #mFilledBuffers}, + * then releasing it and adding it back to {@link #mEmptyBuffers}. + * + * @param encoderBuffers the encoder buffers + * @param timeoutUs a timeout for this operation + * @return true if we want to keep working + */ + public boolean feedEncoder(@NonNull MediaCodecBuffers encoderBuffers, long timeoutUs) { final boolean hasOverflow = mOverflowBuffer.data != null && mOverflowBuffer.data.hasRemaining(); if (mFilledBuffers.isEmpty() && !hasOverflow) { // No audio data - Bail out @@ -142,7 +140,7 @@ public boolean feedEncoder(@SuppressWarnings("SameParameterValue") long timeoutU } // Drain overflow first - final ShortBuffer outBuffer = mEncoderBuffers.getInputBuffer(encoderInBuffIndex).asShortBuffer(); + final ShortBuffer outBuffer = encoderBuffers.getInputBuffer(encoderInBuffIndex).asShortBuffer(); if (hasOverflow) { final long presentationTimeUs = drainOverflow(outBuffer); mEncoder.queueInputBuffer(encoderInBuffIndex, @@ -155,15 +153,17 @@ public boolean feedEncoder(@SuppressWarnings("SameParameterValue") long timeoutU // At this point inBuffer is not null, because we checked mFilledBuffers.isEmpty() // and we don't have overflow (if we had, we got out). //noinspection ConstantConditions - if (inBuffer.bufferIndex == BUFFER_INDEX_END_OF_STREAM) { + if (inBuffer.endOfStream) { mEncoder.queueInputBuffer(encoderInBuffIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM); return false; } final long presentationTimeUs = remixAndMaybeFillOverflow(inBuffer, outBuffer); mEncoder.queueInputBuffer(encoderInBuffIndex, - 0, outBuffer.position() * BYTES_PER_SHORT, - presentationTimeUs, 0); + 0, + outBuffer.position() * BYTES_PER_SHORT, + presentationTimeUs, + 0); mDecoder.releaseOutputBuffer(inBuffer.bufferIndex, false); mEmptyBuffers.add(inBuffer); return true; From 562e9f931ea80204f3846192c562564dfa5d1faf Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Tue, 23 Jul 2019 13:24:01 +0100 Subject: [PATCH 09/22] Refactor Audio internals --- .../transcode/AudioTrackTranscoder.java | 13 +- .../transcode/BaseTrackTranscoder.java | 8 +- .../transcode/internal/AudioChannel.java | 222 +++++++++--------- 3 files changed, 130 insertions(+), 113 deletions(-) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java index 7314f1f4..2449ad8d 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java @@ -16,6 +16,8 @@ public class AudioTrackTranscoder extends BaseTrackTranscoder { private AudioChannel mAudioChannel; + private MediaCodec mEncoder; // to create the channel + private MediaFormat mEncoderOutputFormat; // to create the channel public AudioTrackTranscoder(@NonNull MediaExtractor extractor, int trackIndex, @@ -26,7 +28,8 @@ public AudioTrackTranscoder(@NonNull MediaExtractor extractor, @Override protected void onCodecsStarted(@NonNull MediaFormat inputFormat, @NonNull MediaFormat outputFormat, @NonNull MediaCodec decoder, @NonNull MediaCodec encoder) { super.onCodecsStarted(inputFormat, outputFormat, decoder, encoder); - mAudioChannel = new AudioChannel(decoder, encoder, outputFormat); + mEncoder = encoder; + mEncoderOutputFormat = outputFormat; } @Override @@ -35,9 +38,11 @@ protected boolean onFeedEncoder(@NonNull MediaCodec encoder, @NonNull MediaCodec } @Override - protected void onDecoderOutputFormatChanged(@NonNull MediaFormat format) { - super.onDecoderOutputFormatChanged(format); - mAudioChannel.onDecoderOutputFormat(format); + protected void onDecoderOutputFormatChanged(@NonNull MediaCodec decoder, @NonNull MediaFormat format) { + super.onDecoderOutputFormatChanged(decoder, format); + mAudioChannel = new AudioChannel(decoder, format, mEncoder, mEncoderOutputFormat); + mEncoder = null; + mEncoderOutputFormat = null; } @Override diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java index adf08052..6cdbc966 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java @@ -184,7 +184,7 @@ public final boolean stepPipeline() { * @param format format */ @CallSuper - protected void onDecoderOutputFormatChanged(@NonNull MediaFormat format) {} + protected void onDecoderOutputFormatChanged(@NonNull MediaCodec decoder, @NonNull MediaFormat format) {} /** * Called when the encoder has defined its actual output format. @@ -192,7 +192,7 @@ protected void onDecoderOutputFormatChanged(@NonNull MediaFormat format) {} */ @CallSuper @SuppressWarnings("WeakerAccess") - protected void onEncoderOutputFormatChanged(@NonNull MediaFormat format) { + protected void onEncoderOutputFormatChanged(@NonNull MediaCodec encoder, @NonNull MediaFormat format) { if (mActualOutputFormat != null) { throw new RuntimeException("Audio output format changed twice."); } @@ -236,7 +236,7 @@ private int drainDecoder(long timeoutUs) { case MediaCodec.INFO_TRY_AGAIN_LATER: return DRAIN_STATE_NONE; case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: - onDecoderOutputFormatChanged(mDecoder.getOutputFormat()); + onDecoderOutputFormatChanged(mDecoder, mDecoder.getOutputFormat()); return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; @@ -264,7 +264,7 @@ private int drainEncoder(long timeoutUs) { case MediaCodec.INFO_TRY_AGAIN_LATER: return DRAIN_STATE_NONE; case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: - onEncoderOutputFormatChanged(mEncoder.getOutputFormat()); + onEncoderOutputFormatChanged(mEncoder, mEncoder.getOutputFormat()); return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: mEncoderBuffers.onOutputBuffersChanged(); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioChannel.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioChannel.java index 133d888b..6c1bde6b 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioChannel.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioChannel.java @@ -30,51 +30,53 @@ private static class AudioBuffer { boolean endOfStream; } + private static long sampleCountToDurationUs( + final int sampleCount, + final int sampleRate, + final int channelCount) { + return (sampleCount / (sampleRate * MICROSECONDS_PER_SECOND)) / channelCount; + } + private static final int BYTES_PER_SHORT = 2; - private static final long MICROSECS_PER_SEC = 1000000; + private static final long MICROSECONDS_PER_SECOND = 1000000; private final Queue mEmptyBuffers = new ArrayDeque<>(); private final Queue mFilledBuffers = new ArrayDeque<>(); - private final MediaCodec mDecoder; private final MediaCodec mEncoder; - - private int mInputSampleRate; - private int mOutputSampleRate; - private int mInputChannelCount; - private int mOutputChannelCount; - - private AudioRemixer mRemixer; - private final AudioBuffer mOverflowBuffer = new AudioBuffer(); + private final int mSampleRate; + private final int mInputChannelCount; + private final int mOutputChannelCount; + private final AudioRemixer mRemixer; + public AudioChannel(@NonNull final MediaCodec decoder, - @NonNull final MediaCodec encoder, - @NonNull final MediaFormat outputFormat) { + @NonNull final MediaFormat decoderOutputFormat, + @NonNull final MediaCodec encoder, + @NonNull final MediaFormat encoderOutputFormat) { mDecoder = decoder; mEncoder = encoder; - mOutputSampleRate = outputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE); - mOutputChannelCount = outputFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT); - if (mOutputChannelCount != 1 && mOutputChannelCount != 2) { - throw new UnsupportedOperationException("Output channel count (" + mOutputChannelCount + ") not supported."); - } - } - - /** - * Should be the first method to be called, when we get the {@link MediaCodec#INFO_OUTPUT_FORMAT_CHANGED} - * event from MediaCodec. - * @param decoderOutputFormat the output format - */ - public void onDecoderOutputFormat(@NonNull final MediaFormat decoderOutputFormat) { - mInputSampleRate = decoderOutputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE); - if (mInputSampleRate != mOutputSampleRate) { + // Get and check sample rate. + int outputSampleRate = encoderOutputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE); + int inputSampleRate = decoderOutputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE); + if (inputSampleRate != outputSampleRate) { throw new UnsupportedOperationException("Audio sample rate conversion not supported yet."); } + mSampleRate = inputSampleRate; + + // Check channel count. + mOutputChannelCount = encoderOutputFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT); mInputChannelCount = decoderOutputFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT); + if (mOutputChannelCount != 1 && mOutputChannelCount != 2) { + throw new UnsupportedOperationException("Output channel count (" + mOutputChannelCount + ") not supported."); + } if (mInputChannelCount != 1 && mInputChannelCount != 2) { throw new UnsupportedOperationException("Input channel count (" + mInputChannelCount + ") not supported."); } + + // Create remixer. if (mInputChannelCount > mOutputChannelCount) { mRemixer = AudioRemixer.DOWNMIX; } else if (mInputChannelCount < mOutputChannelCount) { @@ -83,6 +85,7 @@ public void onDecoderOutputFormat(@NonNull final MediaFormat decoderOutputFormat mRemixer = AudioRemixer.PASSTHROUGH; } mOverflowBuffer.presentationTimeUs = 0; + } /** @@ -127,115 +130,124 @@ public void drainDecoder(final int bufferIndex, * @return true if we want to keep working */ public boolean feedEncoder(@NonNull MediaCodecBuffers encoderBuffers, long timeoutUs) { - final boolean hasOverflow = mOverflowBuffer.data != null && mOverflowBuffer.data.hasRemaining(); - if (mFilledBuffers.isEmpty() && !hasOverflow) { - // No audio data - Bail out - return false; - } + final boolean hasOverflow = hasOverflow(); + final boolean hasBuffers = hasBuffers(); + if (!hasBuffers && !hasOverflow) return false; - final int encoderInBuffIndex = mEncoder.dequeueInputBuffer(timeoutUs); - if (encoderInBuffIndex < 0) { - // Encoder is full - Bail out - return false; - } + // Prepare to encode - see if encoder has buffers. + final int encoderBufferIndex = mEncoder.dequeueInputBuffer(timeoutUs); + if (encoderBufferIndex < 0) return false; + ShortBuffer encoderBuffer = encoderBuffers.getInputBuffer(encoderBufferIndex).asShortBuffer(); - // Drain overflow first - final ShortBuffer outBuffer = encoderBuffers.getInputBuffer(encoderInBuffIndex).asShortBuffer(); + // If we have overflow data, process that first. if (hasOverflow) { - final long presentationTimeUs = drainOverflow(outBuffer); - mEncoder.queueInputBuffer(encoderInBuffIndex, - 0, outBuffer.position() * BYTES_PER_SHORT, - presentationTimeUs, 0); + long presentationTimeUs = drainOverflow(encoderBuffer); + mEncoder.queueInputBuffer(encoderBufferIndex, + 0, + encoderBuffer.position() * BYTES_PER_SHORT, + presentationTimeUs, + 0); return true; } - final AudioBuffer inBuffer = mFilledBuffers.poll(); - // At this point inBuffer is not null, because we checked mFilledBuffers.isEmpty() + // At this point buffer is not null, because we checked hasBuffers() // and we don't have overflow (if we had, we got out). + final AudioBuffer decoderBuffer = mFilledBuffers.poll(); + + // When endOfStream, just signal EOS and return false. //noinspection ConstantConditions - if (inBuffer.endOfStream) { - mEncoder.queueInputBuffer(encoderInBuffIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM); + if (decoderBuffer.endOfStream) { + mEncoder.queueInputBuffer(encoderBufferIndex, + 0, + 0, + 0, + MediaCodec.BUFFER_FLAG_END_OF_STREAM); return false; } - final long presentationTimeUs = remixAndMaybeFillOverflow(inBuffer, outBuffer); - mEncoder.queueInputBuffer(encoderInBuffIndex, + // If not, process data and return true. + process(decoderBuffer.data, encoderBuffer, decoderBuffer.presentationTimeUs); + mEncoder.queueInputBuffer(encoderBufferIndex, 0, - outBuffer.position() * BYTES_PER_SHORT, - presentationTimeUs, + encoderBuffer.position() * BYTES_PER_SHORT, + decoderBuffer.presentationTimeUs, 0); - mDecoder.releaseOutputBuffer(inBuffer.bufferIndex, false); - mEmptyBuffers.add(inBuffer); + mDecoder.releaseOutputBuffer(decoderBuffer.bufferIndex, false); + mEmptyBuffers.add(decoderBuffer); return true; } - private static long sampleCountToDurationUs( - final int sampleCount, - final int sampleRate, - final int channelCount) { - return (sampleCount / (sampleRate * MICROSECS_PER_SEC)) / channelCount; + /** + * Returns true if we have overflow data to be drained and set to the encoder. + * The overflow data is already processed. + * + * @return true if data + */ + private boolean hasOverflow() { + return mOverflowBuffer.data != null && mOverflowBuffer.data.hasRemaining(); } - private long drainOverflow(@NonNull final ShortBuffer outBuff) { - final ShortBuffer overflowBuff = mOverflowBuffer.data; - final int overflowLimit = overflowBuff.limit(); - final int overflowSize = overflowBuff.remaining(); + /** + * Returns true if we have filled buffers to be processed. + * @return true if we have + */ + private boolean hasBuffers() { + return !mFilledBuffers.isEmpty(); + } + /** + * Drains the overflow data into the given {@link ShortBuffer}.The overflow data is + * already processed, it must just be copied into the given buffer. + * + * @param outBuffer output buffer + * @return the frame timestamp + */ + private long drainOverflow(@NonNull final ShortBuffer outBuffer) { + final ShortBuffer overflowBuffer = mOverflowBuffer.data; + final int overflowLimit = overflowBuffer.limit(); + final int overflowSize = overflowBuffer.remaining(); final long beginPresentationTimeUs = mOverflowBuffer.presentationTimeUs + - sampleCountToDurationUs(overflowBuff.position(), - mInputSampleRate, + sampleCountToDurationUs(overflowBuffer.position(), + mSampleRate, mOutputChannelCount); - - outBuff.clear(); - // Limit overflowBuff to outBuff's capacity - overflowBuff.limit(outBuff.capacity()); - // Load overflowBuff onto outBuff - outBuff.put(overflowBuff); - - if (overflowSize >= outBuff.capacity()) { - // Overflow fully consumed - Reset - overflowBuff.clear().limit(0); + outBuffer.clear(); + overflowBuffer.limit(outBuffer.capacity()); // Limit overflowBuffer to outBuffer's capacity + outBuffer.put(overflowBuffer); // Load overflowBuffer onto outBuffer + if (overflowSize >= outBuffer.capacity()) { + overflowBuffer.clear().limit(0); // Overflow fully consumed - Reset } else { - // Only partially consumed - Keep position & restore previous limit - overflowBuff.limit(overflowLimit); + overflowBuffer.limit(overflowLimit); // Only partially consumed - Keep position & restore previous limit } return beginPresentationTimeUs; } - private long remixAndMaybeFillOverflow(final AudioBuffer input, - final ShortBuffer outBuff) { - final ShortBuffer inBuff = input.data; - final ShortBuffer overflowBuff = mOverflowBuffer.data; - - outBuff.clear(); + private void process(@NonNull final ShortBuffer inputBuffer, + @NonNull final ShortBuffer outputBuffer, + long inputPresentationTimeUs) { + // Reset position to 0 and set limit to capacity (Since MediaCodec doesn't do that for us) + outputBuffer.clear(); + inputBuffer.clear(); - // Reset position to 0, and set limit to capacity (Since MediaCodec doesn't do that for us) - inBuff.clear(); - - if (inBuff.remaining() > outBuff.remaining()) { - // Overflow - // Limit inBuff to outBuff's capacity - inBuff.limit(outBuff.capacity()); - mRemixer.remix(inBuff, outBuff); - - // Reset limit to its own capacity & Keep position - inBuff.limit(inBuff.capacity()); + if (inputBuffer.remaining() <= outputBuffer.remaining()) { + // Safe case. Just remix. + mRemixer.remix(inputBuffer, outputBuffer); + } else { + // Overflow! + // First remix all we can. + inputBuffer.limit(outputBuffer.capacity()); + mRemixer.remix(inputBuffer, outputBuffer); + inputBuffer.limit(inputBuffer.capacity()); - // Remix the rest onto overflowBuffer + // Then remix the rest into mOverflowBuffer. // NOTE: We should only reach this point when overflow buffer is empty - final long consumedDurationUs = - sampleCountToDurationUs(inBuff.position(), mInputSampleRate, mInputChannelCount); - mRemixer.remix(inBuff, overflowBuff); + final long consumedDurationUs = sampleCountToDurationUs(inputBuffer.position(), mSampleRate, mInputChannelCount); + mRemixer.remix(inputBuffer, mOverflowBuffer.data); - // Seal off overflowBuff & mark limit - overflowBuff.flip(); - mOverflowBuffer.presentationTimeUs = input.presentationTimeUs + consumedDurationUs; - } else { - // No overflow - mRemixer.remix(inBuff, outBuff); - } + // Flip the overflow buffer and mark the presentation time. + mOverflowBuffer.data.flip(); + mOverflowBuffer.presentationTimeUs = inputPresentationTimeUs + consumedDurationUs; - return input.presentationTimeUs; + } } } From 8afbd035f35830132aca6920a07797980144211a Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Tue, 23 Jul 2019 13:46:35 +0100 Subject: [PATCH 10/22] Use AudioEngine and fix sample to us conversion --- .../transcode/AudioTrackTranscoder.java | 11 +++-- .../{AudioChannel.java => AudioEngine.java} | 47 +++++++++++++------ 2 files changed, 39 insertions(+), 19 deletions(-) rename lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/{AudioChannel.java => AudioEngine.java} (85%) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java index 2449ad8d..eae81389 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java @@ -9,13 +9,13 @@ import com.otaliastudios.transcoder.engine.TrackType; import com.otaliastudios.transcoder.engine.TranscoderMuxer; import com.otaliastudios.transcoder.internal.MediaCodecBuffers; -import com.otaliastudios.transcoder.transcode.internal.AudioChannel; +import com.otaliastudios.transcoder.transcode.internal.AudioEngine; import java.nio.ByteBuffer; public class AudioTrackTranscoder extends BaseTrackTranscoder { - private AudioChannel mAudioChannel; + private AudioEngine mAudioEngine; private MediaCodec mEncoder; // to create the channel private MediaFormat mEncoderOutputFormat; // to create the channel @@ -34,19 +34,20 @@ protected void onCodecsStarted(@NonNull MediaFormat inputFormat, @NonNull MediaF @Override protected boolean onFeedEncoder(@NonNull MediaCodec encoder, @NonNull MediaCodecBuffers encoderBuffers, long timeoutUs) { - return mAudioChannel.feedEncoder(encoderBuffers, timeoutUs); + if (mAudioEngine == null) return false; + return mAudioEngine.feedEncoder(encoderBuffers, timeoutUs); } @Override protected void onDecoderOutputFormatChanged(@NonNull MediaCodec decoder, @NonNull MediaFormat format) { super.onDecoderOutputFormatChanged(decoder, format); - mAudioChannel = new AudioChannel(decoder, format, mEncoder, mEncoderOutputFormat); + mAudioEngine = new AudioEngine(decoder, format, mEncoder, mEncoderOutputFormat); mEncoder = null; mEncoderOutputFormat = null; } @Override protected void onDrainDecoder(@NonNull MediaCodec decoder, int bufferIndex, @NonNull ByteBuffer bufferData, long presentationTimeUs, boolean endOfStream) { - mAudioChannel.drainDecoder(bufferIndex, bufferData, presentationTimeUs, endOfStream); + mAudioEngine.drainDecoder(bufferIndex, bufferData, presentationTimeUs, endOfStream); } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioChannel.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java similarity index 85% rename from lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioChannel.java rename to lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java index 6c1bde6b..7549b7a5 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioChannel.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java @@ -21,7 +21,7 @@ * We currently support upmixing from mono to stereo & downmixing from stereo to mono. * Sample rate conversion is not supported yet. */ -public class AudioChannel { +public class AudioEngine { private static class AudioBuffer { int bufferIndex; @@ -30,15 +30,26 @@ private static class AudioBuffer { boolean endOfStream; } - private static long sampleCountToDurationUs( - final int sampleCount, - final int sampleRate, - final int channelCount) { - return (sampleCount / (sampleRate * MICROSECONDS_PER_SECOND)) / channelCount; + private static long bytesToUs( + int bytes /* [bytes] */, + int sampleRate /* [samples/sec] */, + int channels /* [channel] */ + ) { + int bytesPerSamplePerChannel = 2; // [bytes/sample/channel] Assuming 16bit audio so 2 + int byteRatePerChannel = sampleRate * bytesPerSamplePerChannel; // [bytes/sec/channel] + int byteRate = byteRatePerChannel * channels; // [bytes/sec] + return MICROSECONDS_PER_SECOND * bytes / byteRate; // [usec] + } + + private static long shortsToUs( + int shorts, + int sampleRate, + int channels) { + return bytesToUs(shorts * BYTES_PER_SHORT, sampleRate, channels); } private static final int BYTES_PER_SHORT = 2; - private static final long MICROSECONDS_PER_SECOND = 1000000; + private static final long MICROSECONDS_PER_SECOND = 1000000L; private final Queue mEmptyBuffers = new ArrayDeque<>(); private final Queue mFilledBuffers = new ArrayDeque<>(); @@ -50,11 +61,19 @@ private static long sampleCountToDurationUs( private final int mOutputChannelCount; private final AudioRemixer mRemixer; - - public AudioChannel(@NonNull final MediaCodec decoder, - @NonNull final MediaFormat decoderOutputFormat, - @NonNull final MediaCodec encoder, - @NonNull final MediaFormat encoderOutputFormat) { + /** + * The AudioEngine should be created when we know the actual decoded format, + * which means that the decoder has reached {@link MediaCodec#INFO_OUTPUT_FORMAT_CHANGED}. + * + * @param decoder a decoder + * @param decoderOutputFormat the decoder output format + * @param encoder an encoder + * @param encoderOutputFormat the encoder output format + */ + public AudioEngine(@NonNull final MediaCodec decoder, + @NonNull final MediaFormat decoderOutputFormat, + @NonNull final MediaCodec encoder, + @NonNull final MediaFormat encoderOutputFormat) { mDecoder = decoder; mEncoder = encoder; @@ -207,7 +226,7 @@ private long drainOverflow(@NonNull final ShortBuffer outBuffer) { final int overflowLimit = overflowBuffer.limit(); final int overflowSize = overflowBuffer.remaining(); final long beginPresentationTimeUs = mOverflowBuffer.presentationTimeUs + - sampleCountToDurationUs(overflowBuffer.position(), + shortsToUs(overflowBuffer.position(), mSampleRate, mOutputChannelCount); outBuffer.clear(); @@ -241,7 +260,7 @@ private void process(@NonNull final ShortBuffer inputBuffer, // Then remix the rest into mOverflowBuffer. // NOTE: We should only reach this point when overflow buffer is empty - final long consumedDurationUs = sampleCountToDurationUs(inputBuffer.position(), mSampleRate, mInputChannelCount); + long consumedDurationUs = shortsToUs(inputBuffer.position(), mSampleRate, mInputChannelCount); mRemixer.remix(inputBuffer, mOverflowBuffer.data); // Flip the overflow buffer and mark the presentation time. From bb38c4271a2b9698d52a6b1169396beb648cc4c5 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Tue, 23 Jul 2019 13:57:11 +0100 Subject: [PATCH 11/22] Create video frame dropper --- .../transcoder/engine/TranscoderEngine.java | 4 +- .../transcode/AudioTrackTranscoder.java | 6 +- .../transcode/BaseTrackTranscoder.java | 5 +- .../transcode/VideoTrackTranscoder.java | 62 ++-------- .../transcode/internal/AudioEngine.java | 10 +- .../transcode/internal/VideoFrameDropper.java | 108 ++++++++++++++++++ 6 files changed, 129 insertions(+), 66 deletions(-) create mode 100644 lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/VideoFrameDropper.java diff --git a/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderEngine.java b/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderEngine.java index 30fa2d1b..bb08b6c6 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderEngine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderEngine.java @@ -201,8 +201,8 @@ private void setUpTrackTranscoder(@NonNull TranscoderOptions options, status = TrackStatus.PASS_THROUGH; } else { switch (type) { - case VIDEO: transcoder = new VideoTrackTranscoder(mExtractor, index, muxer); break; - case AUDIO: transcoder = new AudioTrackTranscoder(mExtractor, index, muxer); break; + case VIDEO: transcoder = new VideoTrackTranscoder(mExtractor, muxer, index); break; + case AUDIO: transcoder = new AudioTrackTranscoder(mExtractor, muxer, index); break; default: throw new RuntimeException("Unknown type: " + type); } status = TrackStatus.COMPRESSING; diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java index eae81389..60118b9a 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java @@ -20,9 +20,9 @@ public class AudioTrackTranscoder extends BaseTrackTranscoder { private MediaFormat mEncoderOutputFormat; // to create the channel public AudioTrackTranscoder(@NonNull MediaExtractor extractor, - int trackIndex, - @NonNull TranscoderMuxer muxer) { - super(extractor, trackIndex, muxer, TrackType.AUDIO); + @NonNull TranscoderMuxer muxer, + int trackIndex) { + super(extractor, muxer, TrackType.AUDIO, trackIndex); } @Override diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java index 6cdbc966..1b04416f 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java @@ -6,7 +6,6 @@ import androidx.annotation.CallSuper; import androidx.annotation.NonNull; -import androidx.annotation.Nullable; import com.otaliastudios.transcoder.engine.TrackType; import com.otaliastudios.transcoder.engine.TranscoderMuxer; @@ -47,9 +46,9 @@ public abstract class BaseTrackTranscoder implements TrackTranscoder { @SuppressWarnings("WeakerAccess") protected BaseTrackTranscoder(@NonNull MediaExtractor extractor, - int trackIndex, @NonNull TranscoderMuxer muxer, - @NonNull TrackType trackType) { + @NonNull TrackType trackType, + int trackIndex) { mExtractor = extractor; mTrackIndex = trackIndex; mMuxer = muxer; diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java index b809e101..17b7997d 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java @@ -28,6 +28,7 @@ import com.otaliastudios.transcoder.transcode.internal.VideoEncoderInput; import com.otaliastudios.transcoder.internal.Logger; import com.otaliastudios.transcoder.internal.MediaFormatConstants; +import com.otaliastudios.transcoder.transcode.internal.VideoFrameDropper; import java.nio.ByteBuffer; @@ -35,21 +36,18 @@ public class VideoTrackTranscoder extends BaseTrackTranscoder { private static final String TAG = VideoTrackTranscoder.class.getSimpleName(); + @SuppressWarnings("unused") private static final Logger LOG = new Logger(TAG); private VideoDecoderOutput mDecoderOutputSurface; private VideoEncoderInput mEncoderInputSurface; private MediaCodec mEncoder; // Keep this since we want to signal EOS on it. - - private double mInFrameRateReciprocal; - private double mOutFrameRateReciprocal; - private double mFrameRateReciprocalSum; + private VideoFrameDropper mFrameDropper; public VideoTrackTranscoder( @NonNull MediaExtractor extractor, - int trackIndex, - @NonNull TranscoderMuxer muxer) { - super(extractor, trackIndex, muxer, TrackType.VIDEO); + @NonNull TranscoderMuxer muxer, int trackIndex) { + super(extractor, muxer, TrackType.VIDEO, trackIndex); } @Override @@ -72,6 +70,9 @@ protected void onConfigureDecoder(@NonNull MediaFormat format, @NonNull MediaCod @Override protected void onCodecsStarted(@NonNull MediaFormat inputFormat, @NonNull MediaFormat outputFormat, @NonNull MediaCodec decoder, @NonNull MediaCodec encoder) { super.onCodecsStarted(inputFormat, outputFormat, decoder, encoder); + mFrameDropper = VideoFrameDropper.newDropper( + inputFormat.getInteger(MediaFormat.KEY_FRAME_RATE), + outputFormat.getInteger(MediaFormat.KEY_FRAME_RATE)); mEncoder = encoder; // Cropping support. @@ -90,10 +91,6 @@ protected void onCodecsStarted(@NonNull MediaFormat inputFormat, @NonNull MediaF // I don't think we should consider rotation and flip these - we operate on non-rotated // surfaces and pass the input rotation metadata to the output muxer, see TranscoderEngine.setupMetadata. mDecoderOutputSurface.setScale(scaleX, scaleY); - - // Frame dropping support. - int frameRate = outputFormat.getInteger(MediaFormat.KEY_FRAME_RATE); - mTargetAvgStep = (1F / frameRate) * 1000 * 1000; } @Override @@ -121,51 +118,10 @@ protected void onDrainDecoder(@NonNull MediaCodec decoder, int bufferIndex, @Non if (endOfStream) { mEncoder.signalEndOfInputStream(); decoder.releaseOutputBuffer(bufferIndex, false); - } else if (shouldRenderFrame(presentationTimeUs)) { + } else if (mFrameDropper.shouldRenderFrame(presentationTimeUs)) { decoder.releaseOutputBuffer(bufferIndex, true); mDecoderOutputSurface.drawFrame(); mEncoderInputSurface.onFrame(presentationTimeUs); } } - - // TODO improve this. as it is now, rendering a frame after dropping many, - // will not decrease avgStep but rather increase it (for this single frame; then it starts decreasing). - // This has the effect that, when a frame is rendered, the following frame is always rendered, - // because the conditions are worse then before. After this second frame things go back to normal, - // but this is terrible logic. - private boolean shouldRenderFrame(long presentationTimeUs) { - if (mRenderedSteps > 0 && mAvgStep < mTargetAvgStep) { - // We are rendering too much. Drop this frame. - // Always render first 2 frames, we need them to compute the avg. - LOG.v("FRAME: Dropping. avg: " + mAvgStep + " target: " + mTargetAvgStep); - long newLastStep = presentationTimeUs - mLastRenderedUs; - float allSteps = (mAvgStep * mRenderedSteps) - mLastStep + newLastStep; - mAvgStep = allSteps / mRenderedSteps; // we didn't add a step, just increased the last - mLastStep = newLastStep; - return false; - } else { - // Render this frame, since our average step is too long or exact. - LOG.v("FRAME: RENDERING. avg: " + mAvgStep + " target: " + mTargetAvgStep + "New stepCount: " + (mRenderedSteps + 1)); - if (mRenderedSteps >= 0) { - // Update the average value, since now we have mLastRenderedUs. - long step = presentationTimeUs - mLastRenderedUs; - float allSteps = (mAvgStep * mRenderedSteps) + step; - mAvgStep = allSteps / (mRenderedSteps + 1); // we added a step, so +1 - mLastStep = step; - } - // Increment both - mRenderedSteps++; - mLastRenderedUs = presentationTimeUs; - return true; - } - if (mFrameRateReciprocalSum > mOutFrameRateReciprocal) { - mFrameRateReciprocalSum -= mOutFrameRateReciprocal; - // render frame - LOG.v("render this frame -> mFrameRateReciprocalSum: " + mFrameRateReciprocalSum); - return true; - } - // drop frame - LOG.v("drop this frame -> mFrameRateReciprocalSum: " + mFrameRateReciprocalSum); - return false; - } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java index 7549b7a5..42da6b2a 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java @@ -30,13 +30,16 @@ private static class AudioBuffer { boolean endOfStream; } + private static final int BYTES_PER_SAMPLE_PER_CHANNEL = 2; // Assuming 16bit audio, so 2 + private static final int BYTES_PER_SHORT = 2; + private static final long MICROSECONDS_PER_SECOND = 1000000L; + private static long bytesToUs( int bytes /* [bytes] */, int sampleRate /* [samples/sec] */, int channels /* [channel] */ ) { - int bytesPerSamplePerChannel = 2; // [bytes/sample/channel] Assuming 16bit audio so 2 - int byteRatePerChannel = sampleRate * bytesPerSamplePerChannel; // [bytes/sec/channel] + int byteRatePerChannel = sampleRate * BYTES_PER_SAMPLE_PER_CHANNEL; // [bytes/sec/channel] int byteRate = byteRatePerChannel * channels; // [bytes/sec] return MICROSECONDS_PER_SECOND * bytes / byteRate; // [usec] } @@ -48,9 +51,6 @@ private static long shortsToUs( return bytesToUs(shorts * BYTES_PER_SHORT, sampleRate, channels); } - private static final int BYTES_PER_SHORT = 2; - private static final long MICROSECONDS_PER_SECOND = 1000000L; - private final Queue mEmptyBuffers = new ArrayDeque<>(); private final Queue mFilledBuffers = new ArrayDeque<>(); private final MediaCodec mDecoder; diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/VideoFrameDropper.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/VideoFrameDropper.java new file mode 100644 index 00000000..e93bd605 --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/VideoFrameDropper.java @@ -0,0 +1,108 @@ +package com.otaliastudios.transcoder.transcode.internal; + +import androidx.annotation.NonNull; + +import com.otaliastudios.transcoder.internal.Logger; +import com.otaliastudios.transcoder.transcode.VideoTrackTranscoder; + +/** + * Drops input frames to respect the output frame rate. + */ +public abstract class VideoFrameDropper { + + private final static String TAG = VideoFrameDropper.class.getSimpleName(); + private final static Logger LOG = new Logger(TAG); + + private VideoFrameDropper() {} + + public abstract boolean shouldRenderFrame(long presentationTimeUs); + + @NonNull + public static VideoFrameDropper newDropper(int inputFrameRate, int outputFrameRate) { + return new Dropper1(inputFrameRate, outputFrameRate); + } + + private static class Dropper1 extends VideoFrameDropper { + + private double mInFrameRateReciprocal; + private double mOutFrameRateReciprocal; + private double mFrameRateReciprocalSum; + + private Dropper1(int inputFrameRate, int outputFrameRate) { + mInFrameRateReciprocal = 1.0d / inputFrameRate; + mOutFrameRateReciprocal = 1.0d / outputFrameRate; + LOG.i("inFrameRateReciprocal:" + mInFrameRateReciprocal + " outFrameRateReciprocal:" + mOutFrameRateReciprocal); + } + + @Override + public boolean shouldRenderFrame(long presentationTimeUs) { + boolean firstFrame = Double.valueOf(0d).equals(mFrameRateReciprocalSum); + mFrameRateReciprocalSum += mInFrameRateReciprocal; + if (firstFrame) { + LOG.v("RENDERING - frameRateReciprocalSum:" + mFrameRateReciprocalSum); + return true; + } + if (mFrameRateReciprocalSum > mOutFrameRateReciprocal) { + mFrameRateReciprocalSum -= mOutFrameRateReciprocal; + LOG.v("RENDERING - frameRateReciprocalSum:" + mFrameRateReciprocalSum); + return true; + } + LOG.v("DROPPING - frameRateReciprocalSum:" + mFrameRateReciprocalSum); + return false; + } + } + + private static class Dropper2 extends VideoFrameDropper { + + // A step is defined as the microseconds between two frame. + // The average step is basically 1 / frame rate. + private float mAvgStep = 0; + private float mTargetAvgStep; + private int mRenderedSteps = -1; // frames - 1 + private long mLastRenderedUs; + private long mLastStep; + + private Dropper2(int outputFrameRate) { + mTargetAvgStep = (1F / outputFrameRate) * 1000 * 1000; + + } + + /** + * TODO improve this. as it is now, rendering a frame after dropping many, + * will not decrease avgStep but rather increase it (for this single frame; then it starts decreasing). + * This has the effect that, when a frame is rendered, the following frame is always rendered, + * because the conditions are worse then before. After this second frame things go back to normal, + * but this is terrible logic. + */ + @Override + public boolean shouldRenderFrame(long presentationTimeUs) { + if (mRenderedSteps > 0 && mAvgStep < mTargetAvgStep) { + // We are rendering too much. Drop this frame. + // Always render first 2 frames, we need them to compute the avg. + LOG.v("DROPPING - avg:" + mAvgStep + " target:" + mTargetAvgStep); + long newLastStep = presentationTimeUs - mLastRenderedUs; + float allSteps = (mAvgStep * mRenderedSteps) - mLastStep + newLastStep; + mAvgStep = allSteps / mRenderedSteps; // we didn't add a step, just increased the last + mLastStep = newLastStep; + return false; + } else { + // Render this frame, since our average step is too long or exact. + LOG.v("RENDERING - avg:" + mAvgStep + " target:" + mTargetAvgStep + " newStepCount:" + (mRenderedSteps + 1)); + if (mRenderedSteps >= 0) { + // Update the average value, since now we have mLastRenderedUs. + long step = presentationTimeUs - mLastRenderedUs; + float allSteps = (mAvgStep * mRenderedSteps) + step; + mAvgStep = allSteps / (mRenderedSteps + 1); // we added a step, so +1 + mLastStep = step; + } + // Increment both + mRenderedSteps++; + mLastRenderedUs = presentationTimeUs; + return true; + } + } + } + + + +} From 89e409d73bfd0cc1d7f54430bf99558c244ca48c Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Tue, 23 Jul 2019 14:14:49 +0100 Subject: [PATCH 12/22] Use TimeInterpolator in transcoders not muxer --- .../transcoder/engine/TranscoderEngine.java | 12 ++++---- .../transcoder/engine/TranscoderMuxer.java | 13 +-------- .../transcode/AudioTrackTranscoder.java | 9 ++++-- .../transcode/BaseTrackTranscoder.java | 2 +- .../transcode/NoOpTrackTranscoder.java | 2 +- .../transcode/PassThroughTrackTranscoder.java | 16 +++++++--- .../transcoder/transcode/TrackTranscoder.java | 2 +- .../transcode/VideoTrackTranscoder.java | 10 +++++-- .../transcode/internal/AudioEngine.java | 29 +++++++++++++------ 9 files changed, 57 insertions(+), 38 deletions(-) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderEngine.java b/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderEngine.java index bb08b6c6..c7797903 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderEngine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderEngine.java @@ -197,12 +197,12 @@ private void setUpTrackTranscoder(@NonNull TranscoderOptions options, transcoder = new NoOpTrackTranscoder(); status = TrackStatus.REMOVING; } else if (outputFormat == inputFormat) { - transcoder = new PassThroughTrackTranscoder(mExtractor, index, muxer, type); + transcoder = new PassThroughTrackTranscoder(mExtractor, index, muxer, type, options.getTimeInterpolator()); status = TrackStatus.PASS_THROUGH; } else { switch (type) { - case VIDEO: transcoder = new VideoTrackTranscoder(mExtractor, muxer, index); break; - case AUDIO: transcoder = new AudioTrackTranscoder(mExtractor, muxer, index); break; + case VIDEO: transcoder = new VideoTrackTranscoder(mExtractor, muxer, index, options.getTimeInterpolator()); break; + case AUDIO: transcoder = new AudioTrackTranscoder(mExtractor, muxer, index, options.getTimeInterpolator()); break; default: throw new RuntimeException("Unknown type: " + type); } status = TrackStatus.COMPRESSING; @@ -210,7 +210,7 @@ private void setUpTrackTranscoder(@NonNull TranscoderOptions options, } catch (OutputStrategyException strategyException) { if (strategyException.getType() == OutputStrategyException.TYPE_ALREADY_COMPRESSED) { // Should not abort, because the other track might need compression. Use a pass through. - transcoder = new PassThroughTrackTranscoder(mExtractor, index, muxer, type); + transcoder = new PassThroughTrackTranscoder(mExtractor, index, muxer, type, options.getTimeInterpolator()); status = TrackStatus.PASS_THROUGH; } else { // Abort. throw strategyException; @@ -226,7 +226,7 @@ private void setUpTrackTranscoder(@NonNull TranscoderOptions options, private void setupTrackTranscoders(@NonNull TranscoderOptions options) { mTracks = Tracks.create(mExtractor); - TranscoderMuxer muxer = new TranscoderMuxer(mMuxer, mTracks, options.getTimeInterpolator()); + TranscoderMuxer muxer = new TranscoderMuxer(mMuxer, mTracks); setUpTrackTranscoder(options, muxer, TrackType.VIDEO); setUpTrackTranscoder(options, muxer, TrackType.AUDIO); @@ -277,7 +277,7 @@ private void runPipelines() throws InterruptedException { private double getTranscoderProgress(@NonNull TrackTranscoder transcoder, @NonNull TrackStatus status) { if (!status.isTranscoding()) return 0.0; if (transcoder.isFinished()) return 1.0; - return Math.min(1.0, (double) transcoder.getLastWrittenPresentationTime() / mDurationUs); + return Math.min(1.0, (double) transcoder.getLastPresentationTime() / mDurationUs); } private int getTranscodersCount() { diff --git a/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderMuxer.java b/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderMuxer.java index 005f7b7d..bffec0bb 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderMuxer.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderMuxer.java @@ -20,7 +20,6 @@ import android.media.MediaMuxer; import com.otaliastudios.transcoder.internal.Logger; -import com.otaliastudios.transcoder.time.TimeInterpolator; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -62,12 +61,10 @@ private void toBufferInfo(@NonNull MediaCodec.BufferInfo bufferInfo, int offset) private boolean mMuxerStarted; private final List mQueue = new ArrayList<>(); private ByteBuffer mQueueBuffer; - private TimeInterpolator mTimeInterpolator; - TranscoderMuxer(@NonNull MediaMuxer muxer, @NonNull Tracks info, @NonNull TimeInterpolator timeInterpolator) { + TranscoderMuxer(@NonNull MediaMuxer muxer, @NonNull Tracks info) { mMuxer = muxer; mTracks = info; - mTimeInterpolator = timeInterpolator; } /** @@ -111,16 +108,8 @@ public void setOutputFormat(@NonNull TrackType trackType, @NonNull MediaFormat f drainQueue(); } - @SuppressWarnings("StatementWithEmptyBody") public void write(@NonNull TrackType type, @NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) { if (mMuxerStarted) { - boolean isEos = (bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0; - if (isEos && bufferInfo.presentationTimeUs == 0) { - // Do not pass this to the interpolator, it needs increasing timestamps, - // this is not a real buffer, just a signal. - } else { - bufferInfo.presentationTimeUs = mTimeInterpolator.interpolate(type, bufferInfo.presentationTimeUs); - } mMuxer.writeSampleData(mTracks.outputIndex(type), byteBuf, bufferInfo); } else { enqueue(type, byteBuf, bufferInfo); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java index 60118b9a..4673ff6f 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java @@ -9,20 +9,24 @@ import com.otaliastudios.transcoder.engine.TrackType; import com.otaliastudios.transcoder.engine.TranscoderMuxer; import com.otaliastudios.transcoder.internal.MediaCodecBuffers; +import com.otaliastudios.transcoder.time.TimeInterpolator; import com.otaliastudios.transcoder.transcode.internal.AudioEngine; import java.nio.ByteBuffer; public class AudioTrackTranscoder extends BaseTrackTranscoder { + private TimeInterpolator mTimeInterpolator; private AudioEngine mAudioEngine; private MediaCodec mEncoder; // to create the channel private MediaFormat mEncoderOutputFormat; // to create the channel public AudioTrackTranscoder(@NonNull MediaExtractor extractor, @NonNull TranscoderMuxer muxer, - int trackIndex) { + int trackIndex, + @NonNull TimeInterpolator timeInterpolator) { super(extractor, muxer, TrackType.AUDIO, trackIndex); + mTimeInterpolator = timeInterpolator; } @Override @@ -41,9 +45,10 @@ protected boolean onFeedEncoder(@NonNull MediaCodec encoder, @NonNull MediaCodec @Override protected void onDecoderOutputFormatChanged(@NonNull MediaCodec decoder, @NonNull MediaFormat format) { super.onDecoderOutputFormatChanged(decoder, format); - mAudioEngine = new AudioEngine(decoder, format, mEncoder, mEncoderOutputFormat); + mAudioEngine = new AudioEngine(decoder, format, mEncoder, mEncoderOutputFormat, mTimeInterpolator); mEncoder = null; mEncoderOutputFormat = null; + mTimeInterpolator = null; } @Override diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java index 1b04416f..377a607b 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java @@ -133,7 +133,7 @@ protected void onCodecsStarted(@NonNull MediaFormat inputFormat, @NonNull MediaF } @Override - public final long getLastWrittenPresentationTime() { + public final long getLastPresentationTime() { return mWrittenPresentationTimeUs; } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/NoOpTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/NoOpTrackTranscoder.java index e1419955..eda1aa91 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/NoOpTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/NoOpTrackTranscoder.java @@ -34,7 +34,7 @@ public boolean stepPipeline() { } @Override - public long getLastWrittenPresentationTime() { + public long getLastPresentationTime() { return 0; } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/PassThroughTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/PassThroughTrackTranscoder.java index 029c23bd..3f6e131f 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/PassThroughTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/PassThroughTrackTranscoder.java @@ -24,6 +24,7 @@ import com.otaliastudios.transcoder.engine.TrackType; import com.otaliastudios.transcoder.engine.TranscoderMuxer; +import com.otaliastudios.transcoder.time.TimeInterpolator; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -41,10 +42,13 @@ public class PassThroughTrackTranscoder implements TrackTranscoder { private final MediaFormat mActualOutputFormat; private boolean mFirstStepPipeline = true; + private TimeInterpolator mTimeInterpolator; + public PassThroughTrackTranscoder(@NonNull MediaExtractor extractor, int trackIndex, @NonNull TranscoderMuxer muxer, - @NonNull TrackType trackType) { + @NonNull TrackType trackType, + @NonNull TimeInterpolator timeInterpolator) { mExtractor = extractor; mTrackIndex = trackIndex; mMuxer = muxer; @@ -53,6 +57,8 @@ public PassThroughTrackTranscoder(@NonNull MediaExtractor extractor, mActualOutputFormat = mExtractor.getTrackFormat(mTrackIndex); mBufferSize = mActualOutputFormat.getInteger(MediaFormat.KEY_MAX_INPUT_SIZE); mBuffer = ByteBuffer.allocateDirect(mBufferSize).order(ByteOrder.nativeOrder()); + + mTimeInterpolator = timeInterpolator; } @Override @@ -82,16 +88,18 @@ public boolean stepPipeline() { assert sampleSize <= mBufferSize; boolean isKeyFrame = (mExtractor.getSampleFlags() & MediaExtractor.SAMPLE_FLAG_SYNC) != 0; int flags = isKeyFrame ? MediaCodec.BUFFER_FLAG_SYNC_FRAME : 0; - mBufferInfo.set(0, sampleSize, mExtractor.getSampleTime(), flags); + long realTimestampUs = mExtractor.getSampleTime(); + long timestampUs = mTimeInterpolator.interpolate(mTrackType, realTimestampUs); + mBufferInfo.set(0, sampleSize, timestampUs, flags); mMuxer.write(mTrackType, mBuffer, mBufferInfo); - mWrittenPresentationTimeUs = mBufferInfo.presentationTimeUs; + mWrittenPresentationTimeUs = realTimestampUs; mExtractor.advance(); return true; } @Override - public long getLastWrittenPresentationTime() { + public long getLastPresentationTime() { return mWrittenPresentationTimeUs; } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/TrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/TrackTranscoder.java index 9e90e48a..4a40bd3f 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/TrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/TrackTranscoder.java @@ -37,7 +37,7 @@ public interface TrackTranscoder { * * @return Presentation time in micro-second. Return value is undefined if finished writing. */ - long getLastWrittenPresentationTime(); + long getLastPresentationTime(); boolean isFinished(); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java index 17b7997d..c6f5ee8b 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java @@ -24,6 +24,7 @@ import com.otaliastudios.transcoder.engine.TrackType; import com.otaliastudios.transcoder.engine.TranscoderMuxer; import com.otaliastudios.transcoder.internal.MediaCodecBuffers; +import com.otaliastudios.transcoder.time.TimeInterpolator; import com.otaliastudios.transcoder.transcode.internal.VideoDecoderOutput; import com.otaliastudios.transcoder.transcode.internal.VideoEncoderInput; import com.otaliastudios.transcoder.internal.Logger; @@ -43,11 +44,15 @@ public class VideoTrackTranscoder extends BaseTrackTranscoder { private VideoEncoderInput mEncoderInputSurface; private MediaCodec mEncoder; // Keep this since we want to signal EOS on it. private VideoFrameDropper mFrameDropper; + private TimeInterpolator mTimeInterpolator; public VideoTrackTranscoder( @NonNull MediaExtractor extractor, - @NonNull TranscoderMuxer muxer, int trackIndex) { + @NonNull TranscoderMuxer muxer, + int trackIndex, + @NonNull TimeInterpolator timeInterpolator) { super(extractor, muxer, TrackType.VIDEO, trackIndex); + mTimeInterpolator = timeInterpolator; } @Override @@ -120,8 +125,9 @@ protected void onDrainDecoder(@NonNull MediaCodec decoder, int bufferIndex, @Non decoder.releaseOutputBuffer(bufferIndex, false); } else if (mFrameDropper.shouldRenderFrame(presentationTimeUs)) { decoder.releaseOutputBuffer(bufferIndex, true); + long interpolatedTimeUs = mTimeInterpolator.interpolate(TrackType.VIDEO, presentationTimeUs); mDecoderOutputSurface.drawFrame(); - mEncoderInputSurface.onFrame(presentationTimeUs); + mEncoderInputSurface.onFrame(interpolatedTimeUs); } } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java index 42da6b2a..1485f790 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java @@ -5,8 +5,10 @@ import androidx.annotation.NonNull; +import com.otaliastudios.transcoder.engine.TrackType; import com.otaliastudios.transcoder.internal.MediaCodecBuffers; import com.otaliastudios.transcoder.remix.AudioRemixer; +import com.otaliastudios.transcoder.time.TimeInterpolator; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -60,6 +62,8 @@ private static long shortsToUs( private final int mInputChannelCount; private final int mOutputChannelCount; private final AudioRemixer mRemixer; + private final TimeInterpolator mTimeInterpolator; + /** * The AudioEngine should be created when we know the actual decoded format, @@ -70,12 +74,14 @@ private static long shortsToUs( * @param encoder an encoder * @param encoderOutputFormat the encoder output format */ - public AudioEngine(@NonNull final MediaCodec decoder, - @NonNull final MediaFormat decoderOutputFormat, - @NonNull final MediaCodec encoder, - @NonNull final MediaFormat encoderOutputFormat) { + public AudioEngine(@NonNull MediaCodec decoder, + @NonNull MediaFormat decoderOutputFormat, + @NonNull MediaCodec encoder, + @NonNull MediaFormat encoderOutputFormat, + @NonNull TimeInterpolator timeInterpolator) { mDecoder = decoder; mEncoder = encoder; + mTimeInterpolator = timeInterpolator; // Get and check sample rate. int outputSampleRate = encoderOutputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE); @@ -185,11 +191,12 @@ public boolean feedEncoder(@NonNull MediaCodecBuffers encoderBuffers, long timeo } // If not, process data and return true. - process(decoderBuffer.data, encoderBuffer, decoderBuffer.presentationTimeUs); + long inputPresentationTime = decoderBuffer.presentationTimeUs; + long outputPresentationTime = process(decoderBuffer.data, encoderBuffer, inputPresentationTime); mEncoder.queueInputBuffer(encoderBufferIndex, 0, encoderBuffer.position() * BYTES_PER_SHORT, - decoderBuffer.presentationTimeUs, + outputPresentationTime, 0); mDecoder.releaseOutputBuffer(decoderBuffer.bufferIndex, false); mEmptyBuffers.add(decoderBuffer); @@ -241,13 +248,16 @@ private long drainOverflow(@NonNull final ShortBuffer outBuffer) { return beginPresentationTimeUs; } - private void process(@NonNull final ShortBuffer inputBuffer, + private long process(@NonNull final ShortBuffer inputBuffer, @NonNull final ShortBuffer outputBuffer, long inputPresentationTimeUs) { + long outputPresentationTime = mTimeInterpolator.interpolate(TrackType.AUDIO, inputPresentationTimeUs); + // Reset position to 0 and set limit to capacity (Since MediaCodec doesn't do that for us) outputBuffer.clear(); inputBuffer.clear(); + if (inputBuffer.remaining() <= outputBuffer.remaining()) { // Safe case. Just remix. mRemixer.remix(inputBuffer, outputBuffer); @@ -258,6 +268,7 @@ private void process(@NonNull final ShortBuffer inputBuffer, mRemixer.remix(inputBuffer, outputBuffer); inputBuffer.limit(inputBuffer.capacity()); + // TODO check the time logic below if interpolator changes time. // Then remix the rest into mOverflowBuffer. // NOTE: We should only reach this point when overflow buffer is empty long consumedDurationUs = shortsToUs(inputBuffer.position(), mSampleRate, mInputChannelCount); @@ -265,8 +276,8 @@ private void process(@NonNull final ShortBuffer inputBuffer, // Flip the overflow buffer and mark the presentation time. mOverflowBuffer.data.flip(); - mOverflowBuffer.presentationTimeUs = inputPresentationTimeUs + consumedDurationUs; - + mOverflowBuffer.presentationTimeUs = outputPresentationTime + consumedDurationUs; } + return outputPresentationTime; } } From 1602ba16e93f8768a4d26323fe970af22e0ba41e Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Tue, 23 Jul 2019 14:18:37 +0100 Subject: [PATCH 13/22] Comments --- .../transcode/BaseTrackTranscoder.java | 6 ++--- .../transcode/PassThroughTrackTranscoder.java | 25 ++++++++++--------- .../transcoder/transcode/TrackTranscoder.java | 3 ++- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java index 377a607b..3d2f21ef 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java @@ -29,7 +29,7 @@ public abstract class BaseTrackTranscoder implements TrackTranscoder { private final TranscoderMuxer mMuxer; private final TrackType mTrackType; - private long mWrittenPresentationTimeUs; + private long mLastPresentationTimeUs; private final MediaCodec.BufferInfo mBufferInfo = new MediaCodec.BufferInfo(); private MediaCodec mDecoder; @@ -134,7 +134,7 @@ protected void onCodecsStarted(@NonNull MediaFormat inputFormat, @NonNull MediaF @Override public final long getLastPresentationTime() { - return mWrittenPresentationTimeUs; + return mLastPresentationTimeUs; } @Override @@ -284,7 +284,7 @@ private int drainEncoder(long timeoutUs) { return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; } mMuxer.write(mTrackType, mEncoderBuffers.getOutputBuffer(result), mBufferInfo); - mWrittenPresentationTimeUs = mBufferInfo.presentationTimeUs; + mLastPresentationTimeUs = mBufferInfo.presentationTimeUs; mEncoder.releaseOutputBuffer(result, false); return DRAIN_STATE_CONSUMED; } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/PassThroughTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/PassThroughTrackTranscoder.java index 3f6e131f..54b5b6bc 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/PassThroughTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/PassThroughTrackTranscoder.java @@ -38,9 +38,11 @@ public class PassThroughTrackTranscoder implements TrackTranscoder { private int mBufferSize; private ByteBuffer mBuffer; private boolean mIsEOS; - private long mWrittenPresentationTimeUs; - private final MediaFormat mActualOutputFormat; - private boolean mFirstStepPipeline = true; + + private long mLastPresentationTime; + + private final MediaFormat mOutputFormat; + private boolean mOutputFormatSet = false; private TimeInterpolator mTimeInterpolator; @@ -54,24 +56,23 @@ public PassThroughTrackTranscoder(@NonNull MediaExtractor extractor, mMuxer = muxer; mTrackType = trackType; - mActualOutputFormat = mExtractor.getTrackFormat(mTrackIndex); - mBufferSize = mActualOutputFormat.getInteger(MediaFormat.KEY_MAX_INPUT_SIZE); + mOutputFormat = mExtractor.getTrackFormat(mTrackIndex); + mBufferSize = mOutputFormat.getInteger(MediaFormat.KEY_MAX_INPUT_SIZE); mBuffer = ByteBuffer.allocateDirect(mBufferSize).order(ByteOrder.nativeOrder()); mTimeInterpolator = timeInterpolator; } @Override - public void setUp(@NonNull MediaFormat desiredOutputFormat) { - } + public void setUp(@NonNull MediaFormat desiredOutputFormat) { } @SuppressLint("Assert") @Override public boolean stepPipeline() { if (mIsEOS) return false; - if (mFirstStepPipeline) { - mMuxer.setOutputFormat(mTrackType, mActualOutputFormat); - mFirstStepPipeline = false; + if (!mOutputFormatSet) { + mMuxer.setOutputFormat(mTrackType, mOutputFormat); + mOutputFormatSet = true; } int trackIndex = mExtractor.getSampleTrackIndex(); if (trackIndex < 0) { @@ -92,7 +93,7 @@ public boolean stepPipeline() { long timestampUs = mTimeInterpolator.interpolate(mTrackType, realTimestampUs); mBufferInfo.set(0, sampleSize, timestampUs, flags); mMuxer.write(mTrackType, mBuffer, mBufferInfo); - mWrittenPresentationTimeUs = realTimestampUs; + mLastPresentationTime = realTimestampUs; mExtractor.advance(); return true; @@ -100,7 +101,7 @@ public boolean stepPipeline() { @Override public long getLastPresentationTime() { - return mWrittenPresentationTimeUs; + return mLastPresentationTime; } @Override diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/TrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/TrackTranscoder.java index 4a40bd3f..760f51df 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/TrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/TrackTranscoder.java @@ -33,7 +33,8 @@ public interface TrackTranscoder { boolean stepPipeline(); /** - * Get presentation time of last sample written to muxer. + * Get presentation time of last sample taken from encoder. + * This presentation time should not be affected by {@link com.otaliastudios.transcoder.time.TimeInterpolator}s. * * @return Presentation time in micro-second. Return value is undefined if finished writing. */ From bf740a913f9368acd28ccea348c3bf046cdcaec3 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Wed, 24 Jul 2019 11:29:49 +0100 Subject: [PATCH 14/22] Implement Audio speed up without pitch modification --- .../transcoder/demo/TranscoderActivity.java | 2 +- .../transcoder/engine/TranscoderEngine.java | 2 +- .../internal/MediaCodecBuffers.java | 4 +- .../transcoder/remix/AudioRemixer.java | 4 +- .../transcoder/remix/DownMixAudioRemixer.java | 17 +- .../remix/PassThroughAudioRemixer.java | 10 +- .../transcoder/remix/UpMixAudioRemixer.java | 17 +- .../time/SpeedTimeInterpolator.java | 2 +- .../transcode/BaseTrackTranscoder.java | 2 +- .../transcode/NoOpTrackTranscoder.java | 2 +- .../transcode/PassThroughTrackTranscoder.java | 2 +- .../transcoder/transcode/TrackTranscoder.java | 6 +- .../transcode/internal/AudioBuffer.java | 10 + .../transcode/internal/AudioConversions.java | 34 ++ .../transcode/internal/AudioEngine.java | 303 +++++++++--------- .../transcode/internal/AudioStretcher.java | 69 ++++ 16 files changed, 308 insertions(+), 178 deletions(-) create mode 100644 lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioBuffer.java create mode 100644 lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioConversions.java create mode 100644 lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioStretcher.java diff --git a/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java b/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java index dc1cdfd3..88d0a501 100644 --- a/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java +++ b/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java @@ -179,7 +179,7 @@ private void transcode() { case R.id.speed_2x: speed = 2F; break; default: speed = 1F; } - OutputStrategy audioStrategy = speed == 1F ? mTranscodeAudioStrategy : new RemoveTrackStrategy(); + OutputStrategy audioStrategy = mTranscodeAudioStrategy; // speed == 1F ? mTranscodeAudioStrategy : new RemoveTrackStrategy(); OutputStrategy videoStrategy = mTranscodeVideoStrategy; // Launch the transcoding operation. diff --git a/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderEngine.java b/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderEngine.java index c7797903..f2959eb5 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderEngine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderEngine.java @@ -259,7 +259,7 @@ private void runPipelines() throws InterruptedException { TrackTranscoder videoTranscoder = mTranscoders.get(TrackType.VIDEO); TrackTranscoder audioTranscoder = mTranscoders.get(TrackType.AUDIO); while (!(videoTranscoder.isFinished() && audioTranscoder.isFinished())) { - boolean stepped = videoTranscoder.stepPipeline() || audioTranscoder.stepPipeline(); + boolean stepped = videoTranscoder.transcode() || audioTranscoder.transcode(); loopCount++; if (mDurationUs > 0 && loopCount % PROGRESS_INTERVAL_STEPS == 0) { double videoProgress = getTranscoderProgress(videoTranscoder, mTracks.status(TrackType.VIDEO)); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/MediaCodecBuffers.java b/lib/src/main/java/com/otaliastudios/transcoder/internal/MediaCodecBuffers.java index b7315cc0..86d60ead 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/MediaCodecBuffers.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/MediaCodecBuffers.java @@ -36,7 +36,9 @@ public ByteBuffer getInputBuffer(final int index) { //noinspection ConstantConditions return mMediaCodec.getInputBuffer(index); } - return mInputBuffers[index]; + ByteBuffer result = mInputBuffers[index]; + result.clear(); + return result; } @NonNull diff --git a/lib/src/main/java/com/otaliastudios/transcoder/remix/AudioRemixer.java b/lib/src/main/java/com/otaliastudios/transcoder/remix/AudioRemixer.java index ca7767a7..e0a4bfff 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/remix/AudioRemixer.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/remix/AudioRemixer.java @@ -11,7 +11,9 @@ */ public interface AudioRemixer { - void remix(@NonNull final ShortBuffer inSBuff, @NonNull final ShortBuffer outSBuff); + void remix(@NonNull final ShortBuffer inputBuffer, @NonNull final ShortBuffer outputBuffer); + + int getRemixedSize(int inputSize); AudioRemixer DOWNMIX = new DownMixAudioRemixer(); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/remix/DownMixAudioRemixer.java b/lib/src/main/java/com/otaliastudios/transcoder/remix/DownMixAudioRemixer.java index 03bb4692..6385fdd8 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/remix/DownMixAudioRemixer.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/remix/DownMixAudioRemixer.java @@ -13,19 +13,19 @@ public class DownMixAudioRemixer implements AudioRemixer { private static final int UNSIGNED_SHORT_MAX = 65535; @Override - public void remix(@NonNull final ShortBuffer inSBuff, @NonNull final ShortBuffer outSBuff) { + public void remix(@NonNull final ShortBuffer inputBuffer, @NonNull final ShortBuffer outputBuffer) { // Down-mix stereo to mono // Viktor Toth's algorithm - // See: http://www.vttoth.com/CMS/index.php/technical-notes/68 // http://stackoverflow.com/a/25102339 - final int inRemaining = inSBuff.remaining() / 2; - final int outSpace = outSBuff.remaining(); + final int inRemaining = inputBuffer.remaining() / 2; + final int outSpace = outputBuffer.remaining(); final int samplesToBeProcessed = Math.min(inRemaining, outSpace); for (int i = 0; i < samplesToBeProcessed; ++i) { // Convert to unsigned - final int a = inSBuff.get() + SIGNED_SHORT_LIMIT; - final int b = inSBuff.get() + SIGNED_SHORT_LIMIT; + final int a = inputBuffer.get() + SIGNED_SHORT_LIMIT; + final int b = inputBuffer.get() + SIGNED_SHORT_LIMIT; int m; // Pick the equation if ((a < SIGNED_SHORT_LIMIT) || (b < SIGNED_SHORT_LIMIT)) { @@ -38,7 +38,12 @@ public void remix(@NonNull final ShortBuffer inSBuff, @NonNull final ShortBuffer } // Convert output back to signed short if (m == UNSIGNED_SHORT_MAX + 1) m = UNSIGNED_SHORT_MAX; - outSBuff.put((short) (m - SIGNED_SHORT_LIMIT)); + outputBuffer.put((short) (m - SIGNED_SHORT_LIMIT)); } } + + @Override + public int getRemixedSize(int inputSize) { + return inputSize / 2; + } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/remix/PassThroughAudioRemixer.java b/lib/src/main/java/com/otaliastudios/transcoder/remix/PassThroughAudioRemixer.java index 2d43c052..717d7390 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/remix/PassThroughAudioRemixer.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/remix/PassThroughAudioRemixer.java @@ -10,8 +10,12 @@ public class PassThroughAudioRemixer implements AudioRemixer { @Override - public void remix(@NonNull final ShortBuffer inSBuff, @NonNull final ShortBuffer outSBuff) { - // Passthrough - outSBuff.put(inSBuff); + public void remix(@NonNull final ShortBuffer inputBuffer, @NonNull final ShortBuffer outputBuffer) { + outputBuffer.put(inputBuffer); + } + + @Override + public int getRemixedSize(int inputSize) { + return inputSize; } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/remix/UpMixAudioRemixer.java b/lib/src/main/java/com/otaliastudios/transcoder/remix/UpMixAudioRemixer.java index 3baf6737..a0e27ff0 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/remix/UpMixAudioRemixer.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/remix/UpMixAudioRemixer.java @@ -10,16 +10,21 @@ public class UpMixAudioRemixer implements AudioRemixer { @Override - public void remix(@NonNull final ShortBuffer inSBuff, @NonNull final ShortBuffer outSBuff) { + public void remix(@NonNull final ShortBuffer inputBuffer, @NonNull final ShortBuffer outputBuffer) { // Up-mix mono to stereo - final int inRemaining = inSBuff.remaining(); - final int outSpace = outSBuff.remaining() / 2; + final int inRemaining = inputBuffer.remaining(); + final int outSpace = outputBuffer.remaining() / 2; final int samplesToBeProcessed = Math.min(inRemaining, outSpace); for (int i = 0; i < samplesToBeProcessed; ++i) { - final short inSample = inSBuff.get(); - outSBuff.put(inSample); - outSBuff.put(inSample); + final short inSample = inputBuffer.get(); + outputBuffer.put(inSample); + outputBuffer.put(inSample); } } + + @Override + public int getRemixedSize(int inputSize) { + return inputSize * 2; + } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/time/SpeedTimeInterpolator.java b/lib/src/main/java/com/otaliastudios/transcoder/time/SpeedTimeInterpolator.java index 7daebde5..3fa485c4 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/time/SpeedTimeInterpolator.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/time/SpeedTimeInterpolator.java @@ -56,7 +56,7 @@ public long interpolate(@NonNull TrackType type, long time) { data.lastRealTime = time; data.lastCorrectedTime += correctedDelta; } - Log.e("SpeedTimeInterpolator", "Input time: " + time + ", output time: " + data.lastCorrectedTime); + Log.e("SpeedTimeInterpolator", "Track:" + type + " inputTime:" + time + " outputTime:" + data.lastCorrectedTime); return data.lastCorrectedTime; } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java index 3d2f21ef..535c7e48 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java @@ -163,7 +163,7 @@ public void release() { } @Override - public final boolean stepPipeline() { + public final boolean transcode() { boolean busy = false; int status; while (drainEncoder(0) != DRAIN_STATE_NONE) busy = true; diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/NoOpTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/NoOpTrackTranscoder.java index eda1aa91..60eecd3b 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/NoOpTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/NoOpTrackTranscoder.java @@ -29,7 +29,7 @@ public void setUp(@NonNull MediaFormat desiredOutputFormat) { } @Override - public boolean stepPipeline() { + public boolean transcode() { return false; } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/PassThroughTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/PassThroughTrackTranscoder.java index 54b5b6bc..1c88f8d3 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/PassThroughTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/PassThroughTrackTranscoder.java @@ -68,7 +68,7 @@ public void setUp(@NonNull MediaFormat desiredOutputFormat) { } @SuppressLint("Assert") @Override - public boolean stepPipeline() { + public boolean transcode() { if (mIsEOS) return false; if (!mOutputFormatSet) { mMuxer.setOutputFormat(mTrackType, mOutputFormat); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/TrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/TrackTranscoder.java index 760f51df..7be88bd4 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/TrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/TrackTranscoder.java @@ -25,18 +25,18 @@ public interface TrackTranscoder { void setUp(@NonNull MediaFormat desiredOutputFormat); /** - * Step pipeline if output is available in any step of it. + * Perform transcoding if output is available in any step of it. * It assumes muxer has been started, so you should call muxer.start() first. * * @return true if data moved in pipeline. */ - boolean stepPipeline(); + boolean transcode(); /** * Get presentation time of last sample taken from encoder. * This presentation time should not be affected by {@link com.otaliastudios.transcoder.time.TimeInterpolator}s. * - * @return Presentation time in micro-second. Return value is undefined if finished writing. + * @return Presentation time in microseconds. Return value is undefined if finished writing. */ long getLastPresentationTime(); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioBuffer.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioBuffer.java new file mode 100644 index 00000000..0f8150b8 --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioBuffer.java @@ -0,0 +1,10 @@ +package com.otaliastudios.transcoder.transcode.internal; + +import java.nio.ShortBuffer; + +class AudioBuffer { + int decoderBufferIndex = -1; + long decoderTimestampUs = 0; + ShortBuffer decoderData = null; + boolean isEndOfStream = false; +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioConversions.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioConversions.java new file mode 100644 index 00000000..be0df1b8 --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioConversions.java @@ -0,0 +1,34 @@ +package com.otaliastudios.transcoder.transcode.internal; + +class AudioConversions { + + private static final int BYTES_PER_SAMPLE_PER_CHANNEL = 2; // Assuming 16bit audio, so 2 + private static final long MICROSECONDS_PER_SECOND = 1000000L; + private static final int BYTES_PER_SHORT = 2; + + @SuppressWarnings("WeakerAccess") + static long bytesToUs( + int bytes /* bytes */, + int sampleRate /* samples/sec */, + int channels /* channel */ + ) { + int byteRatePerChannel = sampleRate * BYTES_PER_SAMPLE_PER_CHANNEL; // bytes/sec/channel + int byteRate = byteRatePerChannel * channels; // bytes/sec + return MICROSECONDS_PER_SECOND * bytes / byteRate; // usec + } + + @SuppressWarnings("WeakerAccess") + static int usToBytes(long us, int sampleRate, int channels) { + int byteRatePerChannel = sampleRate * BYTES_PER_SAMPLE_PER_CHANNEL; + int byteRate = byteRatePerChannel * channels; + return (int) Math.ceil((double) us * byteRate / MICROSECONDS_PER_SECOND); + } + + static long shortsToUs(int shorts, int sampleRate, int channels) { + return bytesToUs(shorts * BYTES_PER_SHORT, sampleRate, channels); + } + + static int usToShorts(long us, int sampleRate, int channels) { + return usToBytes(us, sampleRate, channels) / BYTES_PER_SHORT; + } +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java index 1485f790..c2a36576 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java @@ -6,10 +6,12 @@ import androidx.annotation.NonNull; import com.otaliastudios.transcoder.engine.TrackType; +import com.otaliastudios.transcoder.internal.Logger; import com.otaliastudios.transcoder.internal.MediaCodecBuffers; import com.otaliastudios.transcoder.remix.AudioRemixer; import com.otaliastudios.transcoder.time.TimeInterpolator; +import java.nio.Buffer; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.ShortBuffer; @@ -25,45 +27,23 @@ */ public class AudioEngine { - private static class AudioBuffer { - int bufferIndex; - long presentationTimeUs; - ShortBuffer data; - boolean endOfStream; - } - - private static final int BYTES_PER_SAMPLE_PER_CHANNEL = 2; // Assuming 16bit audio, so 2 private static final int BYTES_PER_SHORT = 2; - private static final long MICROSECONDS_PER_SECOND = 1000000L; - private static long bytesToUs( - int bytes /* [bytes] */, - int sampleRate /* [samples/sec] */, - int channels /* [channel] */ - ) { - int byteRatePerChannel = sampleRate * BYTES_PER_SAMPLE_PER_CHANNEL; // [bytes/sec/channel] - int byteRate = byteRatePerChannel * channels; // [bytes/sec] - return MICROSECONDS_PER_SECOND * bytes / byteRate; // [usec] - } - - private static long shortsToUs( - int shorts, - int sampleRate, - int channels) { - return bytesToUs(shorts * BYTES_PER_SHORT, sampleRate, channels); - } + private static final String TAG = AudioEngine.class.getSimpleName(); + private static final Logger LOG = new Logger(TAG); private final Queue mEmptyBuffers = new ArrayDeque<>(); - private final Queue mFilledBuffers = new ArrayDeque<>(); + private final Queue mPendingBuffers = new ArrayDeque<>(); private final MediaCodec mDecoder; private final MediaCodec mEncoder; - private final AudioBuffer mOverflowBuffer = new AudioBuffer(); private final int mSampleRate; - private final int mInputChannelCount; - private final int mOutputChannelCount; + private final int mDecoderChannels; + private final int mEncoderChannels; private final AudioRemixer mRemixer; private final TimeInterpolator mTimeInterpolator; - + private long mLastDecoderUs = Long.MIN_VALUE; + private long mLastEncoderUs = Long.MIN_VALUE; + private ShortBuffer mTempBuffer; /** * The AudioEngine should be created when we know the actual decoded format, @@ -92,25 +72,31 @@ public AudioEngine(@NonNull MediaCodec decoder, mSampleRate = inputSampleRate; // Check channel count. - mOutputChannelCount = encoderOutputFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT); - mInputChannelCount = decoderOutputFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT); - if (mOutputChannelCount != 1 && mOutputChannelCount != 2) { - throw new UnsupportedOperationException("Output channel count (" + mOutputChannelCount + ") not supported."); + mEncoderChannels = encoderOutputFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT); + mDecoderChannels = decoderOutputFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT); + if (mEncoderChannels != 1 && mEncoderChannels != 2) { + throw new UnsupportedOperationException("Output channel count (" + mEncoderChannels + ") not supported."); } - if (mInputChannelCount != 1 && mInputChannelCount != 2) { - throw new UnsupportedOperationException("Input channel count (" + mInputChannelCount + ") not supported."); + if (mDecoderChannels != 1 && mDecoderChannels != 2) { + throw new UnsupportedOperationException("Input channel count (" + mDecoderChannels + ") not supported."); } // Create remixer. - if (mInputChannelCount > mOutputChannelCount) { + if (mDecoderChannels > mEncoderChannels) { mRemixer = AudioRemixer.DOWNMIX; - } else if (mInputChannelCount < mOutputChannelCount) { + } else if (mDecoderChannels < mEncoderChannels) { mRemixer = AudioRemixer.UPMIX; } else { mRemixer = AudioRemixer.PASSTHROUGH; } - mOverflowBuffer.presentationTimeUs = 0; + } + /** + * Returns true if we have raw buffers to be processed. + * @return true if we have + */ + private boolean hasPendingBuffers() { + return !mPendingBuffers.isEmpty(); } /** @@ -129,25 +115,16 @@ public void drainDecoder(final int bufferIndex, final boolean endOfStream) { if (mRemixer == null) throw new RuntimeException("Buffer received before format!"); AudioBuffer buffer = mEmptyBuffers.poll(); - if (buffer == null) { - buffer = new AudioBuffer(); - } - buffer.bufferIndex = bufferIndex; - buffer.presentationTimeUs = endOfStream ? 0 : presentationTimeUs; - buffer.data = endOfStream ? null : bufferData.asShortBuffer(); - buffer.endOfStream = endOfStream; - - if (mOverflowBuffer.data == null) { - mOverflowBuffer.data = ByteBuffer.allocateDirect(bufferData.capacity()) - .order(ByteOrder.nativeOrder()) - .asShortBuffer(); - mOverflowBuffer.data.clear().flip(); - } - mFilledBuffers.add(buffer); + if (buffer == null) buffer = new AudioBuffer(); + buffer.decoderBufferIndex = bufferIndex; + buffer.decoderTimestampUs = endOfStream ? 0 : presentationTimeUs; + buffer.decoderData = endOfStream ? null : bufferData.asShortBuffer(); + buffer.isEndOfStream = endOfStream; + mPendingBuffers.add(buffer); } /** - * Feeds the encoder, which in our case means processing a filled buffer from {@link #mFilledBuffers}, + * Feeds the encoder, which in our case means processing a filled buffer, * then releasing it and adding it back to {@link #mEmptyBuffers}. * * @param encoderBuffers the encoder buffers @@ -155,33 +132,21 @@ public void drainDecoder(final int bufferIndex, * @return true if we want to keep working */ public boolean feedEncoder(@NonNull MediaCodecBuffers encoderBuffers, long timeoutUs) { - final boolean hasOverflow = hasOverflow(); - final boolean hasBuffers = hasBuffers(); - if (!hasBuffers && !hasOverflow) return false; + if (!hasPendingBuffers()) return false; - // Prepare to encode - see if encoder has buffers. + // First of all, see if encoder has buffers that we can write into. + // If we don't have an output buffer, there's nothing we can do. final int encoderBufferIndex = mEncoder.dequeueInputBuffer(timeoutUs); if (encoderBufferIndex < 0) return false; ShortBuffer encoderBuffer = encoderBuffers.getInputBuffer(encoderBufferIndex).asShortBuffer(); + encoderBuffer.clear(); - // If we have overflow data, process that first. - if (hasOverflow) { - long presentationTimeUs = drainOverflow(encoderBuffer); - mEncoder.queueInputBuffer(encoderBufferIndex, - 0, - encoderBuffer.position() * BYTES_PER_SHORT, - presentationTimeUs, - 0); - return true; - } - - // At this point buffer is not null, because we checked hasBuffers() - // and we don't have overflow (if we had, we got out). - final AudioBuffer decoderBuffer = mFilledBuffers.poll(); + // Get the latest raw buffer to be processed. + AudioBuffer buffer = mPendingBuffers.peek(); // When endOfStream, just signal EOS and return false. //noinspection ConstantConditions - if (decoderBuffer.endOfStream) { + if (buffer.isEndOfStream) { mEncoder.queueInputBuffer(encoderBufferIndex, 0, 0, @@ -190,94 +155,128 @@ public boolean feedEncoder(@NonNull MediaCodecBuffers encoderBuffers, long timeo return false; } - // If not, process data and return true. - long inputPresentationTime = decoderBuffer.presentationTimeUs; - long outputPresentationTime = process(decoderBuffer.data, encoderBuffer, inputPresentationTime); - mEncoder.queueInputBuffer(encoderBufferIndex, - 0, - encoderBuffer.position() * BYTES_PER_SHORT, - outputPresentationTime, - 0); - mDecoder.releaseOutputBuffer(decoderBuffer.bufferIndex, false); - mEmptyBuffers.add(decoderBuffer); - return true; + // Process the buffer. + boolean overflows = process(buffer, encoderBuffer, encoderBufferIndex); + if (overflows) { + // If this buffer does overflow, we will keep it in the queue and do + // not release. It will be used at the next cycle (presumably soon, + // since we return true). + return true; + } else { + // If this buffer does not overflow, it can be removed from our queue, + // re-added to empty, and also released from the decoder buffers. + mPendingBuffers.remove(); + mEmptyBuffers.add(buffer); + mDecoder.releaseOutputBuffer(buffer.decoderBufferIndex, false); + return true; + } } /** - * Returns true if we have overflow data to be drained and set to the encoder. - * The overflow data is already processed. + * Processes a pending buffer. * - * @return true if data - */ - private boolean hasOverflow() { - return mOverflowBuffer.data != null && mOverflowBuffer.data.hasRemaining(); - } - - /** - * Returns true if we have filled buffers to be processed. - * @return true if we have - */ - private boolean hasBuffers() { - return !mFilledBuffers.isEmpty(); - } - - /** - * Drains the overflow data into the given {@link ShortBuffer}.The overflow data is - * already processed, it must just be copied into the given buffer. + * We have an output buffer of restricted size, and our input size can be already bigger + * or grow after a {@link TimeInterpolator} operation or a {@link AudioRemixer} one. + * + * For this reason, we instead start from the output size and compute the input size that + * would generate an output of that size. We then select this part of the input and only + * process. + * + * If input is restricted, this means that the input buffer must be processed again. + * We will return true in this case. At the next cycle, the input buffer should be identical + * to the previous, but: + * + * - {@link Buffer#position()} will be increased to exclude the already processed values + * - {@link AudioBuffer#decoderTimestampUs} will be increased to include the already processed values * - * @param outBuffer output buffer - * @return the frame timestamp + * So everything should work as expected for repeated cycles. + * + * Before returning, this function should release the encoder buffer using + * {@link MediaCodec#queueInputBuffer(int, int, int, long, int)}. + * + * @param buffer coming from decoder. Has valid limit and position + * @param encoderBuffer coming from encoder. At this point this is in a cleared state + * @param encoderBufferIndex the index of encoderBuffer so we can release it */ - private long drainOverflow(@NonNull final ShortBuffer outBuffer) { - final ShortBuffer overflowBuffer = mOverflowBuffer.data; - final int overflowLimit = overflowBuffer.limit(); - final int overflowSize = overflowBuffer.remaining(); - final long beginPresentationTimeUs = mOverflowBuffer.presentationTimeUs + - shortsToUs(overflowBuffer.position(), - mSampleRate, - mOutputChannelCount); - outBuffer.clear(); - overflowBuffer.limit(outBuffer.capacity()); // Limit overflowBuffer to outBuffer's capacity - outBuffer.put(overflowBuffer); // Load overflowBuffer onto outBuffer - if (overflowSize >= outBuffer.capacity()) { - overflowBuffer.clear().limit(0); // Overflow fully consumed - Reset - } else { - overflowBuffer.limit(overflowLimit); // Only partially consumed - Keep position & restore previous limit + private boolean process(@NonNull AudioBuffer buffer, @NonNull ShortBuffer encoderBuffer, int encoderBufferIndex) { + // Only process the amount of data that can fill in the encoderBuffer. + final int outputSize = encoderBuffer.remaining(); + final int inputSize = buffer.decoderData.remaining(); + int processedInputSize = inputSize; + + // 1. Perform TimeInterpolator computation + long encoderUs = mTimeInterpolator.interpolate(TrackType.AUDIO, buffer.decoderTimestampUs); + if (mLastDecoderUs == Long.MIN_VALUE) { + mLastDecoderUs = buffer.decoderTimestampUs; + mLastEncoderUs = encoderUs; + } + long decoderDeltaUs = buffer.decoderTimestampUs - mLastDecoderUs; + long encoderDeltaUs = encoderUs - mLastEncoderUs; + mLastDecoderUs = buffer.decoderTimestampUs; + mLastEncoderUs = encoderUs; + long stretchUs = encoderDeltaUs - decoderDeltaUs; // microseconds that the TimeInterpolator adds (or removes). + int stretchShorts = AudioConversions.usToShorts(stretchUs, mSampleRate, mDecoderChannels); + LOG.i("process - time stretching - decoderDeltaUs:" + decoderDeltaUs + + " encoderDeltaUs:" + encoderDeltaUs + + " stretchUs:" + stretchUs + + " stretchShorts:" + stretchShorts); + processedInputSize += stretchShorts; + + // 2. Ask remixers how much space they need for the given input + processedInputSize = mRemixer.getRemixedSize(processedInputSize); + + // 3. Compare processedInputSize and outputSize. If processedInputSize > outputSize, we overflow. + // In this case, isolate the valid data. + boolean overflow = processedInputSize > outputSize; + int overflowReduction = 0; + if (overflow) { + // Compute the input size that matches this output size. + double ratio = (double) processedInputSize / inputSize; // > 1 + overflowReduction = inputSize - (int) Math.floor((double) outputSize / ratio); + LOG.w("process - overflowing! Reduction:" + overflowReduction); + buffer.decoderData.limit(buffer.decoderData.limit() - overflowReduction); + } + final int finalInputSize = buffer.decoderData.remaining(); + LOG.i("process - inputSize:" + inputSize + " processedInputSize:" + processedInputSize + " outputSize:" + outputSize + " finalInputSize:" + finalInputSize); + + // 4. Do the stretching. We need a bridge buffer for its output. + ensureTempBuffer(finalInputSize + stretchShorts); + AudioStretcher.CUT_OR_INSERT.stretch(buffer.decoderData, mTempBuffer, mDecoderChannels); + + // 5. Do the actual remixing. + mTempBuffer.position(0); + mRemixer.remix(mTempBuffer, encoderBuffer); + + // 6. Add the bytes we have processed to the decoderTimestampUs, and restore the limit. + // We need an updated timestamp for the next cycle, since we will cycle on the same input + // buffer that has overflown. + if (overflow) { + buffer.decoderTimestampUs += AudioConversions.shortsToUs(finalInputSize, mSampleRate, mDecoderChannels); + buffer.decoderData.limit(buffer.decoderData.limit() + overflowReduction); } - return beginPresentationTimeUs; - } - - private long process(@NonNull final ShortBuffer inputBuffer, - @NonNull final ShortBuffer outputBuffer, - long inputPresentationTimeUs) { - long outputPresentationTime = mTimeInterpolator.interpolate(TrackType.AUDIO, inputPresentationTimeUs); - - // Reset position to 0 and set limit to capacity (Since MediaCodec doesn't do that for us) - outputBuffer.clear(); - inputBuffer.clear(); - - - if (inputBuffer.remaining() <= outputBuffer.remaining()) { - // Safe case. Just remix. - mRemixer.remix(inputBuffer, outputBuffer); - } else { - // Overflow! - // First remix all we can. - inputBuffer.limit(outputBuffer.capacity()); - mRemixer.remix(inputBuffer, outputBuffer); - inputBuffer.limit(inputBuffer.capacity()); + // 5. Write the buffer. + // This is the encoder buffer: we have likely written it all, but let's use + // encoderBuffer.position() to know how much anyway. + mEncoder.queueInputBuffer(encoderBufferIndex, + 0, + encoderBuffer.position() * BYTES_PER_SHORT, + encoderUs, + 0 + ); - // TODO check the time logic below if interpolator changes time. - // Then remix the rest into mOverflowBuffer. - // NOTE: We should only reach this point when overflow buffer is empty - long consumedDurationUs = shortsToUs(inputBuffer.position(), mSampleRate, mInputChannelCount); - mRemixer.remix(inputBuffer, mOverflowBuffer.data); + return overflow; + } - // Flip the overflow buffer and mark the presentation time. - mOverflowBuffer.data.flip(); - mOverflowBuffer.presentationTimeUs = outputPresentationTime + consumedDurationUs; + private void ensureTempBuffer(int desiredSize) { + LOG.w("ensureTempBuffer - desiredSize:" + desiredSize); + if (mTempBuffer == null || mTempBuffer.capacity() < desiredSize) { + LOG.w("ensureTempBuffer - creating new buffer."); + mTempBuffer = ByteBuffer.allocateDirect(desiredSize * BYTES_PER_SHORT) + .order(ByteOrder.nativeOrder()) + .asShortBuffer(); } - return outputPresentationTime; + mTempBuffer.clear(); + mTempBuffer.limit(desiredSize); } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioStretcher.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioStretcher.java new file mode 100644 index 00000000..dfd5e3ce --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioStretcher.java @@ -0,0 +1,69 @@ +package com.otaliastudios.transcoder.transcode.internal; + +import androidx.annotation.NonNull; + +import java.nio.Buffer; +import java.nio.ShortBuffer; + +interface AudioStretcher { + + /** + * Stretches the input into the output, based on the {@link Buffer#remaining()} value of both. + * At the end of this method, the {@link Buffer#position()} of both should be equal to their + * respective {@link Buffer#limit()}. + * + * And of course, both {@link Buffer#limit()}s should remain unchanged. + * + * @param input input buffer + * @param output output buffer + * @param channels audio channels + */ + void stretch(@NonNull ShortBuffer input, @NonNull ShortBuffer output, int channels); + + AudioStretcher PASSTHROUGH = new AudioStretcher() { + @Override + public void stretch(@NonNull ShortBuffer input, @NonNull ShortBuffer output, int channels) { + if (input.remaining() > output.remaining()) { + throw new IllegalArgumentException("Illegal use of AudioStretcher.PASSTHROUGH"); + } + output.put(input); + } + }; + + AudioStretcher CUT = new AudioStretcher() { + @Override + public void stretch(@NonNull ShortBuffer input, @NonNull ShortBuffer output, int channels) { + if (input.remaining() < output.remaining()) { + throw new IllegalArgumentException("Illegal use of AudioStretcher.CUT"); + } + int exceeding = input.remaining() - output.remaining(); + input.limit(input.limit() - exceeding); // Make remaining() the same for both + output.put(input); // Safely bulk-put + input.limit(input.limit() + exceeding); // Restore + input.position(input.limit()); // Make as if we have read it all + } + }; + + AudioStretcher INSERT = new AudioStretcher() { + @Override + public void stretch(@NonNull ShortBuffer input, @NonNull ShortBuffer output, int channels) { + if (input.remaining() >= output.remaining()) { + throw new IllegalArgumentException("Illegal use of AudioStretcher.INSERT"); + } + throw new RuntimeException("AudioStretcher.INSERT not implemented yet."); + } + }; + + AudioStretcher CUT_OR_INSERT = new AudioStretcher() { + @Override + public void stretch(@NonNull ShortBuffer input, @NonNull ShortBuffer output, int channels) { + if (input.remaining() < output.remaining()) { + INSERT.stretch(input, output, channels); + } else if (input.remaining() > output.remaining()) { + CUT.stretch(input, output, channels); + } else { + PASSTHROUGH.stretch(input, output, channels); + } + } + }; +} From 402a553ca7540fa1a61b8cd7d6f9bad4e1ec9d8a Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Wed, 24 Jul 2019 12:01:58 +0100 Subject: [PATCH 15/22] Implement Audio speed down --- .../time/SpeedTimeInterpolator.java | 6 ++- .../transcode/internal/AudioStretcher.java | 37 ++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/time/SpeedTimeInterpolator.java b/lib/src/main/java/com/otaliastudios/transcoder/time/SpeedTimeInterpolator.java index 3fa485c4..d5853195 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/time/SpeedTimeInterpolator.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/time/SpeedTimeInterpolator.java @@ -5,6 +5,7 @@ import androidx.annotation.NonNull; import com.otaliastudios.transcoder.engine.TrackType; +import com.otaliastudios.transcoder.internal.Logger; import java.util.HashMap; import java.util.Map; @@ -16,6 +17,9 @@ */ public class SpeedTimeInterpolator implements TimeInterpolator { + private final static String TAG = SpeedTimeInterpolator.class.getSimpleName(); + private final static Logger LOG = new Logger(TAG); + private double mFactor; private final Map mTrackData = new HashMap<>(); @@ -56,7 +60,7 @@ public long interpolate(@NonNull TrackType type, long time) { data.lastRealTime = time; data.lastCorrectedTime += correctedDelta; } - Log.e("SpeedTimeInterpolator", "Track:" + type + " inputTime:" + time + " outputTime:" + data.lastCorrectedTime); + LOG.i("Track:" + type + " inputTime:" + time + " outputTime:" + data.lastCorrectedTime); return data.lastCorrectedTime; } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioStretcher.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioStretcher.java index dfd5e3ce..af113db9 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioStretcher.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioStretcher.java @@ -4,6 +4,7 @@ import java.nio.Buffer; import java.nio.ShortBuffer; +import java.util.Random; interface AudioStretcher { @@ -45,12 +46,46 @@ public void stretch(@NonNull ShortBuffer input, @NonNull ShortBuffer output, int }; AudioStretcher INSERT = new AudioStretcher() { + + private final Random NOISE = new Random(); + + private short noise() { + return (short) NOISE.nextInt(1000); + } + + private float ratio(int remaining, int all) { + return (float) remaining / all; + } + @Override public void stretch(@NonNull ShortBuffer input, @NonNull ShortBuffer output, int channels) { if (input.remaining() >= output.remaining()) { throw new IllegalArgumentException("Illegal use of AudioStretcher.INSERT"); } - throw new RuntimeException("AudioStretcher.INSERT not implemented yet."); + if (channels != 1 && channels != 2) { + throw new IllegalArgumentException("Illegal use of AudioStretcher.INSERT. Channels:" + channels); + } + final int inputSamples = input.remaining() / channels; + final int fakeSamples = (int) Math.floor((double) (output.remaining() - input.remaining()) / channels); + int remainingInputSamples = inputSamples; + int remainingFakeSamples = fakeSamples; + float remainingInputSamplesRatio = ratio(remainingInputSamples, inputSamples); + float remainingFakeSamplesRatio = ratio(remainingFakeSamples, fakeSamples); + while (remainingInputSamples > 0 && remainingFakeSamples > 0) { + // Will this be an input sample or a fake sample? + // Choose the one with the bigger ratio. + if (remainingInputSamplesRatio >= remainingFakeSamplesRatio) { + output.put(input.get()); + if (channels == 2) output.put(input.get()); + remainingInputSamples--; + remainingInputSamplesRatio = ratio(remainingInputSamples, inputSamples); + } else { + output.put(noise()); + if (channels == 2) output.put(noise()); + remainingFakeSamples--; + remainingFakeSamplesRatio = ratio(remainingFakeSamples, inputSamples); + } + } } }; From acbd631f3f28fc407772f13b305d374fdcaf3544 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Wed, 24 Jul 2019 12:40:23 +0100 Subject: [PATCH 16/22] Expose AUdioStretcher, provide base implementations, add setAudioStretcher --- README.md | 22 +++- .../transcoder/demo/TranscoderActivity.java | 6 +- .../transcoder/TranscoderOptions.java | 20 ++++ .../transcoder/engine/TranscoderEngine.java | 2 +- .../transcoder/remix/AudioRemixer.java | 15 +++ .../transcoder/stretch/AudioStretcher.java | 50 +++++++++ .../transcoder/stretch/CutAudioStretcher.java | 24 ++++ .../stretch/DefaultAudioStretcher.java | 23 ++++ .../stretch/InsertAudioStretcher.java | 55 +++++++++ .../stretch/PassThroughAudioStretcher.java | 19 ++++ .../transcode/AudioTrackTranscoder.java | 9 +- .../transcode/internal/AudioEngine.java | 19 +++- .../transcode/internal/AudioStretcher.java | 104 ------------------ 13 files changed, 247 insertions(+), 121 deletions(-) create mode 100644 lib/src/main/java/com/otaliastudios/transcoder/stretch/AudioStretcher.java create mode 100644 lib/src/main/java/com/otaliastudios/transcoder/stretch/CutAudioStretcher.java create mode 100644 lib/src/main/java/com/otaliastudios/transcoder/stretch/DefaultAudioStretcher.java create mode 100644 lib/src/main/java/com/otaliastudios/transcoder/stretch/InsertAudioStretcher.java create mode 100644 lib/src/main/java/com/otaliastudios/transcoder/stretch/PassThroughAudioStretcher.java delete mode 100644 lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioStretcher.java diff --git a/README.md b/README.md index dfddd313..e8c75f97 100644 --- a/README.md +++ b/README.md @@ -319,8 +319,6 @@ Transcoder.into(filePath) #### Time interpolation -**Note: when time is changed, you must make sure to remove the audio track or the process breaks.** - We offer APIs to change the timestamp of each video and audio frame. You can pass a `TimeInterpolator` to the transcoder builder to be able to receive the frame timestamp as input, and return a new one as output. @@ -347,8 +345,6 @@ the transcoding operation fail. #### Video speed -**Note: when time is changed, you must make sure to remove the audio track or the process breaks.** - We also offer a special time interpolator called `SpeedTimeInterpolator` that accepts a `float` parameter and will modify the video speed. @@ -367,6 +363,24 @@ Transcoder.into(filePath) // ... ``` +#### Audio stretching + +When a time interpolator alters the frames and samples timestamps, you can either remove audio or +stretch the audio samples to the new length. This is done through the `AudioStretcher` interface: + +```java +Transcoder.into(filePath) + .setAudioStretcher(audioStretcher) + // ... +``` + +The default audio stretcher, `DefaultAudioStretcher`, will: + +- When we need to shrink a group of samples, cut the last ones +- When we need to stretch a group of samples, insert noise samples in between + +Please take a look at the implementation and read class documentation. + ## Compatibility As stated pretty much everywhere, **not all codecs/devices/manufacturers support all sizes/options**. diff --git a/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java b/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java index 88d0a501..8a76210a 100644 --- a/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java +++ b/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java @@ -179,8 +179,6 @@ private void transcode() { case R.id.speed_2x: speed = 2F; break; default: speed = 1F; } - OutputStrategy audioStrategy = mTranscodeAudioStrategy; // speed == 1F ? mTranscodeAudioStrategy : new RemoveTrackStrategy(); - OutputStrategy videoStrategy = mTranscodeVideoStrategy; // Launch the transcoding operation. mTranscodeStartTime = SystemClock.uptimeMillis(); @@ -188,8 +186,8 @@ private void transcode() { mTranscodeFuture = Transcoder.into(mTranscodeOutputFile.getAbsolutePath()) .setDataSource(this, mTranscodeInputUri) .setListener(this) - .setAudioOutputStrategy(audioStrategy) - .setVideoOutputStrategy(videoStrategy) + .setAudioOutputStrategy(mTranscodeAudioStrategy) + .setVideoOutputStrategy(mTranscodeVideoStrategy) .setRotation(rotation) .setSpeed(speed) .transcode(); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java b/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java index 32f964b4..324df3d7 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java @@ -12,6 +12,8 @@ import com.otaliastudios.transcoder.strategy.DefaultAudioStrategy; import com.otaliastudios.transcoder.strategy.DefaultVideoStrategies; import com.otaliastudios.transcoder.strategy.OutputStrategy; +import com.otaliastudios.transcoder.stretch.AudioStretcher; +import com.otaliastudios.transcoder.stretch.DefaultAudioStretcher; import com.otaliastudios.transcoder.time.DefaultTimeInterpolator; import com.otaliastudios.transcoder.time.SpeedTimeInterpolator; import com.otaliastudios.transcoder.time.TimeInterpolator; @@ -38,6 +40,7 @@ private TranscoderOptions() {} private Validator validator; private int rotation; private TimeInterpolator timeInterpolator; + private AudioStretcher audioStretcher; TranscoderListener listener; Handler listenerHandler; @@ -77,6 +80,11 @@ public TimeInterpolator getTimeInterpolator() { return timeInterpolator; } + @NonNull + public AudioStretcher getAudioStretcher() { + return audioStretcher; + } + public static class Builder { private String outPath; private DataSource dataSource; @@ -87,6 +95,7 @@ public static class Builder { private Validator validator; private int rotation; private TimeInterpolator timeInterpolator; + private AudioStretcher audioStretcher; Builder(@NonNull String outPath) { this.outPath = outPath; @@ -227,6 +236,13 @@ public Builder setSpeed(float speedFactor) { return setTimeInterpolator(new SpeedTimeInterpolator(speedFactor)); } + @NonNull + @SuppressWarnings("unused") + public Builder setAudioStretcher(@NonNull AudioStretcher audioStretcher) { + this.audioStretcher = audioStretcher; + return this; + } + @NonNull public TranscoderOptions build() { if (listener == null) { @@ -258,6 +274,9 @@ public TranscoderOptions build() { if (timeInterpolator == null) { timeInterpolator = new DefaultTimeInterpolator(); } + if (audioStretcher == null) { + audioStretcher = new DefaultAudioStretcher(); + } TranscoderOptions options = new TranscoderOptions(); options.listener = listener; options.dataSource = dataSource; @@ -268,6 +287,7 @@ public TranscoderOptions build() { options.validator = validator; options.rotation = rotation; options.timeInterpolator = timeInterpolator; + options.audioStretcher = audioStretcher; return options; } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderEngine.java b/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderEngine.java index f2959eb5..d4cdaa95 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderEngine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderEngine.java @@ -202,7 +202,7 @@ private void setUpTrackTranscoder(@NonNull TranscoderOptions options, } else { switch (type) { case VIDEO: transcoder = new VideoTrackTranscoder(mExtractor, muxer, index, options.getTimeInterpolator()); break; - case AUDIO: transcoder = new AudioTrackTranscoder(mExtractor, muxer, index, options.getTimeInterpolator()); break; + case AUDIO: transcoder = new AudioTrackTranscoder(mExtractor, muxer, index, options.getTimeInterpolator(), options.getAudioStretcher()); break; default: throw new RuntimeException("Unknown type: " + type); } status = TrackStatus.COMPRESSING; diff --git a/lib/src/main/java/com/otaliastudios/transcoder/remix/AudioRemixer.java b/lib/src/main/java/com/otaliastudios/transcoder/remix/AudioRemixer.java index e0a4bfff..ce80f45e 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/remix/AudioRemixer.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/remix/AudioRemixer.java @@ -2,6 +2,7 @@ import androidx.annotation.NonNull; +import java.nio.Buffer; import java.nio.ShortBuffer; /** @@ -11,8 +12,22 @@ */ public interface AudioRemixer { + /** + * Remixes input audio from input buffer into the output buffer. + * The output buffer is guaranteed to have a {@link Buffer#remaining()} size that is + * consistent with {@link #getRemixedSize(int)}. + * + * @param inputBuffer the input buffer + * @param outputBuffer the output buffer + */ void remix(@NonNull final ShortBuffer inputBuffer, @NonNull final ShortBuffer outputBuffer); + /** + * Returns the output size (in shorts) needed to process an input buffer of the + * given size (in shorts). + * @param inputSize input size in shorts + * @return output size in shorts + */ int getRemixedSize(int inputSize); AudioRemixer DOWNMIX = new DownMixAudioRemixer(); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/stretch/AudioStretcher.java b/lib/src/main/java/com/otaliastudios/transcoder/stretch/AudioStretcher.java new file mode 100644 index 00000000..e35c4f26 --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/stretch/AudioStretcher.java @@ -0,0 +1,50 @@ +package com.otaliastudios.transcoder.stretch; + +import androidx.annotation.NonNull; + +import java.nio.Buffer; +import java.nio.ShortBuffer; +import java.util.Random; + +/** + * An AudioStretcher will change audio samples duration, in response to a + * {@link com.otaliastudios.transcoder.time.TimeInterpolator} that altered the sample timestamp. + * + * This can mean either shrink the sample (in case of video speed up) or elongate it (in case of + * video slow down) so that it matches the output size. + */ +public interface AudioStretcher { + + /** + * Stretches the input into the output, based on the {@link Buffer#remaining()} value of both. + * At the end of this method, the {@link Buffer#position()} of both should be equal to their + * respective {@link Buffer#limit()}. + * + * And of course, both {@link Buffer#limit()}s should remain unchanged. + * + * @param input input buffer + * @param output output buffer + * @param channels audio channels + */ + void stretch(@NonNull ShortBuffer input, @NonNull ShortBuffer output, int channels); + + AudioStretcher PASSTHROUGH = new PassThroughAudioStretcher(); + + AudioStretcher CUT = new CutAudioStretcher(); + + AudioStretcher INSERT = new InsertAudioStretcher(); + + + AudioStretcher CUT_OR_INSERT = new AudioStretcher() { + @Override + public void stretch(@NonNull ShortBuffer input, @NonNull ShortBuffer output, int channels) { + if (input.remaining() < output.remaining()) { + INSERT.stretch(input, output, channels); + } else if (input.remaining() > output.remaining()) { + CUT.stretch(input, output, channels); + } else { + PASSTHROUGH.stretch(input, output, channels); + } + } + }; +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/stretch/CutAudioStretcher.java b/lib/src/main/java/com/otaliastudios/transcoder/stretch/CutAudioStretcher.java new file mode 100644 index 00000000..c2ea24f8 --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/stretch/CutAudioStretcher.java @@ -0,0 +1,24 @@ +package com.otaliastudios.transcoder.stretch; + +import androidx.annotation.NonNull; + +import java.nio.ShortBuffer; + +/** + * A {@link AudioStretcher} meant to be used when output size is smaller than the input. + * Cutting the latest samples is a way to go that does not modify the audio pitch. + */ +public class CutAudioStretcher implements AudioStretcher { + + @Override + public void stretch(@NonNull ShortBuffer input, @NonNull ShortBuffer output, int channels) { + if (input.remaining() < output.remaining()) { + throw new IllegalArgumentException("Illegal use of CutAudioStretcher"); + } + int exceeding = input.remaining() - output.remaining(); + input.limit(input.limit() - exceeding); // Make remaining() the same for both + output.put(input); // Safely bulk-put + input.limit(input.limit() + exceeding); // Restore + input.position(input.limit()); // Make as if we have read it all + } +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/stretch/DefaultAudioStretcher.java b/lib/src/main/java/com/otaliastudios/transcoder/stretch/DefaultAudioStretcher.java new file mode 100644 index 00000000..b83726ab --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/stretch/DefaultAudioStretcher.java @@ -0,0 +1,23 @@ +package com.otaliastudios.transcoder.stretch; + +import androidx.annotation.NonNull; + +import java.nio.ShortBuffer; + +/** + * An {@link AudioStretcher} that delegates to appropriate classes + * based on input and output size. + */ +public class DefaultAudioStretcher implements AudioStretcher { + + @Override + public void stretch(@NonNull ShortBuffer input, @NonNull ShortBuffer output, int channels) { + if (input.remaining() < output.remaining()) { + INSERT.stretch(input, output, channels); + } else if (input.remaining() > output.remaining()) { + CUT.stretch(input, output, channels); + } else { + PASSTHROUGH.stretch(input, output, channels); + } + } +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/stretch/InsertAudioStretcher.java b/lib/src/main/java/com/otaliastudios/transcoder/stretch/InsertAudioStretcher.java new file mode 100644 index 00000000..00ed8263 --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/stretch/InsertAudioStretcher.java @@ -0,0 +1,55 @@ +package com.otaliastudios.transcoder.stretch; + +import androidx.annotation.NonNull; + +import java.nio.ShortBuffer; +import java.util.Random; + +/** + * A {@link AudioStretcher} meant to be used when output size is bigger than the input. + * It will insert noise samples to fill the gaps, at regular intervals. + * This modifies the audio pitch of course. + */ +public class InsertAudioStretcher implements AudioStretcher { + + private final static Random NOISE = new Random(); + + private static short noise() { + return (short) NOISE.nextInt(1000); + } + + private static float ratio(int remaining, int all) { + return (float) remaining / all; + } + + @Override + public void stretch(@NonNull ShortBuffer input, @NonNull ShortBuffer output, int channels) { + if (input.remaining() >= output.remaining()) { + throw new IllegalArgumentException("Illegal use of AudioStretcher.INSERT"); + } + if (channels != 1 && channels != 2) { + throw new IllegalArgumentException("Illegal use of AudioStretcher.INSERT. Channels:" + channels); + } + final int inputSamples = input.remaining() / channels; + final int fakeSamples = (int) Math.floor((double) (output.remaining() - input.remaining()) / channels); + int remainingInputSamples = inputSamples; + int remainingFakeSamples = fakeSamples; + float remainingInputSamplesRatio = ratio(remainingInputSamples, inputSamples); + float remainingFakeSamplesRatio = ratio(remainingFakeSamples, fakeSamples); + while (remainingInputSamples > 0 && remainingFakeSamples > 0) { + // Will this be an input sample or a fake sample? + // Choose the one with the bigger ratio. + if (remainingInputSamplesRatio >= remainingFakeSamplesRatio) { + output.put(input.get()); + if (channels == 2) output.put(input.get()); + remainingInputSamples--; + remainingInputSamplesRatio = ratio(remainingInputSamples, inputSamples); + } else { + output.put(noise()); + if (channels == 2) output.put(noise()); + remainingFakeSamples--; + remainingFakeSamplesRatio = ratio(remainingFakeSamples, inputSamples); + } + } + } +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/stretch/PassThroughAudioStretcher.java b/lib/src/main/java/com/otaliastudios/transcoder/stretch/PassThroughAudioStretcher.java new file mode 100644 index 00000000..14768c8f --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/stretch/PassThroughAudioStretcher.java @@ -0,0 +1,19 @@ +package com.otaliastudios.transcoder.stretch; + +import androidx.annotation.NonNull; + +import java.nio.ShortBuffer; + +/** + * A no-op {@link AudioStretcher} that copies input into output. + */ +public class PassThroughAudioStretcher implements AudioStretcher { + + @Override + public void stretch(@NonNull ShortBuffer input, @NonNull ShortBuffer output, int channels) { + if (input.remaining() > output.remaining()) { + throw new IllegalArgumentException("Illegal use of PassThroughAudioStretcher"); + } + output.put(input); + } +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java index 4673ff6f..740bf3ea 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/AudioTrackTranscoder.java @@ -9,6 +9,7 @@ import com.otaliastudios.transcoder.engine.TrackType; import com.otaliastudios.transcoder.engine.TranscoderMuxer; import com.otaliastudios.transcoder.internal.MediaCodecBuffers; +import com.otaliastudios.transcoder.stretch.AudioStretcher; import com.otaliastudios.transcoder.time.TimeInterpolator; import com.otaliastudios.transcoder.transcode.internal.AudioEngine; @@ -17,6 +18,7 @@ public class AudioTrackTranscoder extends BaseTrackTranscoder { private TimeInterpolator mTimeInterpolator; + private AudioStretcher mAudioStretcher; private AudioEngine mAudioEngine; private MediaCodec mEncoder; // to create the channel private MediaFormat mEncoderOutputFormat; // to create the channel @@ -24,9 +26,11 @@ public class AudioTrackTranscoder extends BaseTrackTranscoder { public AudioTrackTranscoder(@NonNull MediaExtractor extractor, @NonNull TranscoderMuxer muxer, int trackIndex, - @NonNull TimeInterpolator timeInterpolator) { + @NonNull TimeInterpolator timeInterpolator, + @NonNull AudioStretcher audioStretcher) { super(extractor, muxer, TrackType.AUDIO, trackIndex); mTimeInterpolator = timeInterpolator; + mAudioStretcher = audioStretcher; } @Override @@ -45,10 +49,11 @@ protected boolean onFeedEncoder(@NonNull MediaCodec encoder, @NonNull MediaCodec @Override protected void onDecoderOutputFormatChanged(@NonNull MediaCodec decoder, @NonNull MediaFormat format) { super.onDecoderOutputFormatChanged(decoder, format); - mAudioEngine = new AudioEngine(decoder, format, mEncoder, mEncoderOutputFormat, mTimeInterpolator); + mAudioEngine = new AudioEngine(decoder, format, mEncoder, mEncoderOutputFormat, mTimeInterpolator, mAudioStretcher); mEncoder = null; mEncoderOutputFormat = null; mTimeInterpolator = null; + mAudioStretcher = null; } @Override diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java index c2a36576..da57cf23 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java @@ -9,6 +9,7 @@ import com.otaliastudios.transcoder.internal.Logger; import com.otaliastudios.transcoder.internal.MediaCodecBuffers; import com.otaliastudios.transcoder.remix.AudioRemixer; +import com.otaliastudios.transcoder.stretch.AudioStretcher; import com.otaliastudios.transcoder.time.TimeInterpolator; import java.nio.Buffer; @@ -38,8 +39,9 @@ public class AudioEngine { private final MediaCodec mEncoder; private final int mSampleRate; private final int mDecoderChannels; - private final int mEncoderChannels; + @SuppressWarnings("FieldCanBeLocal") private final int mEncoderChannels; private final AudioRemixer mRemixer; + private final AudioStretcher mStretcher; private final TimeInterpolator mTimeInterpolator; private long mLastDecoderUs = Long.MIN_VALUE; private long mLastEncoderUs = Long.MIN_VALUE; @@ -58,7 +60,8 @@ public AudioEngine(@NonNull MediaCodec decoder, @NonNull MediaFormat decoderOutputFormat, @NonNull MediaCodec encoder, @NonNull MediaFormat encoderOutputFormat, - @NonNull TimeInterpolator timeInterpolator) { + @NonNull TimeInterpolator timeInterpolator, + @NonNull AudioStretcher audioStretcher) { mDecoder = decoder; mEncoder = encoder; mTimeInterpolator = timeInterpolator; @@ -81,7 +84,7 @@ public AudioEngine(@NonNull MediaCodec decoder, throw new UnsupportedOperationException("Input channel count (" + mDecoderChannels + ") not supported."); } - // Create remixer. + // Create remixer and stretcher. if (mDecoderChannels > mEncoderChannels) { mRemixer = AudioRemixer.DOWNMIX; } else if (mDecoderChannels < mEncoderChannels) { @@ -89,6 +92,7 @@ public AudioEngine(@NonNull MediaCodec decoder, } else { mRemixer = AudioRemixer.PASSTHROUGH; } + mStretcher = audioStretcher; } /** @@ -237,11 +241,14 @@ private boolean process(@NonNull AudioBuffer buffer, @NonNull ShortBuffer encode buffer.decoderData.limit(buffer.decoderData.limit() - overflowReduction); } final int finalInputSize = buffer.decoderData.remaining(); - LOG.i("process - inputSize:" + inputSize + " processedInputSize:" + processedInputSize + " outputSize:" + outputSize + " finalInputSize:" + finalInputSize); + LOG.i("process - inputSize:" + inputSize + + " processedInputSize:" + processedInputSize + + " outputSize:" + outputSize + + " finalInputSize:" + finalInputSize); // 4. Do the stretching. We need a bridge buffer for its output. ensureTempBuffer(finalInputSize + stretchShorts); - AudioStretcher.CUT_OR_INSERT.stretch(buffer.decoderData, mTempBuffer, mDecoderChannels); + mStretcher.stretch(buffer.decoderData, mTempBuffer, mDecoderChannels); // 5. Do the actual remixing. mTempBuffer.position(0); @@ -255,7 +262,7 @@ private boolean process(@NonNull AudioBuffer buffer, @NonNull ShortBuffer encode buffer.decoderData.limit(buffer.decoderData.limit() + overflowReduction); } - // 5. Write the buffer. + // 7. Write the buffer. // This is the encoder buffer: we have likely written it all, but let's use // encoderBuffer.position() to know how much anyway. mEncoder.queueInputBuffer(encoderBufferIndex, diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioStretcher.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioStretcher.java deleted file mode 100644 index af113db9..00000000 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioStretcher.java +++ /dev/null @@ -1,104 +0,0 @@ -package com.otaliastudios.transcoder.transcode.internal; - -import androidx.annotation.NonNull; - -import java.nio.Buffer; -import java.nio.ShortBuffer; -import java.util.Random; - -interface AudioStretcher { - - /** - * Stretches the input into the output, based on the {@link Buffer#remaining()} value of both. - * At the end of this method, the {@link Buffer#position()} of both should be equal to their - * respective {@link Buffer#limit()}. - * - * And of course, both {@link Buffer#limit()}s should remain unchanged. - * - * @param input input buffer - * @param output output buffer - * @param channels audio channels - */ - void stretch(@NonNull ShortBuffer input, @NonNull ShortBuffer output, int channels); - - AudioStretcher PASSTHROUGH = new AudioStretcher() { - @Override - public void stretch(@NonNull ShortBuffer input, @NonNull ShortBuffer output, int channels) { - if (input.remaining() > output.remaining()) { - throw new IllegalArgumentException("Illegal use of AudioStretcher.PASSTHROUGH"); - } - output.put(input); - } - }; - - AudioStretcher CUT = new AudioStretcher() { - @Override - public void stretch(@NonNull ShortBuffer input, @NonNull ShortBuffer output, int channels) { - if (input.remaining() < output.remaining()) { - throw new IllegalArgumentException("Illegal use of AudioStretcher.CUT"); - } - int exceeding = input.remaining() - output.remaining(); - input.limit(input.limit() - exceeding); // Make remaining() the same for both - output.put(input); // Safely bulk-put - input.limit(input.limit() + exceeding); // Restore - input.position(input.limit()); // Make as if we have read it all - } - }; - - AudioStretcher INSERT = new AudioStretcher() { - - private final Random NOISE = new Random(); - - private short noise() { - return (short) NOISE.nextInt(1000); - } - - private float ratio(int remaining, int all) { - return (float) remaining / all; - } - - @Override - public void stretch(@NonNull ShortBuffer input, @NonNull ShortBuffer output, int channels) { - if (input.remaining() >= output.remaining()) { - throw new IllegalArgumentException("Illegal use of AudioStretcher.INSERT"); - } - if (channels != 1 && channels != 2) { - throw new IllegalArgumentException("Illegal use of AudioStretcher.INSERT. Channels:" + channels); - } - final int inputSamples = input.remaining() / channels; - final int fakeSamples = (int) Math.floor((double) (output.remaining() - input.remaining()) / channels); - int remainingInputSamples = inputSamples; - int remainingFakeSamples = fakeSamples; - float remainingInputSamplesRatio = ratio(remainingInputSamples, inputSamples); - float remainingFakeSamplesRatio = ratio(remainingFakeSamples, fakeSamples); - while (remainingInputSamples > 0 && remainingFakeSamples > 0) { - // Will this be an input sample or a fake sample? - // Choose the one with the bigger ratio. - if (remainingInputSamplesRatio >= remainingFakeSamplesRatio) { - output.put(input.get()); - if (channels == 2) output.put(input.get()); - remainingInputSamples--; - remainingInputSamplesRatio = ratio(remainingInputSamples, inputSamples); - } else { - output.put(noise()); - if (channels == 2) output.put(noise()); - remainingFakeSamples--; - remainingFakeSamplesRatio = ratio(remainingFakeSamples, inputSamples); - } - } - } - }; - - AudioStretcher CUT_OR_INSERT = new AudioStretcher() { - @Override - public void stretch(@NonNull ShortBuffer input, @NonNull ShortBuffer output, int channels) { - if (input.remaining() < output.remaining()) { - INSERT.stretch(input, output, channels); - } else if (input.remaining() > output.remaining()) { - CUT.stretch(input, output, channels); - } else { - PASSTHROUGH.stretch(input, output, channels); - } - } - }; -} From 4ba297effebe8ae0ee0057f825f0589453135df3 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Wed, 24 Jul 2019 14:47:04 +0200 Subject: [PATCH 17/22] Fix VideoTrackTranscoder --- .../transcoder/transcode/VideoTrackTranscoder.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java index c6f5ee8b..c58440d7 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java @@ -114,7 +114,7 @@ public void release() { @Override protected boolean onFeedEncoder(@NonNull MediaCodec encoder, @NonNull MediaCodecBuffers encoderBuffers, long timeoutUs) { - // We do not feed the encoder, see below. + // We do not feed the encoder, instead we wait for the encoder surface onFrameAvailable callback. return false; } @@ -123,11 +123,15 @@ protected void onDrainDecoder(@NonNull MediaCodec decoder, int bufferIndex, @Non if (endOfStream) { mEncoder.signalEndOfInputStream(); decoder.releaseOutputBuffer(bufferIndex, false); - } else if (mFrameDropper.shouldRenderFrame(presentationTimeUs)) { - decoder.releaseOutputBuffer(bufferIndex, true); + } else { long interpolatedTimeUs = mTimeInterpolator.interpolate(TrackType.VIDEO, presentationTimeUs); - mDecoderOutputSurface.drawFrame(); - mEncoderInputSurface.onFrame(interpolatedTimeUs); + if (mFrameDropper.shouldRenderFrame(interpolatedTimeUs)) { + decoder.releaseOutputBuffer(bufferIndex, true); + mDecoderOutputSurface.drawFrame(); + mEncoderInputSurface.onFrame(interpolatedTimeUs); + } else { + decoder.releaseOutputBuffer(bufferIndex, false); + } } } } From 474f79255a1b2eb7180fcffbe187dd59163982b5 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Wed, 24 Jul 2019 15:02:20 +0200 Subject: [PATCH 18/22] Review README --- README.md | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index e8c75f97..1ec8e242 100644 --- a/README.md +++ b/README.md @@ -28,23 +28,29 @@ Transcoder.into(filePath) Take a look at the demo app for a real example or keep reading below for documentation. -*Note: this project is an improved fork of [ypresto/android-transcoder](https://github.com/ypresto/android-transcoder). -It features a lot of improvements over the original project, including:* - -- *Multithreading support* -- *Crop to any aspect ratio* -- *Set output video rotation* -- *Change output video speed (0.5x, 2x or any float)* -- *Various bugs fixed* -- *[Input](#data-sources): Accept content Uris and other types* -- *[Real error handling](#listening-for-events) instead of errors being thrown* -- *Frame dropping support, which means you can set the video frame rate* -- *Source project is over-conservative when choosing options that *might* not be supported. We prefer to try and let the codec fail* -- *More convenient APIs for transcoding & choosing options* -- *Configurable [Validators](#validators) to e.g. **not** perform transcoding if the source video is already compressed enough* -- *Expose internal logs through Logger (so they can be reported to e.g. Crashlytics)* -- *Handy utilities for track configuration through [Output Strategies](#output-strategies)* -- *Handy utilities for resizing* +## Features + +This project is an improved fork of [ypresto/android-transcoder](https://github.com/ypresto/android-transcoder) +which features a lot of improvements and new functionalities. + +- Fast transcoding to AAC/AVC +- Hardware accelerated +- Multithreaded +- Convenient, fluent API +- Choose output size, with automatic cropping [[docs]](#video-size) +- Choose output rotation [[docs]](#video-rotation) +- Choose output speed [[docs]](#video-speed) +- Choose output frame rate [[docs]](#other-options) +- Choose output audio channels [[docs]](#audio-strategies) +- Override frames timestamp, e.g. to slow down the middle part of the video [[docs]](#time-interpolation) +- Error handling [[docs]](#listening-for-events) +- Configurable validators to e.g. avoid transcoding if the source is already compressed enough [[docs]](#validators) +- Configurable video and audio strategies [[docs](#output-strategies)] + +*With respect to [android-transcoder](https://github.com/ypresto/android-transcoder), which misses most +of the functionality above, we have also fixed a huge number of bugs and are much less conservative +when choosing options that might not be supported. The source project will always throw - for example, +accepting only 16:9, AVC Baseline Profile videos - we prefer to try and let the codec fail if it wants to*. ## Setup @@ -304,7 +310,7 @@ DefaultVideoStrategy strategy = new DefaultVideoStrategy.Builder() .build() ``` -## Other Options +## Advanced Options #### Video rotation From 867cd3739a99826b85b4ffa2182a8910c435d814 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Wed, 24 Jul 2019 15:36:54 +0200 Subject: [PATCH 19/22] Adjust noise --- README.md | 2 +- .../otaliastudios/transcoder/stretch/InsertAudioStretcher.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1ec8e242..938a61db 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ which features a lot of improvements and new functionalities. - Override frames timestamp, e.g. to slow down the middle part of the video [[docs]](#time-interpolation) - Error handling [[docs]](#listening-for-events) - Configurable validators to e.g. avoid transcoding if the source is already compressed enough [[docs]](#validators) -- Configurable video and audio strategies [[docs](#output-strategies)] +- Configurable video and audio strategies [[docs]](#output-strategies) *With respect to [android-transcoder](https://github.com/ypresto/android-transcoder), which misses most of the functionality above, we have also fixed a huge number of bugs and are much less conservative diff --git a/lib/src/main/java/com/otaliastudios/transcoder/stretch/InsertAudioStretcher.java b/lib/src/main/java/com/otaliastudios/transcoder/stretch/InsertAudioStretcher.java index 00ed8263..fe01dd60 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/stretch/InsertAudioStretcher.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/stretch/InsertAudioStretcher.java @@ -15,7 +15,7 @@ public class InsertAudioStretcher implements AudioStretcher { private final static Random NOISE = new Random(); private static short noise() { - return (short) NOISE.nextInt(1000); + return (short) NOISE.nextInt(500); } private static float ratio(int remaining, int all) { From 3a76c1ecfc39119d775e9aee2680d127e8ea2177 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Wed, 24 Jul 2019 15:47:35 +0200 Subject: [PATCH 20/22] Fix progress callback --- .../com/otaliastudios/transcoder/engine/TranscoderEngine.java | 1 + .../otaliastudios/transcoder/stretch/InsertAudioStretcher.java | 2 +- .../otaliastudios/transcoder/transcode/BaseTrackTranscoder.java | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderEngine.java b/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderEngine.java index d4cdaa95..2dc1a85c 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderEngine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/engine/TranscoderEngine.java @@ -264,6 +264,7 @@ private void runPipelines() throws InterruptedException { if (mDurationUs > 0 && loopCount % PROGRESS_INTERVAL_STEPS == 0) { double videoProgress = getTranscoderProgress(videoTranscoder, mTracks.status(TrackType.VIDEO)); double audioProgress = getTranscoderProgress(audioTranscoder, mTracks.status(TrackType.AUDIO)); + LOG.i("progress - video:" + videoProgress + " audio:" + audioProgress); double progress = (videoProgress + audioProgress) / getTranscodersCount(); mProgress = progress; if (mProgressCallback != null) mProgressCallback.onProgress(progress); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/stretch/InsertAudioStretcher.java b/lib/src/main/java/com/otaliastudios/transcoder/stretch/InsertAudioStretcher.java index fe01dd60..7c104cd5 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/stretch/InsertAudioStretcher.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/stretch/InsertAudioStretcher.java @@ -15,7 +15,7 @@ public class InsertAudioStretcher implements AudioStretcher { private final static Random NOISE = new Random(); private static short noise() { - return (short) NOISE.nextInt(500); + return (short) NOISE.nextInt(300); } private static float ratio(int remaining, int all) { diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java index 535c7e48..e2850338 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java @@ -245,6 +245,7 @@ private int drainDecoder(long timeoutUs) { boolean hasSize = mBufferInfo.size > 0; if (isEos) mIsDecoderEOS = true; if (isEos || hasSize) { + mLastPresentationTimeUs = mBufferInfo.presentationTimeUs; onDrainDecoder(mDecoder, result, mDecoderBuffers.getOutputBuffer(result), @@ -284,7 +285,6 @@ private int drainEncoder(long timeoutUs) { return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; } mMuxer.write(mTrackType, mEncoderBuffers.getOutputBuffer(result), mBufferInfo); - mLastPresentationTimeUs = mBufferInfo.presentationTimeUs; mEncoder.releaseOutputBuffer(result, false); return DRAIN_STATE_CONSUMED; } From e7830f2e11007fe415854fecde08a2ef54936121 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Thu, 25 Jul 2019 01:31:39 +0200 Subject: [PATCH 21/22] Small improvements in README --- README.md | 48 +++++++++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 938a61db..18c4725c 100644 --- a/README.md +++ b/README.md @@ -30,9 +30,6 @@ Take a look at the demo app for a real example or keep reading below for documen ## Features -This project is an improved fork of [ypresto/android-transcoder](https://github.com/ypresto/android-transcoder) -which features a lot of improvements and new functionalities. - - Fast transcoding to AAC/AVC - Hardware accelerated - Multithreaded @@ -47,10 +44,11 @@ which features a lot of improvements and new functionalities. - Configurable validators to e.g. avoid transcoding if the source is already compressed enough [[docs]](#validators) - Configurable video and audio strategies [[docs]](#output-strategies) -*With respect to [android-transcoder](https://github.com/ypresto/android-transcoder), which misses most -of the functionality above, we have also fixed a huge number of bugs and are much less conservative -when choosing options that might not be supported. The source project will always throw - for example, -accepting only 16:9, AVC Baseline Profile videos - we prefer to try and let the codec fail if it wants to*. +*This project started as a fork of [ypresto/android-transcoder](https://github.com/ypresto/android-transcoder). +With respect to the source project, which misses most of the functionality listed above, +we have also fixed a huge number of bugs and are much less conservative when choosing options +that might not be supported. The source project will always throw - for example, accepting only 16:9, +AVC Baseline Profile videos - we prefer to try and let the codec fail if it wants to*. ## Setup @@ -135,8 +133,8 @@ Transcoding operation did succeed. The success code can be: |Code|Meaning| |----|-------| -|`MediaTranscoder.SUCCESS_TRANSCODED`|Transcoding was executed successfully. Transcoded file was written to the output path.| -|`MediaTranscoder.SUCCESS_NOT_NEEDED`|Transcoding was not executed because it was considered **not needed** by the `Validator`.| +|`Transcoder.SUCCESS_TRANSCODED`|Transcoding was executed successfully. Transcoded file was written to the output path.| +|`Transcoder.SUCCESS_NOT_NEEDED`|Transcoding was not executed because it was considered **not needed** by the `Validator`.| Keep reading [below](#validators) to know about `Validator`s. @@ -227,9 +225,9 @@ audio stream to AAC format with the specified number of channels. ```java Transcoder.into(filePath) - .setAudioOutputStrategy(DefaultAudioStrategy(1)) // or.. - .setAudioOutputStrategy(DefaultAudioStrategy(2)) // or.. - .setAudioOutputStrategy(DefaultAudioStrategy(DefaultAudioStrategy.AUDIO_CHANNELS_AS_IS)) + .setAudioOutputStrategy(new DefaultAudioStrategy(1)) // or.. + .setAudioOutputStrategy(new DefaultAudioStrategy(2)) // or.. + .setAudioOutputStrategy(new DefaultAudioStrategy(DefaultAudioStrategy.AUDIO_CHANNELS_AS_IS)) // ... ``` @@ -250,16 +248,16 @@ We provide helpers for common tasks: DefaultVideoStrategy strategy; // Sets an exact size. If aspect ratio does not match, cropping will take place. -strategy = DefaultVideoStrategy.exact(1080, 720).build() +strategy = DefaultVideoStrategy.exact(1080, 720).build(); // Keeps the aspect ratio, but scales down the input size with the given fraction. -strategy = DefaultVideoStrategy.fraction(0.5F).build() +strategy = DefaultVideoStrategy.fraction(0.5F).build(); // Ensures that each video size is at most the given value - scales down otherwise. -strategy = DefaultVideoStrategy.atMost(1000).build() +strategy = DefaultVideoStrategy.atMost(1000).build(); // Ensures that minor and major dimension are at most the given values - scales down otherwise. -strategy = DefaultVideoStrategy.atMost(500, 1000).build() +strategy = DefaultVideoStrategy.atMost(500, 1000).build(); ``` In fact, all of these will simply call `new DefaultVideoStrategy.Builder(resizer)` with a special @@ -277,14 +275,14 @@ You can also group resizers through `MultiResizer`, which applies resizers in ch ```java // First scales down, then ensures size is at most 1000. Order matters! -Resizer resizer = new MultiResizer() -resizer.addResizer(new FractionResizer(0.5F)) -resizer.addResizer(new AtMostResizer(1000)) +Resizer resizer = new MultiResizer(); +resizer.addResizer(new FractionResizer(0.5F)); +resizer.addResizer(new AtMostResizer(1000)); // First makes it 16:9, then ensures size is at most 1000. Order matters! -Resizer resizer = new MultiResizer() -resizer.addResizer(new AspectRatioResizer(16F / 9F)) -resizer.addResizer(new AtMostResizer(1000)) +Resizer resizer = new MultiResizer(); +resizer.addResizer(new AspectRatioResizer(16F / 9F)); +resizer.addResizer(new AtMostResizer(1000)); ``` This option is already available through the DefaultVideoStrategy builder, so you can do: @@ -294,7 +292,7 @@ DefaultVideoStrategy strategy = new DefaultVideoStrategy.Builder() .addResizer(new AspectRatioResizer(16F / 9F)) .addResizer(new FractionResizer(0.5F)) .addResizer(new AtMostResizer(1000)) - .build() + .build(); ``` ### Other options @@ -307,7 +305,7 @@ DefaultVideoStrategy strategy = new DefaultVideoStrategy.Builder() .bitRate(DefaultVideoStrategy.BITRATE_UNKNOWN) // tries to estimate .frameRate(frameRate) // will be capped to the input frameRate .iFrameInterval(interval) // interval between I-frames in seconds - .build() + .build(); ``` ## Advanced Options @@ -394,7 +392,7 @@ This is a complex issue which is especially important for video strategies, as a to a transcoding error or corrupted file. Android platform specifies requirements for manufacturers through the [CTS (Compatibility test suite)](https://source.android.com/compatibility/cts). -Only a few codecs and sizes are strictly required to work. +Only a few codecs and sizes are **strictly** required to work. We collect common presets in the `DefaultVideoStrategies` class: From eccaffba8de45aca6f60f1ed4ee1f32c5ad4a8f2 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Thu, 25 Jul 2019 12:24:47 +0200 Subject: [PATCH 22/22] Improve droppers --- .../transcode/internal/VideoFrameDropper.java | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/VideoFrameDropper.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/VideoFrameDropper.java index e93bd605..d1c7ea85 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/VideoFrameDropper.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/VideoFrameDropper.java @@ -3,7 +3,6 @@ import androidx.annotation.NonNull; import com.otaliastudios.transcoder.internal.Logger; -import com.otaliastudios.transcoder.transcode.VideoTrackTranscoder; /** * Drops input frames to respect the output frame rate. @@ -22,11 +21,16 @@ public static VideoFrameDropper newDropper(int inputFrameRate, int outputFrameRa return new Dropper1(inputFrameRate, outputFrameRate); } + /** + * A simple and more elegant dropper. + * Reference: https://stackoverflow.com/questions/4223766/dropping-video-frames + */ private static class Dropper1 extends VideoFrameDropper { private double mInFrameRateReciprocal; private double mOutFrameRateReciprocal; private double mFrameRateReciprocalSum; + private int mFrameCount; private Dropper1(int inputFrameRate, int outputFrameRate) { mInFrameRateReciprocal = 1.0d / inputFrameRate; @@ -36,22 +40,26 @@ private Dropper1(int inputFrameRate, int outputFrameRate) { @Override public boolean shouldRenderFrame(long presentationTimeUs) { - boolean firstFrame = Double.valueOf(0d).equals(mFrameRateReciprocalSum); mFrameRateReciprocalSum += mInFrameRateReciprocal; - if (firstFrame) { - LOG.v("RENDERING - frameRateReciprocalSum:" + mFrameRateReciprocalSum); + if (mFrameCount++ == 0) { + LOG.v("RENDERING (first frame) - frameRateReciprocalSum:" + mFrameRateReciprocalSum); return true; - } - if (mFrameRateReciprocalSum > mOutFrameRateReciprocal) { + } else if (mFrameRateReciprocalSum > mOutFrameRateReciprocal) { mFrameRateReciprocalSum -= mOutFrameRateReciprocal; LOG.v("RENDERING - frameRateReciprocalSum:" + mFrameRateReciprocalSum); return true; + } else { + LOG.v("DROPPING - frameRateReciprocalSum:" + mFrameRateReciprocalSum); + return false; } - LOG.v("DROPPING - frameRateReciprocalSum:" + mFrameRateReciprocalSum); - return false; } } + /** + * The old dropper, keeping here just in case. + * Will test {@link Dropper1} and remove this soon. + */ + @SuppressWarnings("unused") private static class Dropper2 extends VideoFrameDropper { // A step is defined as the microseconds between two frame.