Skip to content

Commit a459a99

Browse files
ChunMinChangChunMinChang
committed
Bug 1978232 - Add batch encoding to various encoders r=media-playback-reviewers,jolin
This patch extends batch encoding support to additional encoders, inclduing FFmpeg encoders, GMP encoder, Android encoder, and macOS encoders. The implementations for GMP and macOS are not optimized for best real-time performance, but their usage will be limited via a preference. A more complete implementation will be addressed in Bug 1984936. Differential Revision: https://phabricator.services.mozilla.com/D263467
1 parent 6d9fcd3 commit a459a99

File tree

10 files changed

+219
-39
lines changed

10 files changed

+219
-39
lines changed

‎dom/media/platforms/agnostic/gmp/GMPVideoEncoder.cpp‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -271,6 +271,25 @@ RefPtr<MediaDataEncoder::EncodePromise> GMPVideoEncoder::Encode(
271271
return promise.forget();
272272
}
273273

274+
// TODO(Bug 1984936): For realtime mode, resolve the promise after the first
275+
// sample's result is available, then continue processing remaining samples.
276+
// This allows the caller to keep submitting new samples while the encoder
277+
// handles pending ones.
278+
RefPtr<MediaDataEncoder::EncodePromise> GMPVideoEncoder::Encode(
279+
nsTArray<RefPtr<MediaData>>&& aSamples) {
280+
MOZ_ASSERT(!aSamples.IsEmpty());
281+
MOZ_ASSERT(IsOnGMPThread());
282+
283+
if (NS_WARN_IF(!IsInitialized())) {
284+
return EncodePromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR,
285+
__func__);
286+
}
287+
288+
RefPtr<EncodePromise> promise = mEncodeBatchPromise.Ensure(__func__);
289+
EncodeNextSample(std::move(aSamples), EncodedData());
290+
return promise;
291+
}
292+
274293
RefPtr<MediaDataEncoder::ReconfigurationPromise> GMPVideoEncoder::Reconfigure(
275294
const RefPtr<const EncoderConfigurationChangeList>& aConfigurationChanges) {
276295
// General reconfiguration interface not implemented right now
@@ -435,6 +454,9 @@ void GMPVideoEncoder::Teardown(const MediaResult& aResult,
435454
// Ensure we are kept alive at least until we return.
436455
RefPtr<GMPVideoEncoder> self(this);
437456

457+
mEncodeBatchPromise.RejectIfExists(aResult, aCallSite);
458+
mEncodeBatchRequest.DisconnectIfExists();
459+
438460
PendingEncodePromises pendingEncodes = std::move(mPendingEncodes);
439461
for (auto i = pendingEncodes.Iter(); !i.Done(); i.Next()) {
440462
i.Data()->Reject(aResult, aCallSite);
@@ -466,4 +488,43 @@ void GMPVideoEncoder::Terminated() {
466488
__func__);
467489
}
468490

491+
void GMPVideoEncoder::EncodeNextSample(
492+
nsTArray<RefPtr<MediaData>>&& aInputs,
493+
MediaDataEncoder::EncodedData&& aOutputs) {
494+
MOZ_ASSERT(IsOnGMPThread());
495+
MOZ_ASSERT(IsInitialized());
496+
MOZ_ASSERT(!mEncodeBatchPromise.IsEmpty());
497+
MOZ_ASSERT(!mEncodeBatchRequest.Exists());
498+
499+
if (aInputs.IsEmpty()) {
500+
GMP_LOG_VERBOSE("[%p] All samples processed. Resolving the encode promise",
501+
this);
502+
mEncodeBatchPromise.Resolve(std::move(aOutputs), __func__);
503+
return;
504+
}
505+
506+
GMP_LOG_VERBOSE("[%p] Processing next sample out of %zu remaining", this,
507+
aInputs.Length());
508+
Encode(aInputs[0])
509+
->Then(
510+
GetCurrentSerialEventTarget(), __func__,
511+
[self = RefPtr{this}, inputs = std::move(aInputs),
512+
outputs = std::move(aOutputs)](
513+
MediaDataEncoder::EncodedData&& aData) mutable {
514+
self->mEncodeBatchRequest.Complete();
515+
inputs.RemoveElementAt(0);
516+
outputs.AppendElements(aData);
517+
self->EncodeNextSample(std::move(inputs), std::move(outputs));
518+
},
519+
[self = RefPtr{this}](const MediaResult& aError) {
520+
self->mEncodeBatchRequest.Complete();
521+
GMP_LOG_ERROR(
522+
"[%p] GMPVideoEncoder::EncodeNextSample -- failed to encode: "
523+
"%s",
524+
self.get(), aError.Description().get());
525+
self->mEncodeBatchPromise.Reject(aError, __func__);
526+
})
527+
->Track(mEncodeBatchRequest);
528+
}
529+
469530
} // namespace mozilla

‎dom/media/platforms/agnostic/gmp/GMPVideoEncoder.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ class GMPVideoEncoder final : public MediaDataEncoder,
2929

3030
RefPtr<InitPromise> Init() override;
3131
RefPtr<EncodePromise> Encode(const MediaData* aSample) override;
32+
RefPtr<EncodePromise> Encode(nsTArray<RefPtr<MediaData>>&& aSamples) override;
3233
RefPtr<ReconfigurationPromise> Reconfigure(
3334
const RefPtr<const EncoderConfigurationChangeList>& aConfigurationChanges)
3435
override;
@@ -65,6 +66,9 @@ class GMPVideoEncoder final : public MediaDataEncoder,
6566

6667
void Teardown(const MediaResult& aResult, StaticString aCallSite);
6768

69+
void EncodeNextSample(nsTArray<RefPtr<MediaData>>&& aInputs,
70+
MediaDataEncoder::EncodedData&& aOutputs);
71+
6872
const EncoderConfig mConfig;
6973
nsCOMPtr<mozIGeckoMediaPluginService> mMPS;
7074
GMPVideoEncoderProxy* mGMP = nullptr;
@@ -75,6 +79,9 @@ class GMPVideoEncoder final : public MediaDataEncoder,
7579
using PendingEncodePromises =
7680
nsRefPtrHashtable<nsUint64HashKey, EncodePromise::Private>;
7781
PendingEncodePromises mPendingEncodes;
82+
83+
MozPromiseHolder<EncodePromise> mEncodeBatchPromise;
84+
MozPromiseRequestHolder<EncodePromise> mEncodeBatchRequest;
7885
};
7986

8087
} // namespace mozilla

‎dom/media/platforms/android/AndroidDataEncoder.cpp‎

Lines changed: 46 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -153,9 +153,26 @@ RefPtr<MediaDataEncoder::EncodePromise> AndroidDataEncoder::Encode(
153153
RefPtr<AndroidDataEncoder> self = this;
154154
MOZ_ASSERT(aSample != nullptr);
155155

156-
RefPtr<const MediaData> sample(aSample);
156+
return InvokeAsync(
157+
mTaskQueue, __func__,
158+
[self, sample = RefPtr<MediaData>(const_cast<MediaData*>(aSample))]() {
159+
return self->ProcessEncode({sample});
160+
});
161+
}
162+
163+
// TODO(Bug 1984936): For realtime mode, resolve the promise after the first
164+
// sample's result is available, then continue processing remaining samples.
165+
// This allows the caller to keep submitting new samples while the encoder
166+
// handles pending ones.
167+
RefPtr<MediaDataEncoder::EncodePromise> AndroidDataEncoder::Encode(
168+
nsTArray<RefPtr<MediaData>>&& aSamples) {
169+
RefPtr<AndroidDataEncoder> self = this;
170+
MOZ_ASSERT(!aSamples.IsEmpty());
171+
157172
return InvokeAsync(mTaskQueue, __func__,
158-
[self, sample]() { return self->ProcessEncode(sample); });
173+
[self, samples = std::move(aSamples)]() mutable {
174+
return self->ProcessEncode(std::move(samples));
175+
});
159176
}
160177

161178
static jni::ByteBuffer::LocalRef ConvertI420ToNV12Buffer(
@@ -193,36 +210,41 @@ static jni::ByteBuffer::LocalRef ConvertI420ToNV12Buffer(
193210
}
194211

195212
RefPtr<MediaDataEncoder::EncodePromise> AndroidDataEncoder::ProcessEncode(
196-
const RefPtr<const MediaData>& aSample) {
213+
nsTArray<RefPtr<MediaData>>&& aSamples) {
197214
AssertOnTaskQueue();
198215

199216
REJECT_IF_ERROR();
200217

201-
RefPtr<const VideoData> sample(aSample->As<const VideoData>());
202-
MOZ_ASSERT(sample);
203-
204-
mInputSampleDuration = aSample->mDuration;
218+
// TODO(Bug 1984936): Looping here for large batches is inefficient, as it can
219+
// take excessive shared memory and file descriptors due to passing both input
220+
// and output buffers between the content and media codec processes.
221+
for (auto& s : aSamples) {
222+
RefPtr<const VideoData> sample(s->As<const VideoData>());
223+
MOZ_ASSERT(sample);
224+
225+
mInputSampleDuration = s->mDuration;
226+
227+
// Bug 1789846: Check with the Encoder if MediaCodec has a stride or height
228+
// value to use.
229+
jni::ByteBuffer::LocalRef buffer = ConvertI420ToNV12Buffer(
230+
sample, mYUVBuffer, mJavaEncoder->GetInputFormatStride(),
231+
mJavaEncoder->GetInputFormatYPlaneHeight());
232+
if (!buffer) {
233+
return EncodePromise::CreateAndReject(NS_ERROR_ILLEGAL_INPUT, __func__);
234+
}
205235

206-
// Bug 1789846: Check with the Encoder if MediaCodec has a stride or height
207-
// value to use.
208-
jni::ByteBuffer::LocalRef buffer = ConvertI420ToNV12Buffer(
209-
sample, mYUVBuffer, mJavaEncoder->GetInputFormatStride(),
210-
mJavaEncoder->GetInputFormatYPlaneHeight());
211-
if (!buffer) {
212-
return EncodePromise::CreateAndReject(NS_ERROR_ILLEGAL_INPUT, __func__);
213-
}
236+
if (s->mKeyframe) {
237+
mInputBufferInfo->Set(0, AssertedCast<int32_t>(mYUVBuffer->Length()),
238+
s->mTime.ToMicroseconds(),
239+
java::sdk::MediaCodec::BUFFER_FLAG_SYNC_FRAME);
240+
} else {
241+
mInputBufferInfo->Set(0, AssertedCast<int32_t>(mYUVBuffer->Length()),
242+
s->mTime.ToMicroseconds(), 0);
243+
}
214244

215-
if (aSample->mKeyframe) {
216-
mInputBufferInfo->Set(0, AssertedCast<int32_t>(mYUVBuffer->Length()),
217-
aSample->mTime.ToMicroseconds(),
218-
java::sdk::MediaCodec::BUFFER_FLAG_SYNC_FRAME);
219-
} else {
220-
mInputBufferInfo->Set(0, AssertedCast<int32_t>(mYUVBuffer->Length()),
221-
aSample->mTime.ToMicroseconds(), 0);
245+
mJavaEncoder->Input(buffer, mInputBufferInfo, nullptr);
222246
}
223247

224-
mJavaEncoder->Input(buffer, mInputBufferInfo, nullptr);
225-
226248
if (mEncodedData.Length() > 0) {
227249
EncodedData pending = std::move(mEncodedData);
228250
return EncodePromise::CreateAndResolve(std::move(pending), __func__);

‎dom/media/platforms/android/AndroidDataEncoder.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ class AndroidDataEncoder final : public MediaDataEncoder {
2626

2727
RefPtr<InitPromise> Init() override;
2828
RefPtr<EncodePromise> Encode(const MediaData* aSample) override;
29+
RefPtr<EncodePromise> Encode(nsTArray<RefPtr<MediaData>>&& aSamples) override;
2930
RefPtr<EncodePromise> Drain() override;
3031
RefPtr<ShutdownPromise> Shutdown() override;
3132
RefPtr<GenericPromise> SetBitrate(uint32_t aBitsPerSec) override;
@@ -70,7 +71,7 @@ class AndroidDataEncoder final : public MediaDataEncoder {
7071

7172
// Methods only called on mTaskQueue.
7273
RefPtr<InitPromise> ProcessInit();
73-
RefPtr<EncodePromise> ProcessEncode(const RefPtr<const MediaData>& aSample);
74+
RefPtr<EncodePromise> ProcessEncode(nsTArray<RefPtr<MediaData>>&& aSamples);
7475
RefPtr<EncodePromise> ProcessDrain();
7576
RefPtr<ShutdownPromise> ProcessShutdown();
7677
void ProcessInput();

‎dom/media/platforms/apple/AppleVTEncoder.cpp‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -871,6 +871,25 @@ RefPtr<MediaDataEncoder::EncodePromise> AppleVTEncoder::Encode(
871871
});
872872
}
873873

874+
// TODO(Bug 1984936): For realtime mode, resolve the promise after the first
875+
// sample's result is available, then continue processing remaining samples.
876+
// This allows the caller to keep submitting new samples while the encoder
877+
// handles pending ones.
878+
RefPtr<MediaDataEncoder::EncodePromise> AppleVTEncoder::Encode(
879+
nsTArray<RefPtr<MediaData>>&& aSamples) {
880+
MOZ_ASSERT(!aSamples.IsEmpty());
881+
882+
RefPtr<AppleVTEncoder> self = this;
883+
return InvokeAsync(
884+
mTaskQueue, __func__, [self, samples = std::move(aSamples)]() mutable {
885+
MOZ_ASSERT(self->mEncodeBatchPromise.IsEmpty(),
886+
"Encode should not be called again before getting results");
887+
RefPtr<EncodePromise> p = self->mEncodeBatchPromise.Ensure(__func__);
888+
self->EncodeNextSample(std::move(samples), EncodedData());
889+
return p;
890+
});
891+
}
892+
874893
RefPtr<MediaDataEncoder::ReconfigurationPromise> AppleVTEncoder::Reconfigure(
875894
const RefPtr<const EncoderConfigurationChangeList>& aConfigurationChanges) {
876895
return InvokeAsync(mTaskQueue, this, __func__,
@@ -1271,6 +1290,39 @@ void AppleVTEncoder::ForceOutputIfNeeded() {
12711290
mTimer = r.unwrap();
12721291
}
12731292

1293+
void AppleVTEncoder::EncodeNextSample(
1294+
nsTArray<RefPtr<MediaData>>&& aInputs,
1295+
MediaDataEncoder::EncodedData&& aOutputs) {
1296+
AssertOnTaskQueue();
1297+
MOZ_ASSERT(!mEncodeBatchPromise.IsEmpty());
1298+
MOZ_ASSERT(!mEncodeBatchRequest.Exists());
1299+
1300+
if (aInputs.IsEmpty()) {
1301+
LOGV("All samples processed. Resolving the encode promise");
1302+
mEncodeBatchPromise.Resolve(std::move(aOutputs), __func__);
1303+
return;
1304+
}
1305+
1306+
LOGV("Processing next sample out of %zu remaining", aInputs.Length());
1307+
Encode(aInputs[0])
1308+
->Then(
1309+
GetCurrentSerialEventTarget(), __func__,
1310+
[self = RefPtr{this}, inputs = std::move(aInputs),
1311+
outputs = std::move(aOutputs)](
1312+
MediaDataEncoder::EncodedData&& aData) mutable {
1313+
self->mEncodeBatchRequest.Complete();
1314+
inputs.RemoveElementAt(0);
1315+
outputs.AppendElements(aData);
1316+
self->EncodeNextSample(std::move(inputs), std::move(outputs));
1317+
},
1318+
[self = RefPtr{this}](const MediaResult& aError) {
1319+
self->mEncodeBatchRequest.Complete();
1320+
LOGE("EncodeNextSample failed: %s", aError.Description().get());
1321+
self->mEncodeBatchPromise.Reject(aError, __func__);
1322+
})
1323+
->Track(mEncodeBatchRequest);
1324+
}
1325+
12741326
#undef LOGE
12751327
#undef LOGW
12761328
#undef LOGD

‎dom/media/platforms/apple/AppleVTEncoder.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ class AppleVTEncoder final : public MediaDataEncoder {
3838

3939
RefPtr<InitPromise> Init() override;
4040
RefPtr<EncodePromise> Encode(const MediaData* aSample) override;
41+
RefPtr<EncodePromise> Encode(nsTArray<RefPtr<MediaData>>&& aSamples) override;
4142
RefPtr<ReconfigurationPromise> Reconfigure(
4243
const RefPtr<const EncoderConfigurationChangeList>& aConfigurationChanges)
4344
override;
@@ -86,6 +87,9 @@ class AppleVTEncoder final : public MediaDataEncoder {
8687
bool IsSettingColorSpaceSupported() const;
8788
MediaResult SetColorSpace(const EncoderConfig::SampleFormat& aFormat);
8889

90+
void EncodeNextSample(nsTArray<RefPtr<MediaData>>&& aInputs,
91+
MediaDataEncoder::EncodedData&& aOutputs);
92+
8993
void AssertOnTaskQueue() { MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); }
9094

9195
EncoderConfig mConfig;
@@ -95,6 +99,8 @@ class AppleVTEncoder final : public MediaDataEncoder {
9599
EncodedData mEncodedData;
96100
// Accessed only in mTaskQueue.
97101
MozPromiseHolder<EncodePromise> mEncodePromise;
102+
MozPromiseHolder<EncodePromise> mEncodeBatchPromise;
103+
MozPromiseRequestHolder<EncodePromise> mEncodeBatchRequest;
98104
RefPtr<MediaByteBuffer> mAvcc; // Stores latest avcC data.
99105
MediaResult mError;
100106

‎dom/media/platforms/ffmpeg/FFmpegDataEncoder.cpp‎

Lines changed: 28 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -201,10 +201,23 @@ RefPtr<MediaDataEncoder::EncodePromise> FFmpegDataEncoder<LIBAV_VER>::Encode(
201201
MOZ_ASSERT(aSample != nullptr);
202202

203203
FFMPEG_LOG("Encode");
204+
return InvokeAsync(
205+
mTaskQueue, __func__,
206+
[self = RefPtr<FFmpegDataEncoder<LIBAV_VER>>(this),
207+
sample = RefPtr<MediaData>(const_cast<MediaData*>(aSample))]() {
208+
return self->ProcessEncode({sample});
209+
});
210+
}
211+
212+
RefPtr<MediaDataEncoder::EncodePromise> FFmpegDataEncoder<LIBAV_VER>::Encode(
213+
nsTArray<RefPtr<MediaData>>&& aSamples) {
214+
MOZ_ASSERT(!aSamples.IsEmpty());
215+
216+
FFMPEG_LOG("Encode: %zu samples", aSamples.Length());
204217
return InvokeAsync(mTaskQueue, __func__,
205218
[self = RefPtr<FFmpegDataEncoder<LIBAV_VER>>(this),
206-
sample = RefPtr<const MediaData>(aSample)]() {
207-
return self->ProcessEncode(sample);
219+
samples = std::move(aSamples)]() mutable {
220+
return self->ProcessEncode(std::move(samples));
208221
});
209222
}
210223

@@ -234,26 +247,28 @@ RefPtr<GenericPromise> FFmpegDataEncoder<LIBAV_VER>::SetBitrate(
234247
return GenericPromise::CreateAndReject(NS_ERROR_NOT_IMPLEMENTED, __func__);
235248
}
236249

237-
RefPtr<MediaDataEncoder::EncodePromise>
238-
FFmpegDataEncoder<LIBAV_VER>::ProcessEncode(RefPtr<const MediaData> aSample) {
250+
RefPtr<MediaDataEncoder::EncodePromise> FFmpegDataEncoder<
251+
LIBAV_VER>::ProcessEncode(nsTArray<RefPtr<MediaData>>&& aSamples) {
239252
MOZ_ASSERT(mTaskQueue->IsOnCurrentThread());
240253

241-
FFMPEG_LOG("ProcessEncode");
254+
FFMPEG_LOG("ProcessEncode: %zu samples", aSamples.Length());
242255

243256
#if LIBAVCODEC_VERSION_MAJOR < 58
244257
// TODO(Bug 1868253): implement encode with avcodec_encode_video2().
245258
MOZ_CRASH("FFmpegDataEncoder needs ffmpeg 58 at least.");
246259
return EncodePromise::CreateAndReject(NS_ERROR_NOT_IMPLEMENTED, __func__);
247260
#else
248-
249-
auto rv = EncodeInputWithModernAPIs(std::move(aSample));
250-
if (rv.isErr()) {
251-
MediaResult e = rv.unwrapErr();
252-
FFMPEG_LOG("%s", e.Description().get());
253-
return EncodePromise::CreateAndReject(e, __func__);
261+
EncodedData output;
262+
for (auto& sample : aSamples) {
263+
auto rv = EncodeInputWithModernAPIs(sample);
264+
if (rv.isErr()) {
265+
MediaResult e = rv.unwrapErr();
266+
FFMPEG_LOG("%s", e.Description().get());
267+
return EncodePromise::CreateAndReject(e, __func__);
268+
}
269+
output.AppendElements(rv.unwrap());
254270
}
255-
256-
return EncodePromise::CreateAndResolve(rv.unwrap(), __func__);
271+
return EncodePromise::CreateAndResolve(std::move(output), __func__);
257272
#endif
258273
}
259274

‎dom/media/platforms/ffmpeg/FFmpegDataEncoder.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ class FFmpegDataEncoder<LIBAV_VER> : public MediaDataEncoder {
4646
// All methods run on the task queue, except for GetDescriptionName.
4747
RefPtr<InitPromise> Init() override = 0; // Implemented in the sub-classes.
4848
RefPtr<EncodePromise> Encode(const MediaData* aSample) override;
49+
RefPtr<EncodePromise> Encode(nsTArray<RefPtr<MediaData>>&& aSamples) override;
4950
RefPtr<ReconfigurationPromise> Reconfigure(
5051
const RefPtr<const EncoderConfigurationChangeList>& aConfigurationChanges)
5152
override;
@@ -62,7 +63,7 @@ class FFmpegDataEncoder<LIBAV_VER> : public MediaDataEncoder {
6263
AVPacket* aPacket);
6364

6465
// Methods only called on mTaskQueue.
65-
RefPtr<EncodePromise> ProcessEncode(RefPtr<const MediaData> aSample);
66+
RefPtr<EncodePromise> ProcessEncode(nsTArray<RefPtr<MediaData>>&& aSamples);
6667
RefPtr<ReconfigurationPromise> ProcessReconfigure(
6768
const RefPtr<const EncoderConfigurationChangeList>&
6869
aConfigurationChanges);

0 commit comments

Comments
 (0)