Skip to content

Commit eb6516a

Browse files
ChunMinChangChunMinChang
committed
Bug 1978232 - Enable batch encoding for RemoteMediaDataEncoder r=media-playback-reviewers,padenot
Follwing the preceding patch, this patch enables batch encoding for `RemoteMediaDataEncoder{Child, Parent}`. Differential Revision: https://phabricator.services.mozilla.com/D261494
1 parent a459a99 commit eb6516a

File tree

4 files changed

+160
-90
lines changed

4 files changed

+160
-90
lines changed

‎dom/media/ipc/PRemoteEncoder.ipdl‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ include protocol PRemoteMediaManager;
1111

1212
using mozilla::EncoderConfig from "EncoderConfig.h";
1313
using mozilla::MediaResult from "MediaResult.h";
14-
using mozilla::TrackInfo::TrackType from "MediaInfo.h";
1514
[RefCounted] using class mozilla::ArrayOfRemoteMediaRawData from "mozilla/RemoteMediaData.h";
1615
[RefCounted] using class mozilla::ArrayOfRemoteAudioData from "mozilla/RemoteMediaData.h";
1716
[RefCounted] using class mozilla::ArrayOfRemoteVideoData from "mozilla/RemoteMediaData.h";

‎dom/media/ipc/RemoteMediaDataEncoderChild.cpp‎

Lines changed: 67 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,6 @@ void RemoteMediaDataEncoderChild::DoSendInit() {
123123

124124
self->mIsHardwareAccelerated = initResponse.hardware();
125125
self->mHardwareAcceleratedReason = initResponse.hardwareReason();
126-
// TODO: Get batch-encoding capability.
127126
self->mInitPromise.ResolveIfExists(true, __func__);
128127
},
129128
[self = RefPtr{this}](const mozilla::ipc::ResponseRejectReason& aReason) {
@@ -154,8 +153,8 @@ RefPtr<MediaDataEncoder::InitPromise> RemoteMediaDataEncoderChild::Init() {
154153
}
155154

156155
RefPtr<PRemoteEncoderChild::EncodePromise>
157-
RemoteMediaDataEncoderChild::DoSendEncode(const MediaData* aSample,
158-
ShmemRecycleTicket* aTicket) {
156+
RemoteMediaDataEncoderChild::DoSendEncode(
157+
const nsTArray<RefPtr<MediaData>>& aSamples, ShmemRecycleTicket* aTicket) {
159158
if (mRemoteCrashed) {
160159
LOGE("[{}] remote crashed", fmt::ptr(this));
161160
nsresult err = NS_ERROR_DOM_MEDIA_REMOTE_CRASHED_UTILITY_ERR;
@@ -169,53 +168,72 @@ RemoteMediaDataEncoderChild::DoSendEncode(const MediaData* aSample,
169168
MediaResult(err, "Remote process crashed"), __func__);
170169
}
171170

172-
if (aSample->mType == MediaData::Type::AUDIO_DATA) {
171+
if (aSamples.IsEmpty()) {
172+
LOGE("[{}] no samples to encode", fmt::ptr(this));
173+
return PRemoteEncoderChild::EncodePromise::CreateAndResolve(
174+
MediaResult(NS_ERROR_INVALID_ARG), __func__);
175+
}
176+
177+
MediaData::Type type = aSamples[0]->mType;
178+
179+
if (type == MediaData::Type::AUDIO_DATA) {
180+
nsTArray<RefPtr<AudioData>> audioSamples;
181+
for (auto& sample : aSamples) {
182+
audioSamples.AppendElement(sample->As<AudioData>());
183+
}
184+
173185
auto samples = MakeRefPtr<ArrayOfRemoteAudioData>();
174-
if (!samples->Fill(aSample->As<const AudioData>(), [&](size_t aSize) {
186+
if (!samples->Fill(audioSamples, [&](size_t aSize) {
175187
return AllocateBuffer(aSize, aTicket);
176188
})) {
177189
LOGE("[{}] buffer audio failed", fmt::ptr(this));
178190
return PRemoteEncoderChild::EncodePromise::CreateAndResolve(
179191
MediaResult(NS_ERROR_OUT_OF_MEMORY), __func__);
180192
}
181-
LOGD("[{}] send audio", fmt::ptr(this));
193+
LOGD("[{}] send {} audio samples", fmt::ptr(this), audioSamples.Length());
182194
return SendEncode(std::move(samples));
183195
}
184196

185-
if (aSample->mType == MediaData::Type::VIDEO_DATA) {
197+
if (type == MediaData::Type::VIDEO_DATA) {
198+
nsTArray<RefPtr<VideoData>> videoSamples;
199+
for (auto& sample : aSamples) {
200+
videoSamples.AppendElement(sample->As<VideoData>());
201+
}
202+
186203
auto samples = MakeRefPtr<ArrayOfRemoteVideoData>();
187-
const auto* videoSample = aSample->As<const VideoData>();
188-
if (layers::Image* videoImage = videoSample->mImage) {
189-
// We don't need to supply a working deallocator because the ticket is
190-
// responsible for that cleanup.
191-
layers::SurfaceDescriptor sd;
192-
nsresult rv = videoImage->BuildSurfaceDescriptorGPUVideoOrBuffer(
193-
sd, layers::Image::BuildSdbFlags::Default,
194-
Some(GetVideoBridgeSourceFromRemoteMediaIn(mLocation)),
195-
[&](uint32_t aBufferSize) {
196-
ShmemBuffer buffer = AllocateBuffer(aBufferSize, aTicket);
197-
if (buffer.Valid()) {
198-
return layers::MemoryOrShmem(std::move(buffer.Get()));
199-
}
200-
return layers::MemoryOrShmem();
201-
},
202-
[&](layers::MemoryOrShmem&&) {});
203-
204-
if (NS_WARN_IF(NS_FAILED(rv))) {
205-
LOGE("[{}] buffer video failed, code={}", fmt::ptr(this),
206-
fmt::enums::format_as(rv));
207-
return PRemoteEncoderChild::EncodePromise::CreateAndResolve(
208-
MediaResult(rv), __func__);
209-
}
204+
for (const auto& videoSample : videoSamples) {
205+
if (layers::Image* videoImage = videoSample->mImage) {
206+
// We don't need to supply a working deallocator because the ticket is
207+
// responsible for that cleanup.
208+
layers::SurfaceDescriptor sd;
209+
nsresult rv = videoImage->BuildSurfaceDescriptorGPUVideoOrBuffer(
210+
sd, layers::Image::BuildSdbFlags::Default,
211+
Some(GetVideoBridgeSourceFromRemoteMediaIn(mLocation)),
212+
[&](uint32_t aBufferSize) {
213+
ShmemBuffer buffer = AllocateBuffer(aBufferSize, aTicket);
214+
if (buffer.Valid()) {
215+
return layers::MemoryOrShmem(std::move(buffer.Get()));
216+
}
217+
return layers::MemoryOrShmem();
218+
},
219+
[&](layers::MemoryOrShmem&&) {});
220+
221+
if (NS_WARN_IF(NS_FAILED(rv))) {
222+
LOGE("[{}] buffer video failed, code={}", fmt::ptr(this),
223+
fmt::enums::format_as(rv));
224+
return PRemoteEncoderChild::EncodePromise::CreateAndResolve(
225+
MediaResult(rv), __func__);
226+
}
210227

211-
samples->Append(RemoteVideoData(
212-
MediaDataIPDL(videoSample->mOffset, videoSample->mTime,
213-
videoSample->mTimecode, videoSample->mDuration,
214-
videoSample->mKeyframe),
215-
videoSample->mDisplay, RemoteImageHolder(std::move(sd)),
216-
videoSample->mFrameID));
228+
samples->Append(RemoteVideoData(
229+
MediaDataIPDL(videoSample->mOffset, videoSample->mTime,
230+
videoSample->mTimecode, videoSample->mDuration,
231+
videoSample->mKeyframe),
232+
videoSample->mDisplay, RemoteImageHolder(std::move(sd)),
233+
videoSample->mFrameID));
234+
}
217235
}
218-
LOGD("[{}] send video", fmt::ptr(this));
236+
LOGD("[{}] send {} video samples", fmt::ptr(this), videoSamples.Length());
219237
return SendEncode(std::move(samples));
220238
}
221239

@@ -225,14 +243,19 @@ RemoteMediaDataEncoderChild::DoSendEncode(const MediaData* aSample,
225243

226244
RefPtr<MediaDataEncoder::EncodePromise> RemoteMediaDataEncoderChild::Encode(
227245
const MediaData* aSample) {
246+
return Encode(nsTArray<RefPtr<MediaData>>{const_cast<MediaData*>(aSample)});
247+
}
248+
249+
RefPtr<MediaDataEncoder::EncodePromise> RemoteMediaDataEncoderChild::Encode(
250+
nsTArray<RefPtr<MediaData>>&& aSamples) {
228251
return InvokeAsync(
229252
mThread, __func__,
230-
[self = RefPtr{this},
231-
sample = RefPtr{aSample}]() -> RefPtr<MediaDataEncoder::EncodePromise> {
253+
[self = RefPtr{this}, samples = std::move(aSamples)]()
254+
-> RefPtr<MediaDataEncoder::EncodePromise> {
232255
auto promise =
233256
MakeRefPtr<MediaDataEncoder::EncodePromise::Private>(__func__);
234257
auto ticket = MakeRefPtr<ShmemRecycleTicket>();
235-
self->DoSendEncode(sample, ticket)
258+
self->DoSendEncode(samples, ticket)
236259
->Then(
237260
self->mThread, __func__,
238261
[self, promise, ticket](EncodeResultIPDL&& aResponse) {
@@ -259,8 +282,10 @@ RefPtr<MediaDataEncoder::EncodePromise> RemoteMediaDataEncoderChild::Encode(
259282
marker.End();
260283
samples.AppendElement(std::move(sample));
261284
} else {
262-
LOGE("[{}] Encode resolved, failed to buffer samples",
263-
fmt::ptr(self.get()));
285+
LOGE(
286+
"[{}] Encode resolved, failed to buffer "
287+
"samples",
288+
fmt::ptr(self.get()));
264289
promise->Reject(MediaResult(NS_ERROR_OUT_OF_MEMORY),
265290
__func__);
266291
return;

‎dom/media/ipc/RemoteMediaDataEncoderChild.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ class RemoteMediaDataEncoderChild final
4343
RefPtr<MediaDataEncoder::InitPromise> Init() override;
4444
RefPtr<MediaDataEncoder::EncodePromise> Encode(
4545
const MediaData* aSample) override;
46+
RefPtr<MediaDataEncoder::EncodePromise> Encode(
47+
nsTArray<RefPtr<MediaData>>&& aSamples) override;
4648
RefPtr<MediaDataEncoder::EncodePromise> Drain() override;
4749
RefPtr<MediaDataEncoder::ReconfigurationPromise> Reconfigure(
4850
const RefPtr<const EncoderConfigurationChangeList>& aConfigurationChanges)
@@ -57,7 +59,7 @@ class RemoteMediaDataEncoderChild final
5759
RemoteMediaManagerChild* GetManager();
5860

5961
virtual RefPtr<PRemoteEncoderChild::EncodePromise> DoSendEncode(
60-
const MediaData* aSample, ShmemRecycleTicket* aTicket);
62+
const nsTArray<RefPtr<MediaData>>& aSamples, ShmemRecycleTicket* aTicket);
6163

6264
void DoSendInit();
6365
void MaybeDestroyActor();

‎dom/media/ipc/RemoteMediaDataEncoderParent.cpp‎

Lines changed: 90 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,24 @@
1313

1414
namespace mozilla {
1515

16+
extern LazyLogModule sPEMLog;
17+
18+
#define LOG_INTERNAL(level, fmt, ...) \
19+
MOZ_LOG_FMT(sPEMLog, LogLevel::level, \
20+
"[RemoteMediaDataEncoderParent] {}: " fmt, __func__, \
21+
__VA_ARGS__)
22+
#define LOGE(fmt, ...) LOG_INTERNAL(Error, fmt, __VA_ARGS__)
23+
#define LOGW(fmt, ...) LOG_INTERNAL(Warning, fmt, __VA_ARGS__)
24+
#define LOGD(fmt, ...) LOG_INTERNAL(Debug, fmt, __VA_ARGS__)
25+
#define LOGV(fmt, ...) LOG_INTERNAL(Verbose, fmt, __VA_ARGS__)
26+
27+
#define LOGE_IF(condition, fmt, ...) \
28+
do { \
29+
if (condition) { \
30+
LOGE(fmt, __VA_ARGS__); \
31+
} \
32+
} while (0)
33+
1634
#define AUTO_MARKER(var, postfix) \
1735
AutoWebCodecsMarker var("RemoteMediaDataEncoderParent", postfix);
1836

@@ -72,7 +90,6 @@ IPCResult RemoteMediaDataEncoderParent::RecvInit(InitResolver&& aResolver) {
7290
return;
7391
}
7492

75-
// TODO: Pass batch-encoding capability.
7693
nsCString hardwareReason;
7794
bool hardware = encoder->IsHardwareAccelerated(hardwareReason);
7895
resolver(EncodeInitCompletionIPDL{encoder->GetDescriptionName(),
@@ -88,75 +105,96 @@ IPCResult RemoteMediaDataEncoderParent::RecvEncode(
88105
return IPC_OK();
89106
}
90107

91-
RefPtr<MediaData> frame;
108+
nsTArray<RefPtr<MediaData>> frames;
92109

93110
if (mConfig.IsAudio() &&
94111
aData.type() == EncodedInputIPDL::TArrayOfRemoteAudioData) {
95112
auto remoteAudioArray = aData.get_ArrayOfRemoteAudioData();
96-
if (remoteAudioArray->Count() != 1) {
113+
if (remoteAudioArray->IsEmpty()) {
114+
LOGE("[{}] no audio frames received", fmt::ptr(this));
97115
aResolver(MediaResult(NS_ERROR_INVALID_ARG, __func__));
98116
return IPC_OK();
99117
}
100118

101-
frame = remoteAudioArray->ElementAt(0).downcast<MediaData>();
119+
LOGV("[{}] recv {} audio frames", fmt::ptr(this),
120+
remoteAudioArray->Count());
121+
for (size_t i = 0; i < remoteAudioArray->Count(); i++) {
122+
frames.AppendElement(
123+
remoteAudioArray->ElementAt(i).downcast<MediaData>());
124+
}
102125
} else if (mConfig.IsVideo() &&
103126
aData.type() == EncodedInputIPDL::TArrayOfRemoteVideoData) {
104127
auto remoteVideoArray = aData.get_ArrayOfRemoteVideoData();
105-
if (remoteVideoArray->Array().Length() != 1) {
128+
if (remoteVideoArray->Array().IsEmpty()) {
129+
LOGE("[{}] no video frames received", fmt::ptr(this));
106130
aResolver(MediaResult(NS_ERROR_INVALID_ARG, __func__));
107131
return IPC_OK();
108132
}
109133

110-
auto data = std::move(remoteVideoArray->Array().LastElement());
111-
if (!data.image().IsEmpty()) {
112-
AUTO_MARKER(marker, ".RecvEncode.TransferToImage");
113-
RefPtr<layers::Image> image =
114-
data.image().TransferToImage(mBufferRecycleBin);
115-
marker.End();
116-
if (image) {
117-
frame = VideoData::CreateFromImage(
118-
data.display(), data.base().offset(), data.base().time(),
119-
data.base().duration(), image, data.base().keyframe(),
120-
data.base().timecode())
121-
.downcast<MediaData>();
134+
LOGV("[{}] recv {} video frames", fmt::ptr(this),
135+
remoteVideoArray->Array().Length());
136+
for (size_t i = 0; i < remoteVideoArray->Array().Length(); i++) {
137+
RefPtr<MediaData> frame;
138+
auto data = std::move(remoteVideoArray->Array().ElementAt(i));
139+
if (!data.image().IsEmpty()) {
140+
AUTO_MARKER(marker, ".RecvEncode.TransferToImage");
141+
RefPtr<layers::Image> image =
142+
data.image().TransferToImage(mBufferRecycleBin);
143+
marker.End();
144+
LOGE_IF(!image, "[{}] failed to get image from video frame at index {}",
145+
fmt::ptr(this), i);
146+
if (image) {
147+
frame = VideoData::CreateFromImage(
148+
data.display(), data.base().offset(), data.base().time(),
149+
data.base().duration(), image, data.base().keyframe(),
150+
data.base().timecode())
151+
.downcast<MediaData>();
152+
}
153+
} else {
154+
LOGW("[{}] empty image in video frame at index {}", fmt::ptr(this), i);
155+
frame = MakeRefPtr<NullData>(data.base().offset(), data.base().time(),
156+
data.base().duration());
157+
}
158+
159+
if (NS_WARN_IF(!frame)) {
160+
LOGE("[{}] failed to create video frame", fmt::ptr(this));
161+
aResolver(MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__));
162+
return IPC_OK();
122163
}
123-
} else {
124-
frame = MakeRefPtr<NullData>(data.base().offset(), data.base().time(),
125-
data.base().duration());
164+
165+
frames.AppendElement(std::move(frame));
126166
}
127167
} else {
168+
LOGE("[{}] invalid input data type", fmt::ptr(this));
128169
aResolver(MediaResult(NS_ERROR_INVALID_ARG, __func__));
129170
return IPC_OK();
130171
}
131172

132-
if (NS_WARN_IF(!frame)) {
133-
aResolver(MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__));
134-
return IPC_OK();
135-
}
136-
137-
mEncoder->Encode(frame)->Then(
138-
GetCurrentSerialEventTarget(), __func__,
139-
[self = RefPtr{this}, resolver = std::move(aResolver)](
140-
MediaDataEncoder::EncodePromise::ResolveOrRejectValue&& aValue) {
141-
if (aValue.IsReject()) {
142-
resolver(aValue.RejectValue());
143-
return;
144-
}
173+
LOGV("[{}] encoding {} frames", fmt::ptr(this), frames.Length());
174+
mEncoder->Encode(std::move(frames))
175+
->Then(
176+
GetCurrentSerialEventTarget(), __func__,
177+
[self = RefPtr{this}, resolver = std::move(aResolver)](
178+
MediaDataEncoder::EncodePromise::ResolveOrRejectValue&& aValue) {
179+
if (aValue.IsReject()) {
180+
resolver(aValue.RejectValue());
181+
return;
182+
}
145183

146-
auto ticket = MakeRefPtr<ShmemRecycleTicket>();
147-
auto samples = MakeRefPtr<ArrayOfRemoteMediaRawData>();
148-
if (!samples->Fill(aValue.ResolveValue(), [&](size_t aSize) {
149-
return self->AllocateBuffer(aSize, ticket);
150-
})) {
151-
self->ReleaseTicket(ticket);
152-
resolver(MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__));
153-
return;
154-
}
184+
auto ticket = MakeRefPtr<ShmemRecycleTicket>();
185+
auto samples = MakeRefPtr<ArrayOfRemoteMediaRawData>();
186+
if (!samples->Fill(aValue.ResolveValue(), [&](size_t aSize) {
187+
return self->AllocateBuffer(aSize, ticket);
188+
})) {
189+
self->ReleaseTicket(ticket);
190+
resolver(MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__));
191+
return;
192+
}
155193

156-
uint32_t ticketId = ++self->mTicketCounter;
157-
self->mTickets[ticketId] = std::move(ticket);
158-
resolver(EncodeCompletionIPDL(samples, ticketId));
159-
});
194+
uint32_t ticketId = ++self->mTicketCounter;
195+
self->mTickets[ticketId] = std::move(ticket);
196+
resolver(EncodeCompletionIPDL(samples, ticketId));
197+
});
160198
return IPC_OK();
161199
}
162200

@@ -273,6 +311,12 @@ void RemoteMediaDataEncoderParent::ActorDestroy(ActorDestroyReason aWhy) {
273311
CleanupShmemRecycleAllocator();
274312
}
275313

314+
#undef LOGE
315+
#undef LOGW
316+
#undef LOGD
317+
#undef LOGV
318+
#undef LOGE_IF
319+
#undef LOG_INTERNAL
276320
#undef AUTO_MARKER
277321

278322
} // namespace mozilla

0 commit comments

Comments
 (0)