Skip to content

Commit

Permalink
fix: use a better estimate of frame rate for cases with very short fi…
Browse files Browse the repository at this point in the history 
…rst sample durations (#838)

Use the second sample in mp4 and webm formats. #835 had issues with
merging due to golden file conflicts. Because we cannot make dependent
pull requests, this is a replica of #835.

---------

Signed-off-by: Cosmin Stejerean <cstejerean@meta.com>
Co-authored-by: Cosmin Stejerean <cstejerean@meta.com>
  • Loading branch information
@koln67 @cosmin
koln67 and cosmin committed Feb 28, 2024
1 parent 4d22e99 commit 5644041
Show file tree
Hide file tree
Showing 19 changed files with 44 additions and 27 deletions.
4 changes: 2 additions & 2 deletions packager/app/test/testdata/audio-video-with-two-trick-play/output.mpd
View file
Expand Up @@ -19,9 +19,9 @@
</SegmentBase>
</Representation>
</AdaptationSet>
<AdaptationSet id="2" contentType="video" width="640" height="360" maxFrameRate="30000/30030" par="16:9">
<AdaptationSet id="2" contentType="video" width="640" height="360" maxFrameRate="30000/22022" par="16:9">
<EssentialProperty schemeIdUri="http://dashif.org/guidelines/trickmode" value="1"/>
<Representation id="2" bandwidth="211545" codecs="avc1.64001e" mimeType="video/mp4" sar="1:1" frameRate="30000/60060" maxPlayoutRate="60" codingDependency="false">
<Representation id="2" bandwidth="211545" codecs="avc1.64001e" mimeType="video/mp4" sar="1:1" frameRate="30000/22022" maxPlayoutRate="60" codingDependency="false">
<BaseURL>bear-640x360-video-trick_play_factor_2.mp4</BaseURL>
<SegmentBase indexRange="870-925" timescale="30000">
<Initialization range="0-869"/>
Expand Down
2 changes: 1 addition & 1 deletion packager/app/test/testdata/ec3-and-hls-single-segment-mp4-encrypted/output.m3u8
View file
Expand Up @@ -5,7 +5,7 @@

#EXT-X-MEDIA:TYPE=AUDIO,URI="bear-640x360-ec3-audio.m3u8",GROUP-ID="default-audio-group",NAME="stream_0",DEFAULT=NO,AUTOSELECT=YES,CHANNELS="2"

#EXT-X-STREAM-INF:BANDWIDTH=1174214,AVERAGE-BANDWIDTH=1061802,CODECS="avc1.64001e,ec-3",RESOLUTION=640x360,FRAME-RATE=9.990,AUDIO="default-audio-group",CLOSED-CAPTIONS=NONE
#EXT-X-STREAM-INF:BANDWIDTH=1174214,AVERAGE-BANDWIDTH=1061802,CODECS="avc1.64001e,ec-3",RESOLUTION=640x360,FRAME-RATE=29.970,AUDIO="default-audio-group",CLOSED-CAPTIONS=NONE
bear-640x360-ec3-video.m3u8

#EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=218705,AVERAGE-BANDWIDTH=159315,CODECS="avc1.64001e",RESOLUTION=640x360,CLOSED-CAPTIONS=NONE,URI="bear-640x360-ec3-video-iframe.m3u8"
4 changes: 2 additions & 2 deletions packager/app/test/testdata/encryption-and-two-trick-plays/output.mpd
View file
Expand Up @@ -27,7 +27,7 @@
</SegmentBase>
</Representation>
</AdaptationSet>
<AdaptationSet id="2" contentType="video" width="640" height="360" maxFrameRate="30000/30030" par="16:9">
<AdaptationSet id="2" contentType="video" width="640" height="360" maxFrameRate="30000/22022" par="16:9">
<ContentProtection value="cenc" schemeIdUri="urn:mpeg:dash:mp4protection:2011" cenc:default_KID="31323334-3536-3738-3930-313233343536"/>
<ContentProtection schemeIdUri="urn:uuid:1077efec-c0b2-4d02-ace3-3c1e52e2fb4b">
<cenc:pssh>AAAANHBzc2gBAAAAEHfv7MCyTQKs4zweUuL7SwAAAAExMjM0NTY3ODkwMTIzNDU2AAAAAA==</cenc:pssh>
Expand All @@ -39,7 +39,7 @@
<Initialization range="0-1137"/>
</SegmentBase>
</Representation>
<Representation id="3" bandwidth="212297" codecs="avc1.64001e" mimeType="video/mp4" sar="1:1" frameRate="30000/60060" maxPlayoutRate="60" codingDependency="false">
<Representation id="3" bandwidth="212297" codecs="avc1.64001e" mimeType="video/mp4" sar="1:1" frameRate="30000/22022" maxPlayoutRate="60" codingDependency="false">
<BaseURL>bear-640x360-video-trick_play_factor_2.mp4</BaseURL>
<SegmentBase indexRange="1138-1193" timescale="30000">
<Initialization range="0-1137"/>
Expand Down
2 changes: 1 addition & 1 deletion packager/app/test/testdata/live-profile-with-webm/output.m3u8
View file
Expand Up @@ -5,5 +5,5 @@

#EXT-X-MEDIA:TYPE=AUDIO,URI="stream_0.m3u8",GROUP-ID="default-audio-group",NAME="stream_0",DEFAULT=NO,AUTOSELECT=YES,CHANNELS="2"

#EXT-X-STREAM-INF:BANDWIDTH=556353,AVERAGE-BANDWIDTH=412719,CODECS="vp08.00.10.08.01.02.02.02.00,vorbis",RESOLUTION=640x360,FRAME-RATE=30.303,AUDIO="default-audio-group",CLOSED-CAPTIONS=NONE
#EXT-X-STREAM-INF:BANDWIDTH=556353,AVERAGE-BANDWIDTH=412719,CODECS="vp08.00.10.08.01.02.02.02.00,vorbis",RESOLUTION=640x360,FRAME-RATE=29.412,AUDIO="default-audio-group",CLOSED-CAPTIONS=NONE
stream_1.m3u8
2 changes: 1 addition & 1 deletion packager/app/test/testdata/live-profile-with-webm/output.mpd
View file
Expand Up @@ -14,7 +14,7 @@
</SegmentTemplate>
</Representation>
</AdaptationSet>
<AdaptationSet id="1" contentType="video" width="640" height="360" frameRate="1000000/33000" segmentAlignment="true" par="16:9">
<AdaptationSet id="1" contentType="video" width="640" height="360" frameRate="1000000/34000" segmentAlignment="true" par="16:9">
<Representation id="1" bandwidth="472384" codecs="vp8" mimeType="video/webm" sar="1:1">
<SegmentTemplate timescale="1000000" initialization="bear-640x360-video-init.webm" media="bear-640x360-video-$Number$.webm" startNumber="1">
<SegmentTimeline>
Expand Down
2 changes: 1 addition & 1 deletion packager/app/test/testdata/opus-vp9-mp4-with-encryption/output.mpd
View file
Expand Up @@ -15,7 +15,7 @@
</SegmentBase>
</Representation>
</AdaptationSet>
<AdaptationSet id="1" contentType="video" width="320" height="240" frameRate="1000000/34000" subsegmentAlignment="true" par="16:9">
<AdaptationSet id="1" contentType="video" width="320" height="240" frameRate="1000000/33000" subsegmentAlignment="true" par="16:9">
<ContentProtection value="cenc" schemeIdUri="urn:mpeg:dash:mp4protection:2011" cenc:default_KID="31323334-3536-3738-3930-313233343536"/>
<ContentProtection schemeIdUri="urn:uuid:1077efec-c0b2-4d02-ace3-3c1e52e2fb4b">
<cenc:pssh>AAAANHBzc2gBAAAAEHfv7MCyTQKs4zweUuL7SwAAAAExMjM0NTY3ODkwMTIzNDU2AAAAAA==</cenc:pssh>
Expand Down
2 changes: 1 addition & 1 deletion packager/app/test/testdata/vp8-mp4-with-encryption/output.mpd
View file
Expand Up @@ -2,7 +2,7 @@
<!--Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>-->
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" xmlns:cenc="urn:mpeg:cenc:2013" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT2.736S">
<Period id="0">
<AdaptationSet id="0" contentType="video" width="640" height="360" frameRate="1000000/33000" subsegmentAlignment="true" par="16:9">
<AdaptationSet id="0" contentType="video" width="640" height="360" frameRate="1000000/34000" subsegmentAlignment="true" par="16:9">
<ContentProtection value="cenc" schemeIdUri="urn:mpeg:dash:mp4protection:2011" cenc:default_KID="31323334-3536-3738-3930-313233343536"/>
<ContentProtection schemeIdUri="urn:uuid:1077efec-c0b2-4d02-ace3-3c1e52e2fb4b">
<cenc:pssh>AAAANHBzc2gBAAAAEHfv7MCyTQKs4zweUuL7SwAAAAExMjM0NTY3ODkwMTIzNDU2AAAAAA==</cenc:pssh>
Expand Down
2 changes: 1 addition & 1 deletion packager/app/test/testdata/vp8-webm/output.mpd
View file
Expand Up @@ -2,7 +2,7 @@
<!--Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>-->
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT2.736S">
<Period id="0">
<AdaptationSet id="0" contentType="video" width="640" height="360" frameRate="1000000/33000" subsegmentAlignment="true" par="16:9">
<AdaptationSet id="0" contentType="video" width="640" height="360" frameRate="1000000/34000" subsegmentAlignment="true" par="16:9">
<Representation id="0" bandwidth="472384" codecs="vp8" mimeType="video/webm" sar="1:1">
<BaseURL>bear-640x360-video.webm</BaseURL>
<SegmentBase indexRange="296-345" timescale="1000000" presentationTimeOffset="26000">
Expand Down
2 changes: 1 addition & 1 deletion packager/app/test/testdata/vp9-webm-with-blockgroup/output.mpd
View file
Expand Up @@ -2,7 +2,7 @@
<!--Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>-->
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT2.736S">
<Period id="0">
<AdaptationSet id="0" contentType="video" width="320" height="240" frameRate="1000000/33000" subsegmentAlignment="true" par="4:3">
<AdaptationSet id="0" contentType="video" width="320" height="240" frameRate="1000000/34000" subsegmentAlignment="true" par="4:3">
<Representation id="0" bandwidth="196039" codecs="vp09.00.20.08.01.02.02.02.00" mimeType="video/webm" sar="1:1">
<BaseURL>bear-vp9-blockgroup-video.webm</BaseURL>
<SegmentBase indexRange="309-327" timescale="1000000">
Expand Down
2 changes: 1 addition & 1 deletion packager/app/test/testdata/vp9-webm/output.mpd
View file
Expand Up @@ -11,7 +11,7 @@
</SegmentBase>
</Representation>
</AdaptationSet>
<AdaptationSet id="1" contentType="video" width="320" height="240" frameRate="1000000/34000" subsegmentAlignment="true" par="16:9">
<AdaptationSet id="1" contentType="video" width="320" height="240" frameRate="1000000/33000" subsegmentAlignment="true" par="16:9">
<Representation id="1" bandwidth="225727" codecs="vp09.00.20.08.01.02.02.02.00" mimeType="video/webm" sar="427:320">
<BaseURL>bear-320x240-vp9-opus-video.webm</BaseURL>
<SegmentBase indexRange="309-357" timescale="1000000" presentationTimeOffset="37000">
Expand Down
2 changes: 1 addition & 1 deletion packager/app/test/testdata/webm-subsample-encryption/output.mpd
View file
Expand Up @@ -2,7 +2,7 @@
<!--Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>-->
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" xmlns:cenc="urn:mpeg:cenc:2013" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT2.769S">
<Period id="0">
<AdaptationSet id="0" contentType="video" width="320" height="180" frameRate="1000000/33000" subsegmentAlignment="true" par="16:9">
<AdaptationSet id="0" contentType="video" width="320" height="180" frameRate="1000000/34000" subsegmentAlignment="true" par="16:9">
<ContentProtection schemeIdUri="urn:uuid:1077efec-c0b2-4d02-ace3-3c1e52e2fb4b" cenc:default_KID="31323334-3536-3738-3930-313233343536">
<cenc:pssh>AAAANHBzc2gBAAAAEHfv7MCyTQKs4zweUuL7SwAAAAExMjM0NTY3ODkwMTIzNDU2AAAAAA==</cenc:pssh>
</ContentProtection>
Expand Down
2 changes: 1 addition & 1 deletion packager/app/test/testdata/webm-vp9-full-sample-encryption/output.mpd
View file
Expand Up @@ -2,7 +2,7 @@
<!--Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>-->
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" xmlns:cenc="urn:mpeg:cenc:2013" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT2.769S">
<Period id="0">
<AdaptationSet id="0" contentType="video" width="320" height="180" frameRate="1000000/33000" subsegmentAlignment="true" par="16:9">
<AdaptationSet id="0" contentType="video" width="320" height="180" frameRate="1000000/34000" subsegmentAlignment="true" par="16:9">
<ContentProtection schemeIdUri="urn:uuid:1077efec-c0b2-4d02-ace3-3c1e52e2fb4b" cenc:default_KID="31323334-3536-3738-3930-313233343536">
<cenc:pssh>AAAANHBzc2gBAAAAEHfv7MCyTQKs4zweUuL7SwAAAAExMjM0NTY3ODkwMTIzNDU2AAAAAA==</cenc:pssh>
</ContentProtection>
Expand Down
2 changes: 1 addition & 1 deletion packager/app/test/testdata/webm-with-encryption/output.mpd
View file
Expand Up @@ -2,7 +2,7 @@
<!--Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>-->
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" xmlns:cenc="urn:mpeg:cenc:2013" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT2.736S">
<Period id="0">
<AdaptationSet id="0" contentType="video" width="640" height="360" frameRate="1000000/33000" subsegmentAlignment="true" par="16:9">
<AdaptationSet id="0" contentType="video" width="640" height="360" frameRate="1000000/34000" subsegmentAlignment="true" par="16:9">
<ContentProtection schemeIdUri="urn:uuid:1077efec-c0b2-4d02-ace3-3c1e52e2fb4b" cenc:default_KID="31323334-3536-3738-3930-313233343536">
<cenc:pssh>AAAANHBzc2gBAAAAEHfv7MCyTQKs4zweUuL7SwAAAAExMjM0NTY3ODkwMTIzNDU2AAAAAA==</cenc:pssh>
</ContentProtection>
Expand Down
3 changes: 3 additions & 0 deletions packager/media/formats/mp2t/ts_muxer.cc
View file
Expand Up @@ -49,6 +49,9 @@ Status TsMuxer::Finalize() {

Status TsMuxer::AddMediaSample(size_t stream_id, const MediaSample& sample) {
DCHECK_EQ(stream_id, 0u);

// The duration of the first sample may have been adjusted, so use
// the duration of the second sample instead.
if (num_samples_ < 2) {
sample_durations_[num_samples_] =
sample.duration() * kTsTimescale / streams().front()->time_scale();
Expand Down
4 changes: 2 additions & 2 deletions packager/media/formats/mp2t/ts_muxer.h
View file
Expand Up @@ -38,8 +38,8 @@ class TsMuxer : public Muxer {
void FireOnMediaEndEvent();

std::unique_ptr<TsSegmenter> segmenter_;
int64_t sample_durations_[2];
int64_t num_samples_ = 0;
int64_t sample_durations_[2] = {0, 0};
size_t num_samples_ = 0;

// Used in multi-segment mode for segment template.
uint64_t segment_number_ = 0;
Expand Down
8 changes: 6 additions & 2 deletions packager/media/formats/mp4/segmenter.cc
View file
Expand Up @@ -141,8 +141,12 @@ Status Segmenter::AddSample(size_t stream_id, const MediaSample& sample) {
if (!status.ok())
return status;

if (sample_duration_ == 0)
sample_duration_ = sample.duration();
// The duration of the first sample may have been adjusted, so use
// the duration of the second sample instead.
if (num_samples_ < 2) {
sample_durations_[num_samples_] = sample.duration();
num_samples_++;
}
stream_durations_[stream_id] += sample.duration();
return Status::OK;
}
Expand Down
7 changes: 5 additions & 2 deletions packager/media/formats/mp4/segmenter.h
View file
Expand Up @@ -99,7 +99,9 @@ class Segmenter {

/// @return The sample duration in the timescale of the media.
/// Returns 0 if no samples are added yet.
int32_t sample_duration() const { return sample_duration_; }
int64_t sample_duration() const {
return sample_durations_[num_samples_ < 2 ? 0 : 1];
}

protected:
/// Update segmentation progress using ProgressListener.
Expand Down Expand Up @@ -147,7 +149,8 @@ class Segmenter {
ProgressListener* progress_listener_ = nullptr;
uint64_t progress_target_ = 0u;
uint64_t accumulated_progress_ = 0u;
int32_t sample_duration_ = 0;
int64_t sample_durations_[2] = {0, 0};
size_t num_samples_ = 0;
std::vector<uint64_t> stream_durations_;
std::vector<KeyFrameInfo> key_frame_infos_;

Expand Down
14 changes: 9 additions & 5 deletions packager/media/formats/webm/segmenter.cc
View file
Expand Up @@ -159,11 +159,15 @@ Status Segmenter::Finalize() {
Status Segmenter::AddSample(const MediaSample& source_sample) {
std::shared_ptr<MediaSample> sample(source_sample.Clone());

if (sample_duration_ == 0) {
first_timestamp_ = sample->pts();
sample_duration_ = sample->duration();
if (muxer_listener_)
muxer_listener_->OnSampleDurationReady(sample_duration_);
// The duration of the first sample may have been adjusted, so use
// the duration of the second sample instead.
if (num_samples_ < 2) {
sample_durations_[num_samples_] = sample->duration();
if (num_samples_ == 0)
first_timestamp_ = sample->pts();
else if (muxer_listener_)
muxer_listener_->OnSampleDurationReady(sample_durations_[num_samples_]);
num_samples_++;
}

UpdateProgress(sample->duration());
Expand Down
5 changes: 4 additions & 1 deletion packager/media/formats/webm/segmenter.h
View file
Expand Up @@ -139,8 +139,11 @@ class Segmenter {
ProgressListener* progress_listener_ = nullptr;
uint64_t progress_target_ = 0;
uint64_t accumulated_progress_ = 0;

int64_t first_timestamp_ = 0;
int64_t sample_duration_ = 0;
int64_t sample_durations_[2] = {0, 0};
size_t num_samples_ = 0;

// The position (in bytes) of the start of the Segment payload in the init
// file. This is also the size of the header before the SeekHead.
uint64_t segment_payload_pos_ = 0;
Expand Down

0 comments on commit 5644041

Please sign in to comment.