Skip to content

Commit

Permalink
Send HEVC HDR display metadata to decoder.
Browse files Browse the repository at this point in the history 
HEVC HDR mastering displaying content volume information and content light level information may not be included in mdcv/clli box, but instead be present in prefix-sei if it is not Atmos. In that case, we rely on extra data after demuxing to parse corresponding prefix-sei and populate the HDR display metadata, and carry it over to decoder for delivering to display.

Bug: 1362411,1362288
Change-Id: I3fa419aea71216c17da5b3651e3d4af7ab8e220e
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3916840
Commit-Queue: Jianlin Qiu <jianlin.qiu@intel.com>
Reviewed-by: Dan Sanders <sandersd@chromium.org>
Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1052136}
  • Loading branch information
@taste1981
taste1981 authored and Chromium LUCI CQ committed Sep 28, 2022
1 parent 3076e81 commit 85af876
Show file tree
Hide file tree
Showing 13 changed files with 236 additions and 84 deletions.
8 changes: 6 additions & 2 deletions media/ffmpeg/ffmpeg_common.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -498,6 +498,7 @@ bool AVStreamToVideoDecoderConfig(const AVStream* stream,
// for now, but may not always be true forever. Fix this in the future.
gfx::Rect visible_rect(codec_context->width, codec_context->height);
gfx::Size coded_size = visible_rect.size();
gfx::HDRMetadata hdr_metadata;

// In some cases a container may have a DAR but no PAR, but FFmpeg translates
// everything to PAR. It is possible to get the render width and height, but I
Expand Down Expand Up @@ -572,6 +573,7 @@ bool AVStreamToVideoDecoderConfig(const AVStream* stream,
// the container
color_space = hevc_config.GetColorSpace();
}
hdr_metadata = hevc_config.GetHDRMetadata();
#endif // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
}
}
Expand Down Expand Up @@ -722,7 +724,6 @@ bool AVStreamToVideoDecoderConfig(const AVStream* stream,
if (side_data.type != AV_PKT_DATA_MASTERING_DISPLAY_METADATA)
continue;

gfx::HDRMetadata hdr_metadata{};
AVMasteringDisplayMetadata* metadata =
reinterpret_cast<AVMasteringDisplayMetadata*>(side_data.data);
if (metadata->has_primaries) {
Expand All @@ -744,10 +745,13 @@ bool AVStreamToVideoDecoderConfig(const AVStream* stream,
hdr_metadata.color_volume_metadata.luminance_min =
av_q2d(metadata->min_luminance);
}
config->set_hdr_metadata(hdr_metadata);
}
}

if (hdr_metadata.IsValid()) {
config->set_hdr_metadata(hdr_metadata);
}

return true;
}

Expand Down
42 changes: 38 additions & 4 deletions media/formats/mp4/box_definitions.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,25 @@ VideoColorSpace ConvertColorParameterInformationToColorSpace(
: gfx::ColorSpace::RangeID::LIMITED);
}

gfx::ColorVolumeMetadata ConvertMdcvToColorVolumeMetadata(
const MasteringDisplayColorVolume& mdcv) {
gfx::ColorVolumeMetadata color_volume_metadata;

color_volume_metadata.primary_r = gfx::ColorVolumeMetadata::Chromaticity(
mdcv.display_primaries_rx, mdcv.display_primaries_ry);
color_volume_metadata.primary_g = gfx::ColorVolumeMetadata::Chromaticity(
mdcv.display_primaries_gx, mdcv.display_primaries_gy);
color_volume_metadata.primary_b = gfx::ColorVolumeMetadata::Chromaticity(
mdcv.display_primaries_bx, mdcv.display_primaries_by);
color_volume_metadata.white_point = gfx::ColorVolumeMetadata::Chromaticity(
mdcv.white_point_x, mdcv.white_point_y);

color_volume_metadata.luminance_max = mdcv.max_display_mastering_luminance;
color_volume_metadata.luminance_min = mdcv.min_display_mastering_luminance;

return color_volume_metadata;
}

} // namespace

FileType::FileType() = default;
Expand Down Expand Up @@ -1089,6 +1108,7 @@ bool VideoSampleEntry::Parse(BoxReader* reader) {
}
}

gfx::HDRMetadata hdr_static_metadata;
const FourCC actual_format =
format == FOURCC_ENCV ? sinf.format.format : format;
switch (actual_format) {
Expand Down Expand Up @@ -1128,6 +1148,7 @@ bool VideoSampleEntry::Parse(BoxReader* reader) {
video_codec_profile = hevcConfig->GetVideoProfile();
#if BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
video_color_space = hevcConfig->GetColorSpace();
hdr_metadata = hevcConfig->GetHDRMetadata();
#endif // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
frame_bitstream_converter =
base::MakeRefCounted<HEVCBitstreamConverter>(std::move(hevcConfig));
Expand Down Expand Up @@ -1171,6 +1192,7 @@ bool VideoSampleEntry::Parse(BoxReader* reader) {
RCHECK(reader->ReadChild(hevcConfig.get()));
#if BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
video_color_space = hevcConfig->GetColorSpace();
hdr_metadata = hevcConfig->GetHDRMetadata();
#endif // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
frame_bitstream_converter =
base::MakeRefCounted<HEVCBitstreamConverter>(std::move(hevcConfig));
Expand Down Expand Up @@ -1199,13 +1221,17 @@ bool VideoSampleEntry::Parse(BoxReader* reader) {
SMPTE2086MasteringDisplayMetadataBox color_volume;
if (reader->HasChild(&color_volume)) {
RCHECK(reader->ReadChild(&color_volume));
mastering_display_color_volume = color_volume;
hdr_static_metadata.color_volume_metadata =
ConvertMdcvToColorVolumeMetadata(color_volume);
}

ContentLightLevel level_information;
if (reader->HasChild(&level_information)) {
RCHECK(reader->ReadChild(&level_information));
content_light_level_information = level_information;
hdr_static_metadata.max_content_light_level =
level_information.max_content_light_level;
hdr_static_metadata.max_frame_average_light_level =
level_information.max_pic_average_light_level;
}
break;
}
Expand Down Expand Up @@ -1240,13 +1266,21 @@ bool VideoSampleEntry::Parse(BoxReader* reader) {
MasteringDisplayColorVolume color_volume;
if (reader->HasChild(&color_volume)) {
RCHECK(reader->ReadChild(&color_volume));
mastering_display_color_volume = color_volume;
hdr_static_metadata.color_volume_metadata =
ConvertMdcvToColorVolumeMetadata(color_volume);
}

ContentLightLevelInformation level_information;
if (reader->HasChild(&level_information)) {
RCHECK(reader->ReadChild(&level_information));
content_light_level_information = level_information;
hdr_static_metadata.max_content_light_level =
level_information.max_content_light_level;
hdr_static_metadata.max_frame_average_light_level =
level_information.max_pic_average_light_level;
}

if (hdr_static_metadata.IsValid()) {
hdr_metadata = hdr_static_metadata;
}

if (video_codec_profile == VIDEO_CODEC_PROFILE_UNKNOWN) {
Expand Down
3 changes: 1 addition & 2 deletions media/formats/mp4/box_definitions.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -334,8 +334,7 @@ struct MEDIA_EXPORT VideoSampleEntry : Box {
VideoCodecLevel video_codec_level;
VideoColorSpace video_color_space;

absl::optional<MasteringDisplayColorVolume> mastering_display_color_volume;
absl::optional<ContentLightLevelInformation> content_light_level_information;
absl::optional<gfx::HDRMetadata> hdr_metadata;

bool IsFormatValid() const;

Expand Down
127 changes: 88 additions & 39 deletions media/formats/mp4/hevc.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,88 @@ bool HEVCDecoderConfigurationRecord::ParseInternal(BufferReader* reader,
}
}

#if BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
// Parse the color space and hdr metadata.
std::vector<uint8_t> param_sets;
HEVC::ConvertConfigToAnnexB(*this, &param_sets);
H265Parser parser;
H265NALU nalu;
parser.SetStream(param_sets.data(), param_sets.size());
while (true) {
H265Parser::Result result = parser.AdvanceToNextNALU(&nalu);
if (result != H265Parser::kOk)
break;

switch (nalu.nal_unit_type) {
case H265NALU::SPS_NUT: {
int sps_id = -1;
result = parser.ParseSPS(&sps_id);
if (result != H265Parser::kOk) {
DVLOG(1) << "Could not parse SPS for fetching colorspace";
break;
}

const H265SPS* sps = parser.GetSPS(sps_id);
DCHECK(sps);
color_space = sps->GetColorSpace();
break;
}
case H265NALU::PREFIX_SEI_NUT: {
H265SEIMessage sei_msg;
result = parser.ParseSEI(&sei_msg);
if (result != H265Parser::kOk) {
DVLOG(1) << "Could not parse SEI for fetching HDR metadata";
break;
}
switch (sei_msg.type) {
case H265SEIMessage::kSEIContentLightLevelInfo:
hdr_metadata.max_content_light_level =
sei_msg.content_light_level_info.max_content_light_level;
hdr_metadata.max_frame_average_light_level =
sei_msg.content_light_level_info
.max_picture_average_light_level;
break;
case H265SEIMessage::kSEIMasteringDisplayInfo: {
constexpr auto kChromaDenominator = 50000.0f;
constexpr auto kLumaDenoninator = 10000.0f;
// display primaries are in G/B/R order in MDCV SEI.
hdr_metadata.color_volume_metadata.primary_r = gfx::PointF(
sei_msg.mastering_display_info.display_primaries[2][0] /
kChromaDenominator,
sei_msg.mastering_display_info.display_primaries[2][1] /
kChromaDenominator);
hdr_metadata.color_volume_metadata.primary_g = gfx::PointF(
sei_msg.mastering_display_info.display_primaries[0][0] /
kChromaDenominator,
sei_msg.mastering_display_info.display_primaries[0][1] /
kChromaDenominator);
hdr_metadata.color_volume_metadata.primary_b = gfx::PointF(
sei_msg.mastering_display_info.display_primaries[1][0] /
kChromaDenominator,
sei_msg.mastering_display_info.display_primaries[1][1] /
kChromaDenominator);
hdr_metadata.color_volume_metadata.white_point =
gfx::PointF(sei_msg.mastering_display_info.white_points[0] /
kChromaDenominator,
sei_msg.mastering_display_info.white_points[1] /
kChromaDenominator);
hdr_metadata.color_volume_metadata.luminance_max =
sei_msg.mastering_display_info.max_luminance / kLumaDenoninator;
hdr_metadata.color_volume_metadata.luminance_min =
sei_msg.mastering_display_info.min_luminance / kLumaDenoninator;
break;
}
default:
break;
}
break;
}
default:
break;
}
}
#endif // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)

return true;
}

Expand Down Expand Up @@ -158,42 +240,11 @@ VideoCodecProfile HEVCDecoderConfigurationRecord::GetVideoProfile() const {

#if BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
VideoColorSpace HEVCDecoderConfigurationRecord::GetColorSpace() {
if (!arrays.size()) {
DVLOG(1) << "HVCCNALArray not found, fallback to default colorspace";
return VideoColorSpace();
}

std::vector<uint8_t> param_sets;
if (!HEVC::ConvertConfigToAnnexB(*this, &param_sets))
return VideoColorSpace();

H265Parser parser;
H265NALU nalu;
parser.SetStream(param_sets.data(), param_sets.size());
while (true) {
H265Parser::Result result = parser.AdvanceToNextNALU(&nalu);

if (result != H265Parser::kOk)
return VideoColorSpace();

switch (nalu.nal_unit_type) {
case H265NALU::SPS_NUT: {
int sps_id = -1;
result = parser.ParseSPS(&sps_id);
if (result != H265Parser::kOk) {
DVLOG(1) << "Could not parse SPS, fallback to default colorspace";
return VideoColorSpace();
}
return color_space;
}

const H265SPS* sps = parser.GetSPS(sps_id);
DCHECK(sps);
return sps->GetColorSpace();
}
default:
break;
}
}
NOTREACHED();
gfx::HDRMetadata HEVCDecoderConfigurationRecord::GetHDRMetadata() {
return hdr_metadata;
}
#endif // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)

Expand Down Expand Up @@ -226,7 +277,7 @@ bool HEVC::InsertParamSetsAnnexB(
start = NULL;

std::vector<uint8_t> param_sets;
RCHECK(HEVC::ConvertConfigToAnnexB(hevc_config, &param_sets));
HEVC::ConvertConfigToAnnexB(hevc_config, &param_sets);
DVLOG(4) << __func__ << " converted hvcC to AnnexB "
<< " size=" << param_sets.size() << " inserted at "
<< (int)(config_insert_point - buffer->begin());
Expand All @@ -247,7 +298,7 @@ bool HEVC::InsertParamSetsAnnexB(
}

// static
bool HEVC::ConvertConfigToAnnexB(
void HEVC::ConvertConfigToAnnexB(
const HEVCDecoderConfigurationRecord& hevc_config,
std::vector<uint8_t>* buffer) {
DCHECK(buffer->empty());
Expand All @@ -264,8 +315,6 @@ bool HEVC::ConvertConfigToAnnexB(
hevc_config.arrays[j].units[i].end());
}
}

return true;
}

// static
Expand Down
5 changes: 4 additions & 1 deletion media/formats/mp4/hevc.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,15 +64,18 @@ struct MEDIA_EXPORT HEVCDecoderConfigurationRecord : Box {
VideoCodecProfile GetVideoProfile() const;
#if BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
VideoColorSpace GetColorSpace();
gfx::HDRMetadata GetHDRMetadata();
#endif // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)

private:
bool ParseInternal(BufferReader* reader, MediaLog* media_log);
VideoColorSpace color_space;
gfx::HDRMetadata hdr_metadata;
};

class MEDIA_EXPORT HEVC {
public:
static bool ConvertConfigToAnnexB(
static void ConvertConfigToAnnexB(
const HEVCDecoderConfigurationRecord& hevc_config,
std::vector<uint8_t>* buffer);

Expand Down
37 changes: 2 additions & 35 deletions media/formats/mp4/mp4_stream_parser.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,25 +59,6 @@ EncryptionScheme GetEncryptionScheme(const ProtectionSchemeInfo& sinf) {
return EncryptionScheme::kUnencrypted;
}

gfx::ColorVolumeMetadata ConvertMdcvToColorVolumeMetadata(
const MasteringDisplayColorVolume& mdcv) {
gfx::ColorVolumeMetadata color_volume_metadata;

color_volume_metadata.primary_r = gfx::ColorVolumeMetadata::Chromaticity(
mdcv.display_primaries_rx, mdcv.display_primaries_ry);
color_volume_metadata.primary_g = gfx::ColorVolumeMetadata::Chromaticity(
mdcv.display_primaries_gx, mdcv.display_primaries_gy);
color_volume_metadata.primary_b = gfx::ColorVolumeMetadata::Chromaticity(
mdcv.display_primaries_bx, mdcv.display_primaries_by);
color_volume_metadata.white_point = gfx::ColorVolumeMetadata::Chromaticity(
mdcv.white_point_x, mdcv.white_point_y);

color_volume_metadata.luminance_max = mdcv.max_display_mastering_luminance;
color_volume_metadata.luminance_min = mdcv.min_display_mastering_luminance;

return color_volume_metadata;
}

} // namespace

MP4StreamParser::MP4StreamParser(const std::set<int>& audio_object_types,
Expand Down Expand Up @@ -579,22 +560,8 @@ bool MP4StreamParser::ParseMoov(BoxReader* reader) {
if (entry.video_color_space.IsSpecified())
video_config.set_color_space_info(entry.video_color_space);

if (entry.mastering_display_color_volume ||
entry.content_light_level_information) {
gfx::HDRMetadata hdr_metadata;
if (entry.mastering_display_color_volume) {
hdr_metadata.color_volume_metadata = ConvertMdcvToColorVolumeMetadata(
*entry.mastering_display_color_volume);
}

if (entry.content_light_level_information) {
hdr_metadata.max_content_light_level =
entry.content_light_level_information->max_content_light_level;
hdr_metadata.max_frame_average_light_level =
entry.content_light_level_information
->max_pic_average_light_level;
}
video_config.set_hdr_metadata(hdr_metadata);
if (entry.hdr_metadata.has_value() && entry.hdr_metadata->IsValid()) {
video_config.set_hdr_metadata(entry.hdr_metadata.value());
}

DVLOG(1) << "video_track_id=" << video_track_id
Expand Down

0 comments on commit 85af876

Please sign in to comment.