feat: Generate the entire AV1 codec string when the colr atom is pres…

…ent (#1205)

As per the AV1 spec, the codec string may contain optional color values.

This extracts the missing color information from the mp4 `colr` atom, if
present, and generates the full AV1 codec string. 

Closes #1007

packager/media/base/fourccs.h

@@ -35,6 +35,7 @@ enum FourCC : uint32_t {
   FOURCC_cbcs = 0x63626373,
   FOURCC_cenc = 0x63656e63,
   FOURCC_cens = 0x63656e73,
+  FOURCC_colr = 0x636f6c72,
   FOURCC_co64 = 0x636f3634,
   FOURCC_cmfc = 0x636d6663,
   FOURCC_cmfs = 0x636d6673,
@@ -102,6 +103,8 @@ enum FourCC : uint32_t {
   FOURCC_mp4v = 0x6d703476,
   FOURCC_mvex = 0x6d766578,
   FOURCC_mvhd = 0x6d766864,
+  FOURCC_nclc = 0x6e636c63,
+  FOURCC_nclx = 0x6e636c78,
   FOURCC_nmhd = 0x6e6d6864,
   FOURCC_pasp = 0x70617370,
   FOURCC_payl = 0x7061796c,

packager/media/codecs/av1_codec_configuration_record.cc

@@ -81,12 +81,27 @@ bool AV1CodecConfigurationRecord::Parse(const uint8_t* data, size_t data_size) {
 // mandatory fields.
 // All the other fields (including their leading '.') are optional, mutually
 // inclusive (all or none) fields.
-// Since some of the optional fields (e.g. colorPrimaries) are not present in
-// AV1CodecConfigurationRecord, we omit all the optional fields.
+
+// When color info is NOT available, generate the basic codec string without the
+// optional fields
 std::string AV1CodecConfigurationRecord::GetCodecString() const {
   return base::StringPrintf("av01.%d.%02d%c.%02d", profile_, level_,
                             tier_ ? 'H' : 'M', bit_depth_);
 }
 
+// When color info IS available, generate the full codec string with optional
+// fields
+std::string AV1CodecConfigurationRecord::GetCodecString(
+    uint16_t color_primaries,
+    uint16_t transfer_characteristics,
+    uint16_t matrix_coefficients,
+    uint8_t video_full_range_flag) const {
+  return base::StringPrintf(
+      "av01.%d.%02d%c.%02d.%d.%d%d%d.%02d.%02d.%02d.%d", profile_, level_,
+      tier_ ? 'H' : 'M', bit_depth_, mono_chrome_, chroma_subsampling_x_,
+      chroma_subsampling_y_, chroma_sample_position_, color_primaries,
+      transfer_characteristics, matrix_coefficients, video_full_range_flag);
+}
+
 }  // namespace media
 }  // namespace shaka

packager/media/codecs/av1_codec_configuration_record.h

@@ -33,6 +33,11 @@ class AV1CodecConfigurationRecord {
   /// @return The codec string.
   std::string GetCodecString() const;
 
+  std::string GetCodecString(uint16_t color_primaries,
+                             uint16_t transfer_characteristics,
+                             uint16_t matrix_coefficients,
+                             uint8_t video_full_range_flag) const;
+
  private:
   int profile_ = 0;
   int level_ = 0;

packager/media/codecs/av1_codec_configuration_record_unittest.cc

@@ -28,6 +28,8 @@ TEST(AV1CodecConfigurationRecordTest, Success) {
                            std::end(kAV1CodecConfigurationData))));
 
   EXPECT_EQ(av1_config.GetCodecString(), "av01.0.04M.10");
+  EXPECT_EQ(av1_config.GetCodecString(10, 8, 4, 1),
+            "av01.0.04M.10.0.112.10.08.04.1");
 }
 
 TEST(AV1CodecConfigurationRecordTest, Success2) {
@@ -47,6 +49,8 @@ TEST(AV1CodecConfigurationRecordTest, Success2) {
                            std::end(kAV1CodecConfigurationData))));
 
   EXPECT_EQ(av1_config.GetCodecString(), "av01.1.21H.12");
+  EXPECT_EQ(av1_config.GetCodecString(1, 1, 1, 0),
+            "av01.1.21H.12.1.010.01.01.01.0");
 }
 
 TEST(AV1CodecConfigurationRecordTest, InsufficientData) {

packager/media/formats/mp4/box_definitions.cc

@@ -1462,6 +1462,39 @@ size_t CodecConfiguration::ComputeSizeInternal() {
   return HeaderSize() + (box_type == FOURCC_vpcC ? 4 : 0) + data.size();
 }
 
+ColorParameters::ColorParameters() = default;
+ColorParameters::~ColorParameters() = default;
+
+FourCC ColorParameters::BoxType() const {
+  return FOURCC_colr;
+}
+
+bool ColorParameters::ReadWriteInternal(BoxBuffer* buffer) {
+  if (buffer->reader()) {
+    RCHECK((buffer->reader())->ReadFourCC(&color_parameter_type) &&
+           (buffer->reader())->Read2(&color_primaries) &&
+           (buffer->reader())->Read2(&transfer_characteristics) &&
+           (buffer->reader())->Read2(&matrix_coefficients));
+    // Type nclc does not contain video_full_range_flag data, and thus, it has 1
+    // less byte than nclx. Only extract video_full_range_flag if of type nclx.
+    if (color_parameter_type == FOURCC_nclx) {
+      RCHECK((buffer->reader())->Read1(&video_full_range_flag));
+    }
+  }
+  // TODO(caitlinocallaghan) Add the ability to write the colr atom and include
+  // it in the muxed mp4.
+  return true;
+}
+
+size_t ColorParameters::ComputeSizeInternal() {
+  // This box is optional. Skip it if it is not initialized.
+  if (color_parameter_type == FOURCC_NULL)
+    return 0;
+  return HeaderSize() + kFourCCSize + sizeof(color_primaries) +
+         sizeof(transfer_characteristics) + sizeof(matrix_coefficients) +
+         sizeof(video_full_range_flag);
+}
+
 PixelAspectRatio::PixelAspectRatio() = default;
 PixelAspectRatio::~PixelAspectRatio() = default;
 
@@ -1597,6 +1630,7 @@ bool VideoSampleEntry::ReadWriteInternal(BoxBuffer* buffer) {
       RCHECK(buffer->ReadWriteChild(&extra_codec_config));
   }
 
+  RCHECK(buffer->TryReadWriteChild(&colr));
   RCHECK(buffer->TryReadWriteChild(&pixel_aspect));
 
   // Somehow Edge does not support having sinf box before codec_configuration,

packager/media/formats/mp4/box_definitions.h

@@ -268,6 +268,16 @@ struct CodecConfiguration : Box {
   std::vector<uint8_t> data;
 };
 
+struct ColorParameters : Box {
+  DECLARE_BOX_METHODS(ColorParameters);
+
+  FourCC color_parameter_type = FOURCC_NULL;
+  uint16_t color_primaries = 1;
+  uint16_t transfer_characteristics = 1;
+  uint16_t matrix_coefficients = 1;
+  uint8_t video_full_range_flag = 0;
+};
+
 struct PixelAspectRatio : Box {
   DECLARE_BOX_METHODS(PixelAspectRatio);
 
@@ -297,6 +307,7 @@ struct VideoSampleEntry : Box {
   uint16_t width = 0u;
   uint16_t height = 0u;
 
+  ColorParameters colr;
   PixelAspectRatio pixel_aspect;
   ProtectionSchemeInfo sinf;
   CodecConfiguration codec_configuration;

packager/media/formats/mp4/box_definitions_unittest.cc

@@ -326,6 +326,14 @@ class BoxDefinitionsTestGeneral : public testing::Test {
     Modify(&metadata->id3v2);
   }
 
+  void Fill(ColorParameters* colr) {
+    colr->color_parameter_type = FOURCC_nclc;
+    colr->color_primaries = 9;
+    colr->transfer_characteristics = 16;
+    colr->matrix_coefficients = 9;
+    colr->video_full_range_flag = 0;
+  }
+
   void Fill(PixelAspectRatio* pasp) {
     pasp->h_spacing = 5;
     pasp->v_spacing = 8;
@@ -360,6 +368,7 @@ class BoxDefinitionsTestGeneral : public testing::Test {
     entry->data_reference_index = 1;
     entry->width = 800;
     entry->height = 600;
+    Fill(&entry->colr);
     Fill(&entry->pixel_aspect);
     Fill(&entry->sinf);
     Fill(&entry->codec_configuration);

packager/media/formats/mp4/mp4_media_parser.cc

@@ -604,7 +604,15 @@ bool MP4MediaParser::ParseMoov(BoxReader* reader) {
             LOG(ERROR) << "Failed to parse av1c.";
             return false;
           }
-          codec_string = av1_config.GetCodecString();
+          // Generate the full codec string if the colr atom is present.
+          if (entry.colr.color_parameter_type != FOURCC_NULL) {
+            codec_string = av1_config.GetCodecString(
+                entry.colr.color_primaries, entry.colr.transfer_characteristics,
+                entry.colr.matrix_coefficients,
+                entry.colr.video_full_range_flag);
+          } else {
+            codec_string = av1_config.GetCodecString();
+          }
           break;
         }
         case FOURCC_avc1: