Skip to content

Commit

Permalink
layers: Add AV1 video decode support
Browse files Browse the repository at this point in the history
  • Loading branch information
@aqnuep @spencer-lunarg
aqnuep authored and spencer-lunarg committed Feb 5, 2024
1 parent 0b85da4 commit 96c8217
Show file tree
Hide file tree
Showing 10 changed files with 945 additions and 55 deletions.
214 changes: 205 additions & 9 deletions layers/core_checks/cc_video.cpp
View file

Large diffs are not rendered by default.

4 changes: 4 additions & 0 deletions layers/core_checks/core_validation.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -562,10 +562,14 @@ class CoreChecks : public ValidationStateTracker {
const Location& loc,
const VkVideoEncodeH265SessionParametersCreateInfoKHR* create_info = nullptr,
const vvl::VideoSessionParameters* template_state = nullptr) const;
bool ValidateDecodeDistinctOutput(const vvl::CommandBuffer& cb_state, const VkVideoDecodeInfoKHR& decode_info,
const Location& loc) const;
bool ValidateVideoDecodeInfoH264(const vvl::CommandBuffer& cb_state, const VkVideoDecodeInfoKHR& decode_info,
const Location& loc) const;
bool ValidateVideoDecodeInfoH265(const vvl::CommandBuffer& cb_state, const VkVideoDecodeInfoKHR& decode_info,
const Location& loc) const;
bool ValidateVideoDecodeInfoAV1(const vvl::CommandBuffer& cb_state, const VkVideoDecodeInfoKHR& decode_info,
const Location& loc) const;
bool ValidateVideoEncodeH264PicType(const vvl::VideoSession& vs_state, StdVideoH264PictureType pic_type, const Location& loc,
const char* where) const;
bool ValidateVideoEncodeInfoH264(const vvl::CommandBuffer& cb_state, const VkVideoEncodeInfoKHR& encode_info,
Expand Down
73 changes: 73 additions & 0 deletions layers/state_tracker/video_session_state.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,20 @@ bool VideoProfileDesc::InitProfile(VkVideoProfileInfoKHR const *profile) {
profile_.base.pNext = &profile_.decode_h265;
break;
}
case VK_VIDEO_CODEC_OPERATION_DECODE_AV1_BIT_KHR: {
auto decode_av1 = vku::FindStructInPNextChain<VkVideoDecodeAV1ProfileInfoKHR>(profile->pNext);
if (decode_av1) {
profile_.valid = true;
profile_.decode_av1 = *decode_av1;
profile_.decode_av1.pNext = nullptr;
} else {
profile_.valid = false;
profile_.decode_av1 = vku::InitStructHelper();
}
profile_.is_decode = true;
profile_.base.pNext = &profile_.decode_av1;
break;
}
case VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR: {
auto encode_h264 = vku::FindStructInPNextChain<VkVideoEncodeH264ProfileInfoKHR>(profile->pNext);
if (encode_h264) {
Expand Down Expand Up @@ -156,6 +170,13 @@ void VideoProfileDesc::InitCapabilities(VkPhysicalDevice physical_device) {
capabilities_.decode_h265 = vku::InitStructHelper();
break;

case VK_VIDEO_CODEC_OPERATION_DECODE_AV1_BIT_KHR:
capabilities_.base.pNext = &capabilities_.decode;
capabilities_.decode = vku::InitStructHelper();
capabilities_.decode.pNext = &capabilities_.decode_av1;
capabilities_.decode_av1 = vku::InitStructHelper();
break;

case VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR:
capabilities_.base.pNext = &capabilities_.encode;
capabilities_.encode = vku::InitStructHelper();
Expand Down Expand Up @@ -250,6 +271,38 @@ VkImageSubresourceRange VideoPictureResource::GetImageSubresourceRange(ImageView
return range;
}

VkOffset3D VideoPictureResource::GetEffectiveImageOffset(const vvl::VideoSession &vs_state) const {
VkOffset3D offset{coded_offset.x, coded_offset.y, 0};

// Round to picture access granularity
const auto gran = vs_state.profile->GetCapabilities().base.pictureAccessGranularity;
offset.x = (offset.x / gran.width) * gran.width;
offset.y = (offset.y / gran.height) * gran.height;

return offset;
}

VkExtent3D VideoPictureResource::GetEffectiveImageExtent(const vvl::VideoSession &vs_state) const {
VkExtent3D extent{coded_extent.width, coded_extent.height, 1};

// H.264 decode interlacing with separate planes only accesses half of the coded height
if (vs_state.GetCodecOp() == VK_VIDEO_CODEC_OPERATION_DECODE_H264_BIT_KHR &&
vs_state.GetH264PictureLayout() == VK_VIDEO_DECODE_H264_PICTURE_LAYOUT_INTERLACED_SEPARATE_PLANES_BIT_KHR) {
extent.height /= 2;
}

// Round to picture access granularity
const auto gran = vs_state.profile->GetCapabilities().base.pictureAccessGranularity;
extent.width = ((extent.width + gran.width - 1) / gran.width) * gran.width;
extent.height = ((extent.height + gran.height - 1) / gran.height) * gran.height;

// Clamp to mip level dimensions
extent.width = std::min(extent.width, image_state->createInfo.extent.width >> range.baseMipLevel);
extent.height = std::min(extent.height, image_state->createInfo.extent.height >> range.baseMipLevel);

return extent;
}

VideoPictureID::VideoPictureID(VideoProfileDesc const &profile, VkVideoReferenceSlotInfoKHR const &slot) {
switch (profile.GetCodecOp()) {
case VK_VIDEO_CODEC_OPERATION_DECODE_H264_BIT_KHR: {
Expand All @@ -262,6 +315,7 @@ VideoPictureID::VideoPictureID(VideoProfileDesc const &profile, VkVideoReference
}

case VK_VIDEO_CODEC_OPERATION_DECODE_H265_BIT_KHR:
case VK_VIDEO_CODEC_OPERATION_DECODE_AV1_BIT_KHR:
case VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR:
case VK_VIDEO_CODEC_OPERATION_ENCODE_H265_BIT_KHR:
break;
Expand Down Expand Up @@ -572,6 +626,11 @@ bool VideoSession::ReferenceSetupRequested(VkVideoDecodeInfoKHR const &decode_in
auto pic_info = vku::FindStructInPNextChain<VkVideoDecodeH265PictureInfoKHR>(decode_info.pNext);
return pic_info != nullptr && pic_info->pStdPictureInfo != nullptr && pic_info->pStdPictureInfo->flags.IsReference;
}
case VK_VIDEO_CODEC_OPERATION_DECODE_AV1_BIT_KHR: {
auto pic_info = vku::FindStructInPNextChain<VkVideoDecodeAV1PictureInfoKHR>(decode_info.pNext);
return pic_info != nullptr && pic_info->pStdPictureInfo != nullptr &&
pic_info->pStdPictureInfo->refresh_frame_flags != 0;
}

default:
return false;
Expand Down Expand Up @@ -642,6 +701,14 @@ VideoSessionParameters::VideoSessionParameters(VkVideoSessionParametersKHR vsp,
break;
}

case VK_VIDEO_CODEC_OPERATION_DECODE_AV1_BIT_KHR: {
const auto decode_av1 = vku::FindStructInPNextChain<VkVideoDecodeAV1SessionParametersCreateInfoKHR>(create_info.pNext);
if (decode_av1->pStdSequenceHeader) {
data_.av1.seq_header = std::make_unique<StdVideoAV1SequenceHeader>(*decode_av1->pStdSequenceHeader);
}
break;
}

case VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR: {
const auto encode_h264 =
vku::FindStructInPNextChain<VkVideoEncodeH264SessionParametersCreateInfoKHR>(create_info.pNext);
Expand Down Expand Up @@ -716,6 +783,12 @@ void VideoSessionParameters::Update(VkVideoSessionParametersUpdateInfoKHR const
break;
}

case VK_VIDEO_CODEC_OPERATION_DECODE_AV1_BIT_KHR: {
// AV1 decode session parameters cannot be updated
assert(false);
break;
}

case VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR: {
auto add_info = vku::FindStructInPNextChain<VkVideoEncodeH264SessionParametersAddInfoKHR>(info->pNext);
if (add_info) {
Expand Down
48 changes: 22 additions & 26 deletions layers/state_tracker/video_session_state.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ class VideoProfileDesc : public std::enable_shared_from_this<VideoProfileDesc> {
union {
VkVideoDecodeH264ProfileInfoKHR decode_h264;
VkVideoDecodeH265ProfileInfoKHR decode_h265;
VkVideoDecodeAV1ProfileInfoKHR decode_av1;
VkVideoEncodeH264ProfileInfoKHR encode_h264;
VkVideoEncodeH265ProfileInfoKHR encode_h265;
};
Expand All @@ -65,6 +66,7 @@ class VideoProfileDesc : public std::enable_shared_from_this<VideoProfileDesc> {
union {
VkVideoDecodeH264CapabilitiesKHR decode_h264;
VkVideoDecodeH265CapabilitiesKHR decode_h265;
VkVideoDecodeAV1CapabilitiesKHR decode_av1;
VkVideoEncodeH264CapabilitiesKHR encode_h264;
VkVideoEncodeH265CapabilitiesKHR encode_h265;
};
Expand All @@ -80,6 +82,7 @@ class VideoProfileDesc : public std::enable_shared_from_this<VideoProfileDesc> {
bool IsEncode() const { return profile_.is_encode; }
VkVideoCodecOperationFlagBitsKHR GetCodecOp() const { return profile_.base.videoCodecOperation; }
VkVideoDecodeH264PictureLayoutFlagBitsKHR GetH264PictureLayout() const { return profile_.decode_h264.pictureLayout; }
bool HasAV1FilmGrainSupport() const { return profile_.decode_av1.filmGrainSupport; }

VkExtent2D GetMaxCodingBlockSize() const {
switch (profile_.base.videoCodecOperation) {
Expand All @@ -99,32 +102,6 @@ class VideoProfileDesc : public std::enable_shared_from_this<VideoProfileDesc> {
}
}

VkOffset3D GetEffectiveImageOffset(VkOffset2D coded_offset) const {
VkOffset3D offset{coded_offset.x, coded_offset.y, 0};

// Round to picture access granularity
const auto gran = capabilities_.base.pictureAccessGranularity;
offset.x = (offset.x / gran.width) * gran.width;
offset.y = (offset.y / gran.height) * gran.height;
return offset;
}

VkExtent3D GetEffectiveImageExtent(VkExtent2D coded_extent) const {
VkExtent3D extent{coded_extent.width, coded_extent.height, 0};

// H.264 decode interlacing with separate planes only accesses half of the coded height
if (GetCodecOp() == VK_VIDEO_CODEC_OPERATION_DECODE_H264_BIT_KHR &&
GetH264PictureLayout() == VK_VIDEO_DECODE_H264_PICTURE_LAYOUT_INTERLACED_SEPARATE_PLANES_BIT_KHR) {
extent.height /= 2;
}

// Round to picture access granularity
const auto gran = capabilities_.base.pictureAccessGranularity;
extent.width = ((extent.width + gran.width - 1) / gran.width) * gran.width;
extent.height = ((extent.height + gran.height - 1) / gran.height) * gran.height;
return extent;
}

struct compare {
public:
bool operator()(VideoProfileDesc const *lhs, VideoProfileDesc const *rhs) const {
Expand Down Expand Up @@ -154,6 +131,11 @@ class VideoProfileDesc : public std::enable_shared_from_this<VideoProfileDesc> {
match = match && lhs->profile_.decode_h265.stdProfileIdc == rhs->profile_.decode_h265.stdProfileIdc;
break;

case VK_VIDEO_CODEC_OPERATION_DECODE_AV1_BIT_KHR:
match = match && lhs->profile_.decode_av1.stdProfile == rhs->profile_.decode_av1.stdProfile &&
lhs->profile_.decode_av1.filmGrainSupport == rhs->profile_.decode_av1.filmGrainSupport;
break;

case VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR:
match = match && lhs->profile_.encode_h264.stdProfileIdc == rhs->profile_.encode_h264.stdProfileIdc;
break;
Expand Down Expand Up @@ -196,6 +178,10 @@ class VideoProfileDesc : public std::enable_shared_from_this<VideoProfileDesc> {
hc << desc->profile_.decode_h265.stdProfileIdc;
break;
}
case VK_VIDEO_CODEC_OPERATION_DECODE_AV1_BIT_KHR: {
hc << desc->profile_.decode_av1.stdProfile << desc->profile_.decode_av1.filmGrainSupport;
break;
}
case VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR: {
hc << desc->profile_.encode_h264.stdProfileIdc;
break;
Expand Down Expand Up @@ -282,6 +268,9 @@ class VideoPictureResource {
}
};

VkOffset3D GetEffectiveImageOffset(const vvl::VideoSession &vs_state) const;
VkExtent3D GetEffectiveImageExtent(const vvl::VideoSession &vs_state) const;

private:
VkImageSubresourceRange GetImageSubresourceRange(ImageView const *image_view_state, uint32_t layer);
};
Expand Down Expand Up @@ -578,10 +567,15 @@ class VideoSessionParameters : public StateObject {
uint32_t pps_capacity;
};

struct AV1Parameters {
std::unique_ptr<StdVideoAV1SequenceHeader> seq_header;
};

struct Data {
uint32_t update_sequence_counter;
H264Parameters h264;
H265Parameters h265;
AV1Parameters av1;
};

struct EncodeConfig {
Expand Down Expand Up @@ -669,6 +663,8 @@ class VideoSessionParameters : public StateObject {
}
}

const StdVideoAV1SequenceHeader *GetAV1SequenceHeader() const { return data_->av1.seq_header.get(); }

private:
std::unique_lock<std::mutex> lock_;
const Data *data_;
Expand Down
8 changes: 4 additions & 4 deletions layers/sync/sync_access_context.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,8 +222,8 @@ void AccessContext::UpdateAccessState(const AttachmentViewGen &view_gen, Attachm
void AccessContext::UpdateAccessState(const vvl::VideoSession &vs_state, const vvl::VideoPictureResource &resource,
SyncStageAccessIndex current_usage, ResourceUsageTag tag) {
const auto image = static_cast<const ImageState *>(resource.image_state.get());
const auto offset = vs_state.profile->GetEffectiveImageOffset(resource.coded_offset);
const auto extent = vs_state.profile->GetEffectiveImageExtent(resource.coded_extent);
const auto offset = resource.GetEffectiveImageOffset(vs_state);
const auto extent = resource.GetEffectiveImageExtent(vs_state);
ImageRangeGen range_gen(image->MakeImageRangeGen(resource.range, offset, extent, false));
UpdateAccessState(range_gen, current_usage, SyncOrdering::kNonAttachment, tag);
}
Expand Down Expand Up @@ -356,8 +356,8 @@ HazardResult AccessContext::DetectHazard(const AttachmentViewGen &view_gen, Atta
HazardResult AccessContext::DetectHazard(const vvl::VideoSession &vs_state, const vvl::VideoPictureResource &resource,
SyncStageAccessIndex current_usage) const {
const auto image = static_cast<const ImageState *>(resource.image_state.get());
const auto offset = vs_state.profile->GetEffectiveImageOffset(resource.coded_offset);
const auto extent = vs_state.profile->GetEffectiveImageExtent(resource.coded_extent);
const auto offset = resource.GetEffectiveImageOffset(vs_state);
const auto extent = resource.GetEffectiveImageExtent(vs_state);
ImageRangeGen range_gen(image->MakeImageRangeGen(resource.range, offset, extent, false));
HazardDetector detector(current_usage);
return DetectHazardGeneratedRanges(detector, range_gen, DetectOptions::kDetectAll);
Expand Down
4 changes: 2 additions & 2 deletions scripts/known_good.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@
"sub_dir": "Vulkan-Profiles",
"build_dir": "Vulkan-Profiles/build",
"install_dir": "Vulkan-Profiles/build/install",
"commit": "912cedbc49bd13713a030bc600c9e6f548c67355",
"commit": "439ed61787525eda26e1d8fa7de7398cc8a824fa",
"build_step": "skip",
"optional": [
"tests"
Expand All @@ -136,7 +136,7 @@
"sub_dir": "Vulkan-Tools",
"build_dir": "Vulkan-Tools/build",
"install_dir": "Vulkan-Tools/build/install",
"commit": "87130b22e74cb208bf8ecd755dcf40dabd365dad",
"commit": "f24bab1ea1c521cd77991260cad1b11384eec5f0",
"build_step": "skip",
"optional": [
"tests"
Expand Down
4 changes: 3 additions & 1 deletion tests/device_profiles/max_profile.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1945,6 +1945,7 @@
"VK_KHR_timeline_semaphore": 1,
"VK_KHR_uniform_buffer_standard_layout": 1,
"VK_KHR_variable_pointers": 1,
"VK_KHR_video_decode_av1": 1,
"VK_KHR_video_decode_h264": 1,
"VK_KHR_video_decode_h265": 1,
"VK_KHR_video_decode_queue": 1,
Expand Down Expand Up @@ -7625,7 +7626,8 @@
"VkQueueFamilyVideoPropertiesKHR": {
"videoCodecOperations": [
"VK_VIDEO_CODEC_OPERATION_DECODE_H264_BIT_KHR",
"VK_VIDEO_CODEC_OPERATION_DECODE_H265_BIT_KHR"
"VK_VIDEO_CODEC_OPERATION_DECODE_H265_BIT_KHR",
"VK_VIDEO_CODEC_OPERATION_DECODE_AV1_BIT_KHR"
]
}
},
Expand Down
Loading

0 comments on commit 96c8217

Please sign in to comment.