diff --git a/CHANGELOG.md b/CHANGELOG.md index 81d4c8226..f1b94b987 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ Documentation for MIVisionX is available at * Support for advanced GPUs * Support for PreEmphasis Filter augmentation in openVX extensions +* Support for Spectrogram augmentation in openVX extensions ### Optimizations diff --git a/amd_openvx_extensions/amd_rpp/CMakeLists.txt b/amd_openvx_extensions/amd_rpp/CMakeLists.txt index 6ebb147a9..bb091a4f1 100644 --- a/amd_openvx_extensions/amd_rpp/CMakeLists.txt +++ b/amd_openvx_extensions/amd_rpp/CMakeLists.txt @@ -157,6 +157,7 @@ list(APPEND SOURCES source/tensor/Saturation.cpp source/tensor/SequenceRearrange.cpp source/tensor/Snow.cpp + source/tensor/Spectrogram.cpp source/tensor/Vignette.cpp source/tensor/WarpAffine.cpp source/tensor/SequenceRearrange.cpp diff --git a/amd_openvx_extensions/amd_rpp/include/internal_publishKernels.h b/amd_openvx_extensions/amd_rpp/include/internal_publishKernels.h index d96a927cc..9c63cb450 100644 --- a/amd_openvx_extensions/amd_rpp/include/internal_publishKernels.h +++ b/amd_openvx_extensions/amd_rpp/include/internal_publishKernels.h @@ -156,6 +156,7 @@ vx_status Snow_Register(vx_context); vx_status Vignette_Register(vx_context); vx_status WarpAffine_Register(vx_context); vx_status SequenceRearrange_Register(vx_context); +vx_status Spectrogram_Register(vx_context); // kernel names #define VX_KERNEL_RPP_NOPBATCHPD_NAME "org.rpp.NopbatchPD" @@ -274,12 +275,13 @@ vx_status SequenceRearrange_Register(vx_context); #define VX_KERNEL_RPP_PIXELATE_NAME "org.rpp.Pixelate" #define VX_KERNEL_RPP_VIGNETTE_NAME "org.rpp.Vignette" #define VX_KERNEL_RPP_WARPAFFINE_NAME "org.rpp.WarpAffine" -#define VX_KERNEL_RPP_BRIGHTNESS_NAME "org.rpp.Brightness" -#define VX_KERNEL_RPP_COPY_NAME "org.rpp.Copy" -#define VX_KERNEL_RPP_CROPMIRRORNORMALIZE_NAME "org.rpp.CropMirrorNormalize" -#define VX_KERNEL_RPP_NOP_NAME "org.rpp.Nop" -#define VX_KERNEL_RPP_RESIZE_NAME "org.rpp.Resize" -#define VX_KERNEL_RPP_SEQUENCEREARRANGE_NAME "org.rpp.SequenceRearrange" -#define VX_KERNEL_RPP_PREEMPHASISFILTER_NAME "org.rpp.PreemphasisFilter" +#define VX_KERNEL_RPP_BRIGHTNESS_NAME "org.rpp.Brightness" +#define VX_KERNEL_RPP_COPY_NAME "org.rpp.Copy" +#define VX_KERNEL_RPP_CROPMIRRORNORMALIZE_NAME "org.rpp.CropMirrorNormalize" +#define VX_KERNEL_RPP_NOP_NAME "org.rpp.Nop" +#define VX_KERNEL_RPP_RESIZE_NAME "org.rpp.Resize" +#define VX_KERNEL_RPP_SEQUENCEREARRANGE_NAME "org.rpp.SequenceRearrange" +#define VX_KERNEL_RPP_PREEMPHASISFILTER_NAME "org.rpp.PreemphasisFilter" +#define VX_KERNEL_RPP_SPECTROGRAM_NAME "org.rpp.Spectrogram" #endif //_AMDVX_EXT__PUBLISH_KERNELS_H_ diff --git a/amd_openvx_extensions/amd_rpp/include/internal_rpp.h b/amd_openvx_extensions/amd_rpp/include/internal_rpp.h index 61f5dbb29..ee9ea06c9 100644 --- a/amd_openvx_extensions/amd_rpp/include/internal_rpp.h +++ b/amd_openvx_extensions/amd_rpp/include/internal_rpp.h @@ -68,7 +68,10 @@ enum vxTensorLayout { VX_NHWC = 0, VX_NCHW = 1, VX_NFHWC = 2, - VX_NFCHW = 3 + VX_NFCHW = 3, + VX_NHW = 4, // Audio/2D layout + VX_NFT = 5, // Frequency major, Used for Spectrogram/MelFilterBank + VX_NTF = 6 // Time major, Used for Spectrogram/MelFilterBank }; //! Brief The utility functions @@ -76,8 +79,9 @@ vx_node createNode(vx_graph graph, vx_enum kernelEnum, vx_reference params[], vx vx_status createRPPHandle(vx_node node, vxRppHandle ** pHandle, Rpp32u batchSize, Rpp32u deviceType); vx_status releaseRPPHandle(vx_node node, vxRppHandle * handle, Rpp32u deviceType); void fillDescriptionPtrfromDims(RpptDescPtr &descPtr, vxTensorLayout layout, size_t *tensorDims); -void fillAudioDescriptionPtrFromDims(RpptDescPtr &descPtr, size_t *tensorDims); +void fillAudioDescriptionPtrFromDims(RpptDescPtr &descPtr, size_t *tensorDims, vxTensorLayout layout = vxTensorLayout::VX_NHW); RpptDataType getRpptDataType(vx_enum dataType); +RpptLayout getRpptLayout(vxTensorLayout layout); class Kernellist { diff --git a/amd_openvx_extensions/amd_rpp/include/kernels_rpp.h b/amd_openvx_extensions/amd_rpp/include/kernels_rpp.h index e83d1ac0b..c34a45804 100644 --- a/amd_openvx_extensions/amd_rpp/include/kernels_rpp.h +++ b/amd_openvx_extensions/amd_rpp/include/kernels_rpp.h @@ -148,7 +148,8 @@ extern "C" VX_KERNEL_RPP_SNOW = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x71, VX_KERNEL_RPP_VIGNETTE = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x72, VX_KERNEL_RPP_WARPAFFINE = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x73, - VX_KERNEL_RPP_PREEMPHASISFILTER = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x74 + VX_KERNEL_RPP_PREEMPHASISFILTER = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x74, + VX_KERNEL_RPP_SPECTROGRAM = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x75 }; #ifdef __cplusplus diff --git a/amd_openvx_extensions/amd_rpp/include/vx_ext_rpp.h b/amd_openvx_extensions/amd_rpp/include/vx_ext_rpp.h index 618cd1a2f..a28891d1a 100644 --- a/amd_openvx_extensions/amd_rpp/include/vx_ext_rpp.h +++ b/amd_openvx_extensions/amd_rpp/include/vx_ext_rpp.h @@ -1876,6 +1876,26 @@ extern "C" * \return A node reference \ref vx_node. Any possible errors preventing a successful creation should be checked using \ref vxGetStatus. */ SHARED_PUBLIC vx_node VX_API_CALL vxExtRppPreemphasisFilter(vx_graph graph, vx_tensor pSrc, vx_tensor pSrcRoi, vx_tensor pDst, vx_array pPreemphCoeff, vx_scalar borderType); + + /*! \brief [Graph] Produces a spectrogram from a 1D signal. + * \ingroup group_amd_rpp + * \param [in] graph The handle to the graph. + * \param [in] pSrc The input tensor in \ref VX_TYPE_FLOAT32 format data. + * \param [in] pSrcRoi The input tensor of batch size in unsigned int containing the roi values for the input in xywh/ltrb format. + * \param [out] pDst The output tensor (begin) in \ref VX_TYPE_FLOAT32 format data. + * \param [in] pDstRoi The input tensor of batch size in unsigned int containing the roi values for the output tensor in xywh/ltrb format. + * \param [in] windowFunction The input array in \ref VX_TYPE_FLOAT32 format containing the samples of the window function that will be multiplied to each extracted window when calculating the STFT. + * \param [in] centerWindow The input scalar in \ref VX_TYPE_BOOL format indicates whether extracted windows should be padded so that the window function is centered at multiples of window_step. + * \param [in] reflectPadding The input scalar in \ref VX_TYPE_BOOL format indicates the padding policy when sampling outside the bounds of the signal. + * \param [in] spectrogramLayout The input scalar in \ref VX_TYPE_INT32 format containing the Output spectrogram layout. + * \param [in] power The input scalar in \ref VX_TYPE_INT32 format containing the exponent of the magnitude of the spectrum. + * \param [in] nfft The input scalar in \ref VX_TYPE_INT32 format containing the size of the FFT. + * \param [in] windowLength The input scalar in \ref VX_TYPE_INT32 format containing Window size in number of samples. + * \param [in] windowStep The input array in \ref VX_TYPE_INT32 format containing the step between the STFT windows in number of samples. + * \return A node reference \ref vx_node. Any possible errors preventing a successful creation should be checked using \ref vxGetStatus. + */ + SHARED_PUBLIC vx_node VX_API_CALL vxExtRppSpectrogram(vx_graph graph, vx_tensor pSrc, vx_tensor pSrcRoi, vx_tensor pDst, vx_tensor pDstRoi, vx_array windowFunction, vx_scalar centerWindow, vx_scalar reflectPadding, vx_scalar spectrogramLayout, vx_scalar power, vx_scalar nfft, vx_scalar windowLength, vx_scalar windowStep); + #ifdef __cplusplus } #endif diff --git a/amd_openvx_extensions/amd_rpp/source/internal_publishKernels.cpp b/amd_openvx_extensions/amd_rpp/source/internal_publishKernels.cpp index 3ee050805..f01a35d48 100644 --- a/amd_openvx_extensions/amd_rpp/source/internal_publishKernels.cpp +++ b/amd_openvx_extensions/amd_rpp/source/internal_publishKernels.cpp @@ -161,6 +161,7 @@ vx_status get_kernels_to_publish() STATUS_ERROR_CHECK(ADD_KERNEL(Snow_Register)); STATUS_ERROR_CHECK(ADD_KERNEL(Vignette_Register)); STATUS_ERROR_CHECK(ADD_KERNEL(WarpAffine_Register)); + STATUS_ERROR_CHECK(ADD_KERNEL(Spectrogram_Register)); return status; } diff --git a/amd_openvx_extensions/amd_rpp/source/kernel_rpp.cpp b/amd_openvx_extensions/amd_rpp/source/kernel_rpp.cpp index c245a8e97..050bf1515 100644 --- a/amd_openvx_extensions/amd_rpp/source/kernel_rpp.cpp +++ b/amd_openvx_extensions/amd_rpp/source/kernel_rpp.cpp @@ -2558,6 +2558,32 @@ VX_API_ENTRY vx_node VX_API_CALL vxExtRppPreemphasisFilter(vx_graph graph, vx_te return node; } +VX_API_ENTRY vx_node VX_API_CALL vxExtRppSpectrogram(vx_graph graph, vx_tensor pSrc, vx_tensor pSrcRoi, vx_tensor pDst, vx_tensor pDstRoi, vx_array windowFunction, vx_scalar centerWindows, vx_scalar reflectPadding, vx_scalar spectrogramLayout, + vx_scalar power, vx_scalar nfft, vx_scalar windowLength, vx_scalar windowStep) { + vx_node node = NULL; + vx_context context = vxGetContext((vx_reference)graph); + if (vxGetStatus((vx_reference)context) == VX_SUCCESS) { + vx_uint32 devtype = getGraphAffinity(graph); + vx_scalar deviceType = vxCreateScalar(vxGetContext((vx_reference)graph), VX_TYPE_UINT32, &devtype); + vx_reference params[] = { + (vx_reference)pSrc, + (vx_reference)pSrcRoi, + (vx_reference)pDst, + (vx_reference)pDstRoi, + (vx_reference)windowFunction, + (vx_reference)centerWindows, + (vx_reference)reflectPadding, + (vx_reference)spectrogramLayout, + (vx_reference)power, + (vx_reference)nfft, + (vx_reference)windowLength, + (vx_reference)windowStep, + (vx_reference)deviceType}; + node = createNode(graph, VX_KERNEL_RPP_SPECTROGRAM, params, 13); + } + return node; +} + RpptDataType getRpptDataType(vx_enum vxDataType) { switch(vxDataType) { case vx_type_e::VX_TYPE_FLOAT32: @@ -2571,6 +2597,34 @@ RpptDataType getRpptDataType(vx_enum vxDataType) { } } +RpptLayout getRpptLayout(vxTensorLayout layout) { + switch(layout) { + case vxTensorLayout::VX_NHWC: + return RpptLayout::NHWC; + case vxTensorLayout::VX_NCHW: + return RpptLayout::NCHW; + case vxTensorLayout::VX_NFHWC: + return RpptLayout::NHWC; + case vxTensorLayout::VX_NFCHW: + return RpptLayout::NCHW; +#if RPP_AUDIO + case vxTensorLayout::VX_NHW: + return RpptLayout::NHW; + case vxTensorLayout::VX_NFT: + return RpptLayout::NFT; + case vxTensorLayout::VX_NTF: + return RpptLayout::NTF; +#else + case vxTensorLayout::VX_NHW: + case vxTensorLayout::VX_NFT: + case vxTensorLayout::VX_NTF: + throw std::runtime_error("RPP_AUDIO flag disabled, Audio layouts are not supported"); +#endif + default: + throw std::runtime_error("Invalid layout"); + } +} + void fillDescriptionPtrfromDims(RpptDescPtr &descPtr, vxTensorLayout layout, size_t *tensorDims) { switch(layout) { case vxTensorLayout::VX_NHWC: { @@ -2627,16 +2681,17 @@ void fillDescriptionPtrfromDims(RpptDescPtr &descPtr, vxTensorLayout layout, siz } } -void fillAudioDescriptionPtrFromDims(RpptDescPtr &descPtr, size_t *tensorDims) { - descPtr->n = tensorDims[0]; - descPtr->h = tensorDims[2]; - descPtr->w = tensorDims[1]; +void fillAudioDescriptionPtrFromDims(RpptDescPtr &descPtr, size_t *maxTensorDims, vxTensorLayout layout) { + descPtr->n = maxTensorDims[0]; + descPtr->h = maxTensorDims[1]; + descPtr->w = maxTensorDims[2]; descPtr->c = 1; descPtr->strides.nStride = descPtr->c * descPtr->w * descPtr->h; descPtr->strides.hStride = descPtr->c * descPtr->w; descPtr->strides.wStride = descPtr->c; descPtr->strides.cStride = 1; descPtr->numDims = 4; + descPtr->layout = getRpptLayout(layout); } // utility functions diff --git a/amd_openvx_extensions/amd_rpp/source/tensor/PreemphasisFilter.cpp b/amd_openvx_extensions/amd_rpp/source/tensor/PreemphasisFilter.cpp index abf2780d8..416d919b1 100644 --- a/amd_openvx_extensions/amd_rpp/source/tensor/PreemphasisFilter.cpp +++ b/amd_openvx_extensions/amd_rpp/source/tensor/PreemphasisFilter.cpp @@ -98,8 +98,12 @@ static vx_status VX_CALLBACK processPreemphasisFilter(vx_node node, const vx_ref #endif } if (data->deviceType == AGO_TARGET_AFFINITY_CPU) { +#if RPP_AUDIO rpp_status = rppt_pre_emphasis_filter_host((float *)data->pSrc, data->pSrcDesc, (float *)data->pDst, data->pDstDesc, (Rpp32s *)data->pSampleSize, data->pPreemphCoeff, RpptAudioBorderType(data->borderType), data->handle->rppHandle); return_status = (rpp_status == RPP_SUCCESS) ? VX_SUCCESS : VX_FAILURE; +#else + return_status = VX_ERROR_NOT_SUPPORTED; +#endif } return return_status; } diff --git a/amd_openvx_extensions/amd_rpp/source/tensor/Spectrogram.cpp b/amd_openvx_extensions/amd_rpp/source/tensor/Spectrogram.cpp new file mode 100644 index 000000000..610f2bcfb --- /dev/null +++ b/amd_openvx_extensions/amd_rpp/source/tensor/Spectrogram.cpp @@ -0,0 +1,272 @@ +/* +Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include "internal_publishKernels.h" + +struct SpectrogramLocalData { + vxRppHandle *handle; + Rpp32u deviceType; + RppPtr_t pSrc; + RppPtr_t pDst; + bool centerWindows; + bool reflectPadding; + vxTensorLayout spectrogramLayout; + Rpp32s power; + Rpp32s nfft; + Rpp32s windowLength; + Rpp32s windowOffset; + Rpp32s windowStep; + RpptDescPtr pSrcDesc; + RpptDescPtr pDstDesc; + Rpp32s *pSrcLength; + Rpp32f *pWindowFn; + size_t inputTensorDims[RPP_MAX_TENSOR_DIMS]; + size_t outputTensorDims[RPP_MAX_TENSOR_DIMS]; +}; + +void updateDstRoi(SpectrogramLocalData *data, RpptROI *src_roi, RpptROI *dst_roi) { + const Rpp32s num_frames = ((data->nfft / 2) + 1); + for (unsigned i = 0; i < data->inputTensorDims[0]; i++) { + data->pSrcLength[i] = static_cast(src_roi[i].xywhROI.roiWidth); + if (data->spectrogramLayout == vxTensorLayout::VX_NTF) { + dst_roi[i].xywhROI.roiWidth = ((data->pSrcLength[i] - data->windowOffset) / data->windowStep) + 1; + dst_roi[i].xywhROI.roiHeight = num_frames; + } else if (data->spectrogramLayout == vxTensorLayout::VX_NFT) { + dst_roi[i].xywhROI.roiWidth = num_frames; + dst_roi[i].xywhROI.roiHeight = ((data->pSrcLength[i] - data->windowOffset) / data->windowStep) + 1; + } + } +} + +static vx_status VX_CALLBACK refreshSpectrogram(vx_node node, const vx_reference *parameters, SpectrogramLocalData *data) { + vx_status status = VX_SUCCESS; + vx_status return_status = VX_SUCCESS; + void *roi_tensor_ptr_src, *roi_tensor_ptr_dst; + if (data->deviceType == AGO_TARGET_AFFINITY_GPU) { +#if ENABLE_OPENCL || ENABLE_HIP + return_status = VX_ERROR_NOT_IMPLEMENTED; +#endif + } else if (data->deviceType == AGO_TARGET_AFFINITY_CPU) { + STATUS_ERROR_CHECK(vxQueryTensor((vx_tensor)parameters[0], VX_TENSOR_BUFFER_HOST, &data->pSrc, sizeof(data->pSrc))); + STATUS_ERROR_CHECK(vxQueryTensor((vx_tensor)parameters[1], VX_TENSOR_BUFFER_HOST, &roi_tensor_ptr_src, sizeof(roi_tensor_ptr_src))); + STATUS_ERROR_CHECK(vxQueryTensor((vx_tensor)parameters[2], VX_TENSOR_BUFFER_HOST, &data->pDst, sizeof(data->pDst))); + STATUS_ERROR_CHECK(vxQueryTensor((vx_tensor)parameters[3], VX_TENSOR_BUFFER_HOST, &roi_tensor_ptr_dst, sizeof(roi_tensor_ptr_dst))); + } + updateDstRoi(data, reinterpret_cast(roi_tensor_ptr_src), reinterpret_cast(roi_tensor_ptr_dst)); + return status; +} + +static vx_status VX_CALLBACK validateSpectrogram(vx_node node, const vx_reference parameters[], vx_uint32 num, vx_meta_format metas[]) { + vx_status status = VX_SUCCESS; + vx_enum scalar_type; + STATUS_ERROR_CHECK(vxQueryScalar((vx_scalar)parameters[5], VX_SCALAR_TYPE, &scalar_type, sizeof(scalar_type))); + if (scalar_type != VX_TYPE_BOOL) + return ERRMSG(VX_ERROR_INVALID_TYPE, "validate: Parameter: #5 type=%d (must be size)\n", scalar_type); + STATUS_ERROR_CHECK(vxQueryScalar((vx_scalar)parameters[6], VX_SCALAR_TYPE, &scalar_type, sizeof(scalar_type))); + if (scalar_type != VX_TYPE_BOOL) + return ERRMSG(VX_ERROR_INVALID_TYPE, "validate: Parameter: #6 type=%d (must be size)\n", scalar_type); + STATUS_ERROR_CHECK(vxQueryScalar((vx_scalar)parameters[7], VX_SCALAR_TYPE, &scalar_type, sizeof(scalar_type))); + if (scalar_type != VX_TYPE_INT32) + return ERRMSG(VX_ERROR_INVALID_TYPE, "validate: Parameter: #7 type=%d (must be size)\n", scalar_type); + STATUS_ERROR_CHECK(vxQueryScalar((vx_scalar)parameters[8], VX_SCALAR_TYPE, &scalar_type, sizeof(scalar_type))); + if (scalar_type != VX_TYPE_INT32) + return ERRMSG(VX_ERROR_INVALID_TYPE, "validate: Parameter: #8 type=%d (must be size)\n", scalar_type); + STATUS_ERROR_CHECK(vxQueryScalar((vx_scalar)parameters[9], VX_SCALAR_TYPE, &scalar_type, sizeof(scalar_type))); + if (scalar_type != VX_TYPE_INT32) + return ERRMSG(VX_ERROR_INVALID_TYPE, "validate: Parameter: #9 type=%d (must be size)\n", scalar_type); + STATUS_ERROR_CHECK(vxQueryScalar((vx_scalar)parameters[10], VX_SCALAR_TYPE, &scalar_type, sizeof(scalar_type))); + if (scalar_type != VX_TYPE_INT32) + return ERRMSG(VX_ERROR_INVALID_TYPE, "validate: Parameter: #10 type=%d (must be size)\n", scalar_type); + STATUS_ERROR_CHECK(vxQueryScalar((vx_scalar)parameters[11], VX_SCALAR_TYPE, &scalar_type, sizeof(scalar_type))); + if (scalar_type != VX_TYPE_INT32) + return ERRMSG(VX_ERROR_INVALID_TYPE, "validate: Parameter: #11 type=%d (must be size)\n", scalar_type); + STATUS_ERROR_CHECK(vxQueryScalar((vx_scalar)parameters[12], VX_SCALAR_TYPE, &scalar_type, sizeof(scalar_type))); + if (scalar_type != VX_TYPE_UINT32) + return ERRMSG(VX_ERROR_INVALID_TYPE, "validate: Parameter: #12 type=%d (must be size)\n", scalar_type); + + // Check for input parameters + size_t num_tensor_dims; + STATUS_ERROR_CHECK(vxQueryTensor((vx_tensor)parameters[0], VX_TENSOR_NUMBER_OF_DIMS, &num_tensor_dims, sizeof(num_tensor_dims))); + if (num_tensor_dims < 3) return ERRMSG(VX_ERROR_INVALID_DIMENSION, "validate: Spectrogram: tensor: #0 dimensions=%lu (must be greater than or equal to 3)\n", num_tensor_dims); + + // Check for output parameters + vx_uint8 tensor_fixed_point_position; + size_t tensor_dims[RPP_MAX_TENSOR_DIMS]; + vx_enum tensor_datatype; + STATUS_ERROR_CHECK(vxQueryTensor((vx_tensor)parameters[2], VX_TENSOR_NUMBER_OF_DIMS, &num_tensor_dims, sizeof(num_tensor_dims))); + if (num_tensor_dims < 3) return ERRMSG(VX_ERROR_INVALID_DIMENSION, "validate: Spectrogram: tensor: #2 dimensions=%lu (must be greater than or equal to 3)\n", num_tensor_dims); + STATUS_ERROR_CHECK(vxQueryTensor((vx_tensor)parameters[2], VX_TENSOR_DIMS, &tensor_dims, sizeof(tensor_dims))); + STATUS_ERROR_CHECK(vxQueryTensor((vx_tensor)parameters[2], VX_TENSOR_DATA_TYPE, &tensor_datatype, sizeof(tensor_datatype))); + STATUS_ERROR_CHECK(vxQueryTensor((vx_tensor)parameters[2], VX_TENSOR_FIXED_POINT_POSITION, &tensor_fixed_point_position, sizeof(tensor_fixed_point_position))); + STATUS_ERROR_CHECK(vxSetMetaFormatAttribute(metas[2], VX_TENSOR_NUMBER_OF_DIMS, &num_tensor_dims, sizeof(num_tensor_dims))); + STATUS_ERROR_CHECK(vxSetMetaFormatAttribute(metas[2], VX_TENSOR_DIMS, &tensor_dims, sizeof(tensor_dims))); + STATUS_ERROR_CHECK(vxSetMetaFormatAttribute(metas[2], VX_TENSOR_DATA_TYPE, &tensor_datatype, sizeof(tensor_datatype))); + STATUS_ERROR_CHECK(vxSetMetaFormatAttribute(metas[2], VX_TENSOR_FIXED_POINT_POSITION, &tensor_fixed_point_position, sizeof(tensor_fixed_point_position))); + return status; +} + +static vx_status VX_CALLBACK processSpectrogram(vx_node node, const vx_reference *parameters, vx_uint32 num) { + RppStatus rpp_status = RPP_SUCCESS; + vx_status return_status = VX_SUCCESS; + SpectrogramLocalData *data = NULL; + STATUS_ERROR_CHECK(vxQueryNode(node, VX_NODE_LOCAL_DATA_PTR, &data, sizeof(data))); + refreshSpectrogram(node, parameters, data); + if (data->deviceType == AGO_TARGET_AFFINITY_GPU) { +#if ENABLE_OPENCL || ENABLE_HIP + return_status = VX_ERROR_NOT_IMPLEMENTED; +#endif + } else if (data->deviceType == AGO_TARGET_AFFINITY_CPU) { +#if RPP_AUDIO + rpp_status = rppt_spectrogram_host(data->pSrc, data->pSrcDesc, data->pDst, data->pDstDesc, data->pSrcLength, data->centerWindows, data->reflectPadding, + data->pWindowFn, data->nfft, data->power, data->windowLength, data->windowStep, data->handle->rppHandle); + return_status = (rpp_status == RPP_SUCCESS) ? VX_SUCCESS : VX_FAILURE; +#else + return_status = VX_ERROR_NOT_SUPPORTED; +#endif + } + return return_status; +} + +static vx_status VX_CALLBACK initializeSpectrogram(vx_node node, const vx_reference *parameters, vx_uint32 num) { + SpectrogramLocalData *data = new SpectrogramLocalData; + if (data) { + memset(data, 0, sizeof(SpectrogramLocalData)); + + vx_enum input_tensor_datatype, output_tensor_datatype; + int spectrogram_layout; + STATUS_ERROR_CHECK(vxReadScalarValue((vx_scalar)parameters[5], &data->centerWindows)); + STATUS_ERROR_CHECK(vxReadScalarValue((vx_scalar)parameters[6], &data->reflectPadding)); + STATUS_ERROR_CHECK(vxReadScalarValue((vx_scalar)parameters[7], &spectrogram_layout)); + STATUS_ERROR_CHECK(vxReadScalarValue((vx_scalar)parameters[8], &data->power)); + STATUS_ERROR_CHECK(vxReadScalarValue((vx_scalar)parameters[9], &data->nfft)); + STATUS_ERROR_CHECK(vxReadScalarValue((vx_scalar)parameters[10], &data->windowLength)); + STATUS_ERROR_CHECK(vxReadScalarValue((vx_scalar)parameters[11], &data->windowStep)); + STATUS_ERROR_CHECK(vxCopyScalar((vx_scalar)parameters[12], &data->deviceType, VX_READ_ONLY, VX_MEMORY_TYPE_HOST)); + data->spectrogramLayout = static_cast(spectrogram_layout); + data->windowOffset = (!data->centerWindows) ? data->windowLength : 0; + + // Querying for input tensor + data->pSrcDesc = new RpptDesc; + STATUS_ERROR_CHECK(vxQueryTensor((vx_tensor)parameters[0], VX_TENSOR_NUMBER_OF_DIMS, &data->pSrcDesc->numDims, sizeof(data->pSrcDesc->numDims))); + STATUS_ERROR_CHECK(vxQueryTensor((vx_tensor)parameters[0], VX_TENSOR_DIMS, &data->inputTensorDims, sizeof(vx_size) * data->pSrcDesc->numDims)); + STATUS_ERROR_CHECK(vxQueryTensor((vx_tensor)parameters[0], VX_TENSOR_DATA_TYPE, &input_tensor_datatype, sizeof(input_tensor_datatype))); + data->pSrcDesc->dataType = getRpptDataType(input_tensor_datatype); + data->pSrcDesc->offsetInBytes = 0; + fillAudioDescriptionPtrFromDims(data->pSrcDesc, data->inputTensorDims); + + // Querying for output tensor + data->pDstDesc = new RpptDesc; + STATUS_ERROR_CHECK(vxQueryTensor((vx_tensor)parameters[2], VX_TENSOR_NUMBER_OF_DIMS, &data->pDstDesc->numDims, sizeof(data->pDstDesc->numDims))); + STATUS_ERROR_CHECK(vxQueryTensor((vx_tensor)parameters[2], VX_TENSOR_DIMS, &data->outputTensorDims, sizeof(vx_size) * data->pDstDesc->numDims)); + STATUS_ERROR_CHECK(vxQueryTensor((vx_tensor)parameters[2], VX_TENSOR_DATA_TYPE, &output_tensor_datatype, sizeof(output_tensor_datatype))); + data->pDstDesc->dataType = getRpptDataType(output_tensor_datatype); + data->pDstDesc->offsetInBytes = 0; + fillAudioDescriptionPtrFromDims(data->pDstDesc, data->outputTensorDims, data->spectrogramLayout); + + data->pSrcLength = new int[data->pSrcDesc->n]; + data->pWindowFn = new float[data->windowLength]; + + STATUS_ERROR_CHECK(vxCopyArrayRange((vx_array)parameters[4], 0, data->windowLength, sizeof(float), data->pWindowFn, VX_READ_ONLY, VX_MEMORY_TYPE_HOST)); + STATUS_ERROR_CHECK(createRPPHandle(node, &data->handle, data->pSrcDesc->n, data->deviceType)); + STATUS_ERROR_CHECK(vxSetNodeAttribute(node, VX_NODE_LOCAL_DATA_PTR, &data, sizeof(data))); + return VX_SUCCESS; + } else { + return VX_FAILURE;; + } +} + +static vx_status VX_CALLBACK uninitializeSpectrogram(vx_node node, const vx_reference *parameters, vx_uint32 num) { + SpectrogramLocalData *data; + STATUS_ERROR_CHECK(vxQueryNode(node, VX_NODE_LOCAL_DATA_PTR, &data, sizeof(data))); + if (data->pSrcLength) delete[] data->pSrcLength; + if (data->pWindowFn) delete[] data->pWindowFn; + if (data->pSrcDesc) delete data->pSrcDesc; + if (data->pDstDesc) delete data->pDstDesc; + STATUS_ERROR_CHECK(releaseRPPHandle(node, data->handle, data->deviceType)); + delete data; + return VX_SUCCESS; +} + +//! \brief The kernel target support callback. +// TODO::currently the node is setting the same affinity as context. This needs to change when we have hybrid modes in the same graph +static vx_status VX_CALLBACK query_target_support(vx_graph graph, vx_node node, + vx_bool use_opencl_1_2, // [input] false: OpenCL driver is 2.0+; true: OpenCL driver is 1.2 + vx_uint32 &supported_target_affinity // [output] must be set to AGO_TARGET_AFFINITY_CPU or AGO_TARGET_AFFINITY_GPU or (AGO_TARGET_AFFINITY_CPU | AGO_TARGET_AFFINITY_GPU) +) { + vx_context context = vxGetContext((vx_reference)graph); + AgoTargetAffinityInfo affinity; + vxQueryContext(context, VX_CONTEXT_ATTRIBUTE_AMD_AFFINITY, &affinity, sizeof(affinity)); + if (affinity.device_type == AGO_TARGET_AFFINITY_GPU) + supported_target_affinity = AGO_TARGET_AFFINITY_GPU; + else + supported_target_affinity = AGO_TARGET_AFFINITY_CPU; + + return VX_SUCCESS; +} + +vx_status Spectrogram_Register(vx_context context) { + vx_status status = VX_SUCCESS; + // Add kernel to the context with callbacks + vx_kernel kernel = vxAddUserKernel(context, "org.rpp.Spectrogram", + VX_KERNEL_RPP_SPECTROGRAM, + processSpectrogram, + 13, + validateSpectrogram, + initializeSpectrogram, + uninitializeSpectrogram); + ERROR_CHECK_OBJECT(kernel); + AgoTargetAffinityInfo affinity; + vxQueryContext(context, VX_CONTEXT_ATTRIBUTE_AMD_AFFINITY, &affinity, sizeof(affinity)); +#if ENABLE_HIP + vx_bool enableBufferAccess = vx_true_e; + if (affinity.device_type == AGO_TARGET_AFFINITY_GPU) + STATUS_ERROR_CHECK(vxSetKernelAttribute(kernel, VX_KERNEL_ATTRIBUTE_AMD_GPU_BUFFER_ACCESS_ENABLE, &enableBufferAccess, sizeof(enableBufferAccess))); +#else + vx_bool enableBufferAccess = vx_false_e; +#endif + amd_kernel_query_target_support_f query_target_support_f = query_target_support; + + if (kernel) { + STATUS_ERROR_CHECK(vxSetKernelAttribute(kernel, VX_KERNEL_ATTRIBUTE_AMD_QUERY_TARGET_SUPPORT, &query_target_support_f, sizeof(query_target_support_f))); + PARAM_ERROR_CHECK(vxAddParameterToKernel(kernel, 0, VX_INPUT, VX_TYPE_TENSOR, VX_PARAMETER_STATE_REQUIRED)); + PARAM_ERROR_CHECK(vxAddParameterToKernel(kernel, 1, VX_INPUT, VX_TYPE_TENSOR, VX_PARAMETER_STATE_REQUIRED)); + PARAM_ERROR_CHECK(vxAddParameterToKernel(kernel, 2, VX_OUTPUT, VX_TYPE_TENSOR, VX_PARAMETER_STATE_REQUIRED)); + PARAM_ERROR_CHECK(vxAddParameterToKernel(kernel, 3, VX_INPUT, VX_TYPE_TENSOR, VX_PARAMETER_STATE_REQUIRED)); + PARAM_ERROR_CHECK(vxAddParameterToKernel(kernel, 4, VX_INPUT, VX_TYPE_ARRAY, VX_PARAMETER_STATE_REQUIRED)); + PARAM_ERROR_CHECK(vxAddParameterToKernel(kernel, 5, VX_INPUT, VX_TYPE_SCALAR, VX_PARAMETER_STATE_REQUIRED)); + PARAM_ERROR_CHECK(vxAddParameterToKernel(kernel, 6, VX_INPUT, VX_TYPE_SCALAR, VX_PARAMETER_STATE_REQUIRED)); + PARAM_ERROR_CHECK(vxAddParameterToKernel(kernel, 7, VX_INPUT, VX_TYPE_SCALAR, VX_PARAMETER_STATE_REQUIRED)); + PARAM_ERROR_CHECK(vxAddParameterToKernel(kernel, 8, VX_INPUT, VX_TYPE_SCALAR, VX_PARAMETER_STATE_REQUIRED)); + PARAM_ERROR_CHECK(vxAddParameterToKernel(kernel, 9, VX_INPUT, VX_TYPE_SCALAR, VX_PARAMETER_STATE_REQUIRED)); + PARAM_ERROR_CHECK(vxAddParameterToKernel(kernel, 10, VX_INPUT, VX_TYPE_SCALAR, VX_PARAMETER_STATE_REQUIRED)); + PARAM_ERROR_CHECK(vxAddParameterToKernel(kernel, 11, VX_INPUT, VX_TYPE_SCALAR, VX_PARAMETER_STATE_REQUIRED)); + PARAM_ERROR_CHECK(vxAddParameterToKernel(kernel, 12, VX_INPUT, VX_TYPE_SCALAR, VX_PARAMETER_STATE_REQUIRED)); + PARAM_ERROR_CHECK(vxFinalizeKernel(kernel)); + } + if (status != VX_SUCCESS) { + exit: + vxRemoveKernel(kernel); + return VX_FAILURE; + } + + return status; +}