From 021e5184bcfdcfd2743a36635f46699ef013637b Mon Sep 17 00:00:00 2001 From: Abduragim Shtanchaev <44877829+Abdurrahheem@users.noreply.github.com> Date: Tue, 14 May 2024 16:45:56 +0400 Subject: [PATCH] Merge pull request #25567 from Abdurrahheem:ash/01D-einsum-test 0/1D Einsum Layer Test #25567 This PR introduces 0/1D test cases for Einsum layer. TODO: - Add support for 0D tensors to Einsum layer ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/dnn/src/layers/einsum_layer.cpp | 18 +++++- modules/dnn/test/test_layers_1d.cpp | 75 +++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 3 deletions(-) diff --git a/modules/dnn/src/layers/einsum_layer.cpp b/modules/dnn/src/layers/einsum_layer.cpp index ca935989aaac..64bfa9c77ee2 100644 --- a/modules/dnn/src/layers/einsum_layer.cpp +++ b/modules/dnn/src/layers/einsum_layer.cpp @@ -65,12 +65,12 @@ static Mat Transpose( bool IsTransposeRequired(size_t input_rank, const std::vector& permutation) { - CV_Assert(input_rank == permutation.size()); // No transpose required for scalars - if (input_rank == 0){ + if (input_rank == 0 || permutation.size() == 0){ return false; } + CV_Assert(input_rank == permutation.size()); // Weeds out cases where permutation is something like [0, 1, 2] for a 3D input and so on bool transpose_required = false; @@ -616,6 +616,10 @@ void LayerEinsumImpl::preProcessInputs(InputArrayOfArrays& inputs_arr) // variable to hold processed version of the original input MatShape input_dims = shape(input); + if (inputSubscriptIndices.empty()){ + homogenizedInputDims.emplace_back(MatShape(numLetterIndices, 1)); + continue; + } const auto& currSubscriptIndices = inputSubscriptIndices[inputIter]; // There should be subscript index (subscript label) for each dim of the input @@ -870,6 +874,9 @@ void LayerEinsumImpl::processEquation(const std::vector& inputs) // Check if number of tokens in equal to number of inputs. // For install "ij, jk -> ik" needs to have 2 inputs tensors int num_input_tensors = inputs.size(); + if (lhs_eq_tokens.empty() || (lhs_eq_tokens.size() == 1 && lhs_eq_tokens[0].empty() && lhs_eq == ",") ) { + return; + } CV_CheckEQ(static_cast(lhs_eq_tokens.size()), num_input_tensors, "Number of input tensors does not match the number of subscripts in the input equation"); @@ -1363,7 +1370,12 @@ Mat LayerEinsumImpl::batchwiseMatMul( } output = Mat(M, N, reshapedInput1.type()); - fastGemm(false, false, 1.0, reshapedInput1, reshapedInput2, 0.0, output, opt); + if (shape(reshapedInput1).empty() && shape(reshapedInput2).empty()) + { + output = reshapedInput1.mul(reshapedInput2); // fastGemm does not support 0D * 0D multiplication + } else { + fastGemm(false, false, 1.0, reshapedInput1, reshapedInput2, 0.0, output, opt); + } output = output.reshape(1, {1, M, N}); } diff --git a/modules/dnn/test/test_layers_1d.cpp b/modules/dnn/test/test_layers_1d.cpp index fd3d299170f0..308d8f38a455 100644 --- a/modules/dnn/test/test_layers_1d.cpp +++ b/modules/dnn/test/test_layers_1d.cpp @@ -682,4 +682,79 @@ INSTANTIATE_TEST_CASE_P(/*nothing*/, Layer_Const_Test, testing::Values( std::vector({4, 1}) )); +typedef testing::TestWithParam, std::string>> Layer_Einsum_Test; +TEST_P(Layer_Einsum_Test, Accuracy_01D) +{ + auto tup = GetParam(); + std::vector input_shape = std::get<0>(tup); + std::string equation = std::get<1>(tup); + + LayerParams lp; + lp.type = "Einsum"; + lp.name = "EinsumLayer"; + lp.set("equation", equation); + lp.set("inputSize", 2); + lp.set("outputSize", 1); + lp.set("inputShapes0", DictValue::arrayInt(&input_shape[0], input_shape.size())); + lp.set("inputShapes1", DictValue::arrayInt(&input_shape[0], input_shape.size())); + + Ptr layer = EinsumLayer::create(lp); + + cv::Mat input1(input_shape.size(), input_shape.data(), CV_32F); + cv::Mat input2(input_shape.size(), input_shape.data(), CV_32F); + cv::randn(input1, 0.0, 1.0); cv::randn(input2, 0.0, 1.0); + + std::vector inputs = {input1, input2}; + std::vector outputs; + runLayer(layer, inputs, outputs); + ASSERT_EQ(1, outputs.size()); + + // create output_ref to compare with outputs + cv::Mat output_ref; + int size[] = {1}; + if (equation == ",->"){ + output_ref = input1.mul(input2); + }else if (equation == "i, i->i"){ + output_ref = input1.mul(input2); + } else if (equation == "i, i->"){ + output_ref = input1.mul(input2); + cv::Scalar sum = cv::sum(output_ref); + output_ref = cv::Mat(0, nullptr, CV_32F, sum[0]); + } else if (equation == "ij, ij->ij"){ + output_ref = input1.mul(input2); + } else if (equation == "ij, ij->i"){ + output_ref = input1.mul(input2); + if (input_shape[0] == 1){ + cv::Scalar sum = cv::sum(output_ref); + output_ref = cv::Mat(1, size, CV_32F, sum[0]); + } else if (input_shape[1] == 1){ + size[0] = input_shape[0]; + output_ref = output_ref.reshape(1, 1, size); + } else { + cv::reduce(output_ref, output_ref, 1, cv::REDUCE_SUM, CV_32F); + size[0] = input_shape[0]; + output_ref = output_ref.reshape(1, 1, size); + } + } else { + output_ref = cv::Mat(); + } + + ASSERT_EQ(shape(output_ref), shape(outputs[0])); + normAssert(output_ref, outputs[0]); +} + +INSTANTIATE_TEST_CASE_P(/*nothing*/, Layer_Einsum_Test, testing::Values( + std::make_tuple(std::vector({}), std::string(",->")), + std::make_tuple(std::vector({1}), std::string("i, i->i")), + std::make_tuple(std::vector({1}), std::string("i, i->")), + std::make_tuple(std::vector({4}), std::string("i, i->i")), + std::make_tuple(std::vector({4}), std::string("i, i->")), + std::make_tuple(std::vector({1, 4}), std::string("ij, ij->ij")), + std::make_tuple(std::vector({4, 1}), std::string("ij, ij->ij")), + std::make_tuple(std::vector({1, 4}), std::string("ij, ij->i")), + std::make_tuple(std::vector({4, 1}), std::string("ij, ij->i")), + std::make_tuple(std::vector({4, 4}), std::string("ij, ij->i")) + )); + + }}