Skip to content

Commit

Permalink
Merge pull request opencv#23401 from fengyuentau:fix_cann_layer_support
Browse files Browse the repository at this point in the history 
dnn: Support more operators in CANN backend opencv#23401

This PR adds the support of following layers:

- [x] Sub
- [x] PRelu
- [x] DeConv
- [x] Also warn users if backend is switched back to default if some of the layers are not supported.
- [ ] [Dropped] LSTM: some hacks (adding layers) were introduced which makes it even harder to build the graph for CANN backend.

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [x] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake
  • Loading branch information
@fengyuentau @thewoz
fengyuentau authored and thewoz committed Jan 4, 2024
1 parent 5d2948d commit e51a634
Show file tree
Hide file tree
Showing 21 changed files with 178 additions and 80 deletions.
6 changes: 4 additions & 2 deletions modules/dnn/include/opencv2/dnn/dnn.hpp
View file
Expand Up @@ -347,10 +347,12 @@ CV__DNN_INLINE_NS_BEGIN
/**
* @brief Returns a CANN backend node
*
* @param inputsWrapper input tensors of this CANN operator
* @param inputs input tensors of CANN operator
* @param outputs output tensors of CANN operator
* @param nodes nodes of input tensors
*/
virtual Ptr<BackendNode> initCann(const std::vector<Ptr<BackendWrapper> > &inputsWrapper,
virtual Ptr<BackendNode> initCann(const std::vector<Ptr<BackendWrapper> > &inputs,
const std::vector<Ptr<BackendWrapper> > &outputs,
const std::vector<Ptr<BackendNode> >& nodes);

/**
Expand Down
3 changes: 2 additions & 1 deletion modules/dnn/src/layer.cpp
View file
Expand Up @@ -84,7 +84,8 @@ Ptr<BackendNode> Layer::initTimVX(void* timVxInfo,
return Ptr<BackendNode>();
}

Ptr<BackendNode> Layer::initCann(const std::vector<Ptr<BackendWrapper> > &inputsWrapper,
Ptr<BackendNode> Layer::initCann(const std::vector<Ptr<BackendWrapper> > &inputs,
const std::vector<Ptr<BackendWrapper> > &outputs,
const std::vector<Ptr<BackendNode> >& nodes)
{
CV_Error(Error::StsNotImplemented, "CANN pipeline of " + type + " layers is not defined.");
Expand Down
5 changes: 3 additions & 2 deletions modules/dnn/src/layers/batch_norm_layer.cpp
View file
Expand Up @@ -392,13 +392,14 @@ class BatchNormLayerImpl CV_FINAL : public BatchNormLayer
#endif // HAVE_HALIDE

#ifdef HAVE_CANN
virtual Ptr<BackendNode> initCann(const std::vector<Ptr<BackendWrapper> > &inputsWrapper,
virtual Ptr<BackendNode> initCann(const std::vector<Ptr<BackendWrapper> > &inputs,
const std::vector<Ptr<BackendWrapper> > &outputs,
const std::vector<Ptr<BackendNode> >& nodes) CV_OVERRIDE
{
CV_Assert(nodes.size() == 1);
CV_Assert(blobs.size() == 4); // must have scale, offset, mean and variance

auto x = inputsWrapper[0].dynamicCast<CannBackendWrapper>();
auto x = inputs[0].dynamicCast<CannBackendWrapper>();
auto channel = x->host->size[1];

// create operator
Expand Down
5 changes: 3 additions & 2 deletions modules/dnn/src/layers/blank_layer.cpp
View file
Expand Up @@ -121,10 +121,11 @@ class BlankLayerImpl CV_FINAL : public BlankLayer
}

#ifdef HAVE_CANN
virtual Ptr<BackendNode> initCann(const std::vector<Ptr<BackendWrapper> > &inputsWrapper,
virtual Ptr<BackendNode> initCann(const std::vector<Ptr<BackendWrapper> > &inputs,
const std::vector<Ptr<BackendWrapper> > &outputs,
const std::vector<Ptr<BackendNode> >& nodes) CV_OVERRIDE
{
auto x = inputsWrapper[0].dynamicCast<CannBackendWrapper>();
auto x = inputs[0].dynamicCast<CannBackendWrapper>();
auto x_desc = x->getTensorDesc();
auto op_x = nodes[0].dynamicCast<CannBackendNode>()->getOp();
auto output_desc = std::make_shared<ge::TensorDesc>(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT);
Expand Down
9 changes: 5 additions & 4 deletions modules/dnn/src/layers/concat_layer.cpp
View file
Expand Up @@ -367,24 +367,25 @@ class ConcatLayerImpl CV_FINAL : public ConcatLayer
}

#ifdef HAVE_CANN
virtual Ptr<BackendNode> initCann(const std::vector<Ptr<BackendWrapper> > &inputsWrapper,
virtual Ptr<BackendNode> initCann(const std::vector<Ptr<BackendWrapper> > &inputs,
const std::vector<Ptr<BackendWrapper> > &outputs,
const std::vector<Ptr<BackendNode> >& nodes) CV_OVERRIDE
{
CV_Assert(inputsWrapper.size() == nodes.size());
CV_Assert(inputs.size() == nodes.size());

// create operator
auto op = std::make_shared<ge::op::ConcatD>(name);

// set attributes
int N = inputsWrapper.size();
int N = inputs.size();
op->set_attr_concat_dim(axis);
op->set_attr_N(N);

// set inputs : x (dynamic)
op->create_dynamic_input_x(N);
for (int i = 0; i < N; i++)
{
auto x_i = inputsWrapper[i].dynamicCast<CannBackendWrapper>();
auto x_i = inputs[i].dynamicCast<CannBackendWrapper>();
auto x_i_desc = x_i->getTensorDesc();
auto op_x_i = nodes[i].dynamicCast<CannBackendNode>()->getOp();
op->set_dynamic_input_x(i, *op_x_i, x_i->name.c_str());
Expand Down
3 changes: 2 additions & 1 deletion modules/dnn/src/layers/const_layer.cpp
View file
Expand Up @@ -84,7 +84,8 @@ class ConstLayerImpl CV_FINAL : public ConstLayer
}

#ifdef HAVE_CANN
virtual Ptr<BackendNode> initCann(const std::vector<Ptr<BackendWrapper> > &inputsWrapper,
virtual Ptr<BackendNode> initCann(const std::vector<Ptr<BackendWrapper> > &inputs,
const std::vector<Ptr<BackendWrapper> > &outputs,
const std::vector<Ptr<BackendNode> >& nodes) CV_OVERRIDE
{
auto mat_shape = shape(blobs[0]);
Expand Down
83 changes: 79 additions & 4 deletions modules/dnn/src/layers/convolution_layer.cpp
View file
Expand Up @@ -782,16 +782,17 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl
}

#ifdef HAVE_CANN
virtual Ptr<BackendNode> initCann(const std::vector<Ptr<BackendWrapper> > &inputsWrapper,
virtual Ptr<BackendNode> initCann(const std::vector<Ptr<BackendWrapper> > &inputs,
const std::vector<Ptr<BackendWrapper> > &outputs,
const std::vector<Ptr<BackendNode> >& nodes) CV_OVERRIDE
{
CV_Assert(!blobs.empty());
CV_Assert(inputsWrapper.size() == 1);
CV_Assert(inputs.size() == 1);
CV_Assert(nodes.size() == 1);

bool has_bias = hasBias() || fusedBias;

auto x = inputsWrapper[0].dynamicCast<CannBackendWrapper>();
auto x = inputs[0].dynamicCast<CannBackendWrapper>();
const auto shape_x = x->host->size; // [b, c, h, w]
const int filter_out_channel = blobs[0].size[1];
const int groups = shape_x[1] / filter_out_channel;
Expand Down Expand Up @@ -1611,7 +1612,8 @@ class DeConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl
#endif // HAVE_INF_ENGINE
{
return backendId == DNN_BACKEND_CUDA ||
(kernel_size.size() == 2 && (backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE));
(kernel_size.size() == 2 && (backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE)) ||
(kernel_size.size() == 2 && backendId == DNN_BACKEND_CANN);
}
}

Expand Down Expand Up @@ -2272,6 +2274,79 @@ class DeConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl
return Ptr<BackendNode>();
}

#ifdef HAVE_CANN
virtual Ptr<BackendNode> initCann(const std::vector<Ptr<BackendWrapper> > &inputs,
const std::vector<Ptr<BackendWrapper> > &outputs,
const std::vector<Ptr<BackendNode> >& nodes) CV_OVERRIDE
{
CV_Assert(!blobs.empty());
CV_Assert(inputs.size() == 1);
CV_Assert(nodes.size() == 1);

bool has_bias = hasBias() || fusedBias;

auto x = inputs[0].dynamicCast<CannBackendWrapper>();
auto y = outputs[0].dynamicCast<CannBackendWrapper>();
const auto shape_x = x->host->size; // [N, C, H, W]
const auto shape_y = y->host->size; // [N, C, H, W]
const int filter_out_channel = blobs[0].size[0];
const int groups = shape_x[1] / filter_out_channel;

// create operator
auto op = std::make_shared<ge::op::Conv2DTransposeD>(name);

// set attributes
op->set_attr_input_size(
ge::Operator::OpListInt({(int64_t)shape_y[0],
(int64_t)shape_y[1],
(int64_t)shape_y[2],
(int64_t)shape_y[3],})
);
op->set_attr_strides(
ge::Operator::OpListInt({1, 1, (int64_t)strides[0], (int64_t)strides[1]})
);
op->set_attr_pads(ge::Operator::OpListInt(
{(int64_t)pads_begin[1], (int64_t)pads_end[1], (int64_t)pads_begin[0], (int64_t)pads_end[0]}
));
op->set_attr_dilations(ge::Operator::OpListInt(
{1, 1, (int64_t)dilations[0], (int64_t)dilations[1]}
));
op->set_attr_groups(groups);
op->set_attr_data_format("NCHW");
op->set_attr_output_padding(
ge::Operator::OpListInt({0, 0, (int64_t)adjust_pads[0], (int64_t)adjust_pads[1]}) // adjust_pads: [height, width]
);

// set inputs
// set inputs : x
auto op_x = nodes[0].dynamicCast<CannBackendNode>()->getOp();
op->set_input_x_by_name(*op_x, x->name.c_str());
auto desc_x = x->getTensorDesc();
op->update_input_desc_x(*desc_x);
// set inputs : weight
const Mat& mat_w = blobs[0];
auto op_const_w = std::make_shared<CannConstOp>(mat_w.data, mat_w.type(), shape(mat_w), cv::format("%s_w", name.c_str()));
op->set_input_filter(*(op_const_w->getOp()));
op->update_input_desc_filter(*(op_const_w->getTensorDesc()));
// set inputs : bias
if (has_bias)
{
int out_channel = blobs[0].size[0];
const Mat& mat_b = blobs[1];

std::vector<int> shape_b{out_channel};
auto op_const_b = std::make_shared<CannConstOp>(mat_b.data, mat_b.type(), shape_b, cv::format("%s_b", name.c_str()));
op->set_input_bias(*(op_const_b->getOp()));
op->update_input_desc_bias(*(op_const_b->getTensorDesc()));
}

// set outputs
auto desc_output = std::make_shared<ge::TensorDesc>(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT);
op->update_output_desc_y(*desc_output);

return Ptr<BackendNode>(new CannBackendNode(op));
}
#endif // HAVE_CANN

#ifdef HAVE_DNN_NGRAPH
virtual Ptr<BackendNode> initNgraph(const std::vector<Ptr<BackendWrapper> > &inputs,
Expand Down

0 comments on commit e51a634

Please sign in to comment.