-
-
Notifications
You must be signed in to change notification settings - Fork 55.6k
/
dnn.hpp
1744 lines (1534 loc) · 83.8 KB
/
dnn.hpp
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2013, OpenCV Foundation, all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of the copyright holders may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
#ifndef OPENCV_DNN_DNN_HPP
#define OPENCV_DNN_DNN_HPP
#include <vector>
#include <opencv2/core.hpp>
#include "opencv2/core/async.hpp"
#include "../dnn/version.hpp"
#include <opencv2/dnn/dict.hpp>
namespace cv {
namespace dnn {
CV__DNN_INLINE_NS_BEGIN
//! @addtogroup dnn
//! @{
typedef std::vector<int> MatShape;
/**
* @brief Enum of computation backends supported by layers.
* @see Net::setPreferableBackend
*/
enum Backend
{
//! DNN_BACKEND_DEFAULT equals to DNN_BACKEND_INFERENCE_ENGINE if
//! OpenCV is built with Intel's Inference Engine library or
//! DNN_BACKEND_OPENCV otherwise.
DNN_BACKEND_DEFAULT = 0,
DNN_BACKEND_HALIDE,
DNN_BACKEND_INFERENCE_ENGINE, //!< Intel's Inference Engine computational backend
//!< @sa setInferenceEngineBackendType
DNN_BACKEND_OPENCV,
DNN_BACKEND_VKCOM,
DNN_BACKEND_CUDA,
DNN_BACKEND_WEBNN,
#ifdef __OPENCV_BUILD
DNN_BACKEND_INFERENCE_ENGINE_NGRAPH = 1000000, // internal - use DNN_BACKEND_INFERENCE_ENGINE + setInferenceEngineBackendType()
DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019, // internal - use DNN_BACKEND_INFERENCE_ENGINE + setInferenceEngineBackendType()
#endif
};
/**
* @brief Enum of target devices for computations.
* @see Net::setPreferableTarget
*/
enum Target
{
DNN_TARGET_CPU = 0,
DNN_TARGET_OPENCL,
DNN_TARGET_OPENCL_FP16,
DNN_TARGET_MYRIAD,
DNN_TARGET_VULKAN,
DNN_TARGET_FPGA, //!< FPGA device with CPU fallbacks using Inference Engine's Heterogeneous plugin.
DNN_TARGET_CUDA,
DNN_TARGET_CUDA_FP16,
DNN_TARGET_HDDL
};
CV_EXPORTS std::vector< std::pair<Backend, Target> > getAvailableBackends();
CV_EXPORTS_W std::vector<Target> getAvailableTargets(dnn::Backend be);
/**
* @brief Enables detailed logging of the DNN model loading with CV DNN API.
* @param[in] isDiagnosticsMode Indicates whether diagnostic mode should be set.
*
* Diagnostic mode provides detailed logging of the model loading stage to explore
* potential problems (ex.: not implemented layer type).
*
* @note In diagnostic mode series of assertions will be skipped, it can lead to the
* expected application crash.
*/
CV_EXPORTS void enableModelDiagnostics(bool isDiagnosticsMode);
/** @brief This class provides all data needed to initialize layer.
*
* It includes dictionary with scalar params (which can be read by using Dict interface),
* blob params #blobs and optional meta information: #name and #type of layer instance.
*/
class CV_EXPORTS LayerParams : public Dict
{
public:
//TODO: Add ability to name blob params
std::vector<Mat> blobs; //!< List of learned parameters stored as blobs.
String name; //!< Name of the layer instance (optional, can be used internal purposes).
String type; //!< Type name which was used for creating layer by layer factory (optional).
};
/**
* @brief Derivatives of this class encapsulates functions of certain backends.
*/
class BackendNode
{
public:
explicit BackendNode(int backendId);
virtual ~BackendNode(); //!< Virtual destructor to make polymorphism.
int backendId; //!< Backend identifier.
};
/**
* @brief Derivatives of this class wraps cv::Mat for different backends and targets.
*/
class BackendWrapper
{
public:
BackendWrapper(int backendId, int targetId);
/**
* @brief Wrap cv::Mat for specific backend and target.
* @param[in] targetId Target identifier.
* @param[in] m cv::Mat for wrapping.
*
* Make CPU->GPU data transfer if it's require for the target.
*/
BackendWrapper(int targetId, const cv::Mat& m);
/**
* @brief Make wrapper for reused cv::Mat.
* @param[in] base Wrapper of cv::Mat that will be reused.
* @param[in] shape Specific shape.
*
* Initialize wrapper from another one. It'll wrap the same host CPU
* memory and mustn't allocate memory on device(i.e. GPU). It might
* has different shape. Use in case of CPU memory reusing for reuse
* associated memory on device too.
*/
BackendWrapper(const Ptr<BackendWrapper>& base, const MatShape& shape);
virtual ~BackendWrapper(); //!< Virtual destructor to make polymorphism.
/**
* @brief Transfer data to CPU host memory.
*/
virtual void copyToHost() = 0;
/**
* @brief Indicate that an actual data is on CPU.
*/
virtual void setHostDirty() = 0;
int backendId; //!< Backend identifier.
int targetId; //!< Target identifier.
};
class CV_EXPORTS ActivationLayer;
/** @brief This interface class allows to build new Layers - are building blocks of networks.
*
* Each class, derived from Layer, must implement allocate() methods to declare own outputs and forward() to compute outputs.
* Also before using the new layer into networks you must register your layer by using one of @ref dnnLayerFactory "LayerFactory" macros.
*/
class CV_EXPORTS_W Layer : public Algorithm
{
public:
//! List of learned parameters must be stored here to allow read them by using Net::getParam().
CV_PROP_RW std::vector<Mat> blobs;
/** @brief Computes and sets internal parameters according to inputs, outputs and blobs.
* @deprecated Use Layer::finalize(InputArrayOfArrays, OutputArrayOfArrays) instead
* @param[in] input vector of already allocated input blobs
* @param[out] output vector of already allocated output blobs
*
* If this method is called after network has allocated all memory for input and output blobs
* and before inferencing.
*/
CV_DEPRECATED_EXTERNAL
virtual void finalize(const std::vector<Mat*> &input, std::vector<Mat> &output);
/** @brief Computes and sets internal parameters according to inputs, outputs and blobs.
* @param[in] inputs vector of already allocated input blobs
* @param[out] outputs vector of already allocated output blobs
*
* If this method is called after network has allocated all memory for input and output blobs
* and before inferencing.
*/
CV_WRAP virtual void finalize(InputArrayOfArrays inputs, OutputArrayOfArrays outputs);
/** @brief Given the @p input blobs, computes the output @p blobs.
* @deprecated Use Layer::forward(InputArrayOfArrays, OutputArrayOfArrays, OutputArrayOfArrays) instead
* @param[in] input the input blobs.
* @param[out] output allocated output blobs, which will store results of the computation.
* @param[out] internals allocated internal blobs
*/
CV_DEPRECATED_EXTERNAL
virtual void forward(std::vector<Mat*> &input, std::vector<Mat> &output, std::vector<Mat> &internals);
/** @brief Given the @p input blobs, computes the output @p blobs.
* @param[in] inputs the input blobs.
* @param[out] outputs allocated output blobs, which will store results of the computation.
* @param[out] internals allocated internal blobs
*/
virtual void forward(InputArrayOfArrays inputs, OutputArrayOfArrays outputs, OutputArrayOfArrays internals);
/** @brief Tries to quantize the given layer and compute the quantization parameters required for fixed point implementation.
* @param[in] scales input and output scales.
* @param[in] zeropoints input and output zeropoints.
* @param[out] params Quantized parameters required for fixed point implementation of that layer.
* @returns True if layer can be quantized.
*/
virtual bool tryQuantize(const std::vector<std::vector<float> > &scales,
const std::vector<std::vector<int> > &zeropoints, LayerParams& params);
/** @brief Given the @p input blobs, computes the output @p blobs.
* @param[in] inputs the input blobs.
* @param[out] outputs allocated output blobs, which will store results of the computation.
* @param[out] internals allocated internal blobs
*/
void forward_fallback(InputArrayOfArrays inputs, OutputArrayOfArrays outputs, OutputArrayOfArrays internals);
/** @brief
* @overload
* @deprecated Use Layer::finalize(InputArrayOfArrays, OutputArrayOfArrays) instead
*/
CV_DEPRECATED_EXTERNAL
void finalize(const std::vector<Mat> &inputs, CV_OUT std::vector<Mat> &outputs);
/** @brief
* @overload
* @deprecated Use Layer::finalize(InputArrayOfArrays, OutputArrayOfArrays) instead
*/
CV_DEPRECATED std::vector<Mat> finalize(const std::vector<Mat> &inputs);
/** @brief Allocates layer and computes output.
* @deprecated This method will be removed in the future release.
*/
CV_DEPRECATED CV_WRAP void run(const std::vector<Mat> &inputs, CV_OUT std::vector<Mat> &outputs,
CV_IN_OUT std::vector<Mat> &internals);
/** @brief Returns index of input blob into the input array.
* @param inputName label of input blob
*
* Each layer input and output can be labeled to easily identify them using "%<layer_name%>[.output_name]" notation.
* This method maps label of input blob to its index into input vector.
*/
virtual int inputNameToIndex(String inputName); // FIXIT const
/** @brief Returns index of output blob in output array.
* @see inputNameToIndex()
*/
CV_WRAP virtual int outputNameToIndex(const String& outputName); // FIXIT const
/**
* @brief Ask layer if it support specific backend for doing computations.
* @param[in] backendId computation backend identifier.
* @see Backend
*/
virtual bool supportBackend(int backendId); // FIXIT const
/**
* @brief Returns Halide backend node.
* @param[in] inputs Input Halide buffers.
* @see BackendNode, BackendWrapper
*
* Input buffers should be exactly the same that will be used in forward invocations.
* Despite we can use Halide::ImageParam based on input shape only,
* it helps prevent some memory management issues (if something wrong,
* Halide tests will be failed).
*/
virtual Ptr<BackendNode> initHalide(const std::vector<Ptr<BackendWrapper> > &inputs);
virtual Ptr<BackendNode> initNgraph(const std::vector<Ptr<BackendWrapper> > &inputs, const std::vector<Ptr<BackendNode> >& nodes);
virtual Ptr<BackendNode> initVkCom(const std::vector<Ptr<BackendWrapper> > &inputs);
virtual Ptr<BackendNode> initWebnn(const std::vector<Ptr<BackendWrapper> > &inputs, const std::vector<Ptr<BackendNode> >& nodes);
/**
* @brief Returns a CUDA backend node
*
* @param context void pointer to CSLContext object
* @param inputs layer inputs
* @param outputs layer outputs
*/
virtual Ptr<BackendNode> initCUDA(
void *context,
const std::vector<Ptr<BackendWrapper>>& inputs,
const std::vector<Ptr<BackendWrapper>>& outputs
);
/**
* @brief Automatic Halide scheduling based on layer hyper-parameters.
* @param[in] node Backend node with Halide functions.
* @param[in] inputs Blobs that will be used in forward invocations.
* @param[in] outputs Blobs that will be used in forward invocations.
* @param[in] targetId Target identifier
* @see BackendNode, Target
*
* Layer don't use own Halide::Func members because we can have applied
* layers fusing. In this way the fused function should be scheduled.
*/
virtual void applyHalideScheduler(Ptr<BackendNode>& node,
const std::vector<Mat*> &inputs,
const std::vector<Mat> &outputs,
int targetId) const;
/**
* @brief Implement layers fusing.
* @param[in] node Backend node of bottom layer.
* @see BackendNode
*
* Actual for graph-based backends. If layer attached successfully,
* returns non-empty cv::Ptr to node of the same backend.
* Fuse only over the last function.
*/
virtual Ptr<BackendNode> tryAttach(const Ptr<BackendNode>& node);
/**
* @brief Tries to attach to the layer the subsequent activation layer, i.e. do the layer fusion in a partial case.
* @param[in] layer The subsequent activation layer.
*
* Returns true if the activation layer has been attached successfully.
*/
virtual bool setActivation(const Ptr<ActivationLayer>& layer);
/**
* @brief Try to fuse current layer with a next one
* @param[in] top Next layer to be fused.
* @returns True if fusion was performed.
*/
virtual bool tryFuse(Ptr<Layer>& top);
/**
* @brief Returns parameters of layers with channel-wise multiplication and addition.
* @param[out] scale Channel-wise multipliers. Total number of values should
* be equal to number of channels.
* @param[out] shift Channel-wise offsets. Total number of values should
* be equal to number of channels.
*
* Some layers can fuse their transformations with further layers.
* In example, convolution + batch normalization. This way base layer
* use weights from layer after it. Fused layer is skipped.
* By default, @p scale and @p shift are empty that means layer has no
* element-wise multiplications or additions.
*/
virtual void getScaleShift(Mat& scale, Mat& shift) const;
/**
* @brief Returns scale and zeropoint of layers
* @param[out] scale Output scale
* @param[out] zeropoint Output zeropoint
*
* By default, @p scale is 1 and @p zeropoint is 0.
*/
virtual void getScaleZeropoint(float& scale, int& zeropoint) const;
/**
* @brief "Deattaches" all the layers, attached to particular layer.
*/
virtual void unsetAttached();
virtual bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int requiredOutputs,
std::vector<MatShape> &outputs,
std::vector<MatShape> &internals) const;
virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
const std::vector<MatShape> &outputs) const {CV_UNUSED(inputs); CV_UNUSED(outputs); return 0;}
virtual bool updateMemoryShapes(const std::vector<MatShape> &inputs);
CV_PROP String name; //!< Name of the layer instance, can be used for logging or other internal purposes.
CV_PROP String type; //!< Type name which was used for creating layer by layer factory.
CV_PROP int preferableTarget; //!< prefer target for layer forwarding
Layer();
explicit Layer(const LayerParams ¶ms); //!< Initializes only #name, #type and #blobs fields.
void setParamsFrom(const LayerParams ¶ms); //!< Initializes only #name, #type and #blobs fields.
virtual ~Layer();
};
/** @brief This class allows to create and manipulate comprehensive artificial neural networks.
*
* Neural network is presented as directed acyclic graph (DAG), where vertices are Layer instances,
* and edges specify relationships between layers inputs and outputs.
*
* Each network layer has unique integer id and unique string name inside its network.
* LayerId can store either layer name or layer id.
*
* This class supports reference counting of its instances, i. e. copies point to the same instance.
*/
class CV_EXPORTS_W_SIMPLE Net
{
public:
CV_WRAP Net(); //!< Default constructor.
CV_WRAP ~Net(); //!< Destructor frees the net only if there aren't references to the net anymore.
/** @brief Create a network from Intel's Model Optimizer intermediate representation (IR).
* @param[in] xml XML configuration file with network's topology.
* @param[in] bin Binary file with trained weights.
* Networks imported from Intel's Model Optimizer are launched in Intel's Inference Engine
* backend.
*/
CV_WRAP static Net readFromModelOptimizer(const String& xml, const String& bin);
/** @brief Create a network from Intel's Model Optimizer in-memory buffers with intermediate representation (IR).
* @param[in] bufferModelConfig buffer with model's configuration.
* @param[in] bufferWeights buffer with model's trained weights.
* @returns Net object.
*/
CV_WRAP static
Net readFromModelOptimizer(const std::vector<uchar>& bufferModelConfig, const std::vector<uchar>& bufferWeights);
/** @brief Create a network from Intel's Model Optimizer in-memory buffers with intermediate representation (IR).
* @param[in] bufferModelConfigPtr buffer pointer of model's configuration.
* @param[in] bufferModelConfigSize buffer size of model's configuration.
* @param[in] bufferWeightsPtr buffer pointer of model's trained weights.
* @param[in] bufferWeightsSize buffer size of model's trained weights.
* @returns Net object.
*/
static
Net readFromModelOptimizer(const uchar* bufferModelConfigPtr, size_t bufferModelConfigSize,
const uchar* bufferWeightsPtr, size_t bufferWeightsSize);
/** Returns true if there are no layers in the network. */
CV_WRAP bool empty() const;
/** @brief Dump net to String
* @returns String with structure, hyperparameters, backend, target and fusion
* Call method after setInput(). To see correct backend, target and fusion run after forward().
*/
CV_WRAP String dump();
/** @brief Dump net structure, hyperparameters, backend, target and fusion to dot file
* @param path path to output file with .dot extension
* @see dump()
*/
CV_WRAP void dumpToFile(const String& path);
/** @brief Adds new layer to the net.
* @param name unique name of the adding layer.
* @param type typename of the adding layer (type must be registered in LayerRegister).
* @param dtype datatype of output blobs.
* @param params parameters which will be used to initialize the creating layer.
* @returns unique identifier of created layer, or -1 if a failure will happen.
*/
int addLayer(const String &name, const String &type, const int &dtype, LayerParams ¶ms);
/** @overload Datatype of output blobs set to default CV_32F */
int addLayer(const String &name, const String &type, LayerParams ¶ms);
/** @brief Adds new layer and connects its first input to the first output of previously added layer.
* @see addLayer()
*/
int addLayerToPrev(const String &name, const String &type, const int &dtype, LayerParams ¶ms);
/** @overload */
int addLayerToPrev(const String &name, const String &type, LayerParams ¶ms);
/** @brief Converts string name of the layer to the integer identifier.
* @returns id of the layer, or -1 if the layer wasn't found.
*/
CV_WRAP int getLayerId(const String &layer) const;
CV_WRAP std::vector<String> getLayerNames() const;
/** @brief Container for strings and integers.
*
* @deprecated Use getLayerId() with int result.
*/
typedef DictValue LayerId;
/** @brief Returns pointer to layer with specified id or name which the network use. */
CV_WRAP Ptr<Layer> getLayer(int layerId) const;
/** @overload
* @deprecated Use int getLayerId(const String &layer)
*/
CV_WRAP inline Ptr<Layer> getLayer(const String& layerName) const { return getLayer(getLayerId(layerName)); }
/** @overload
* @deprecated to be removed
*/
CV_WRAP Ptr<Layer> getLayer(const LayerId& layerId) const;
/** @brief Returns pointers to input layers of specific layer. */
std::vector<Ptr<Layer> > getLayerInputs(int layerId) const; // FIXIT: CV_WRAP
/** @brief Connects output of the first layer to input of the second layer.
* @param outPin descriptor of the first layer output.
* @param inpPin descriptor of the second layer input.
*
* Descriptors have the following template <DFN><layer_name>[.input_number]</DFN>:
* - the first part of the template <DFN>layer_name</DFN> is string name of the added layer.
* If this part is empty then the network input pseudo layer will be used;
* - the second optional part of the template <DFN>input_number</DFN>
* is either number of the layer input, either label one.
* If this part is omitted then the first layer input will be used.
*
* @see setNetInputs(), Layer::inputNameToIndex(), Layer::outputNameToIndex()
*/
CV_WRAP void connect(String outPin, String inpPin);
/** @brief Connects #@p outNum output of the first layer to #@p inNum input of the second layer.
* @param outLayerId identifier of the first layer
* @param outNum number of the first layer output
* @param inpLayerId identifier of the second layer
* @param inpNum number of the second layer input
*/
void connect(int outLayerId, int outNum, int inpLayerId, int inpNum);
/** @brief Registers network output with name
*
* Function may create additional 'Identity' layer.
*
* @param outputName identifier of the output
* @param layerId identifier of the second layer
* @param outputPort number of the second layer input
*
* @returns index of bound layer (the same as layerId or newly created)
*/
int registerOutput(const std::string& outputName, int layerId, int outputPort);
/** @brief Sets outputs names of the network input pseudo layer.
*
* Each net always has special own the network input pseudo layer with id=0.
* This layer stores the user blobs only and don't make any computations.
* In fact, this layer provides the only way to pass user data into the network.
* As any other layer, this layer can label its outputs and this function provides an easy way to do this.
*/
CV_WRAP void setInputsNames(const std::vector<String> &inputBlobNames);
/** @brief Specify shape of network input.
*/
CV_WRAP void setInputShape(const String &inputName, const MatShape& shape);
/** @brief Runs forward pass to compute output of layer with name @p outputName.
* @param outputName name for layer which output is needed to get
* @return blob for first output of specified layer.
* @details By default runs forward pass for the whole network.
*/
CV_WRAP Mat forward(const String& outputName = String());
/** @brief Runs forward pass to compute output of layer with name @p outputName.
* @param outputName name for layer which output is needed to get
* @details By default runs forward pass for the whole network.
*
* This is an asynchronous version of forward(const String&).
* dnn::DNN_BACKEND_INFERENCE_ENGINE backend is required.
*/
CV_WRAP AsyncArray forwardAsync(const String& outputName = String());
/** @brief Runs forward pass to compute output of layer with name @p outputName.
* @param outputBlobs contains all output blobs for specified layer.
* @param outputName name for layer which output is needed to get
* @details If @p outputName is empty, runs forward pass for the whole network.
*/
CV_WRAP void forward(OutputArrayOfArrays outputBlobs, const String& outputName = String());
/** @brief Runs forward pass to compute outputs of layers listed in @p outBlobNames.
* @param outputBlobs contains blobs for first outputs of specified layers.
* @param outBlobNames names for layers which outputs are needed to get
*/
CV_WRAP void forward(OutputArrayOfArrays outputBlobs,
const std::vector<String>& outBlobNames);
/** @brief Runs forward pass to compute outputs of layers listed in @p outBlobNames.
* @param outputBlobs contains all output blobs for each layer specified in @p outBlobNames.
* @param outBlobNames names for layers which outputs are needed to get
*/
CV_WRAP_AS(forwardAndRetrieve) void forward(CV_OUT std::vector<std::vector<Mat> >& outputBlobs,
const std::vector<String>& outBlobNames);
/** @brief Returns a quantized Net from a floating-point Net.
* @param calibData Calibration data to compute the quantization parameters.
* @param inputsDtype Datatype of quantized net's inputs. Can be CV_32F or CV_8S.
* @param outputsDtype Datatype of quantized net's outputs. Can be CV_32F or CV_8S.
*/
CV_WRAP Net quantize(InputArrayOfArrays calibData, int inputsDtype, int outputsDtype);
/** @brief Returns input scale and zeropoint for a quantized Net.
* @param scales output parameter for returning input scales.
* @param zeropoints output parameter for returning input zeropoints.
*/
CV_WRAP void getInputDetails(CV_OUT std::vector<float>& scales, CV_OUT std::vector<int>& zeropoints) const;
/** @brief Returns output scale and zeropoint for a quantized Net.
* @param scales output parameter for returning output scales.
* @param zeropoints output parameter for returning output zeropoints.
*/
CV_WRAP void getOutputDetails(CV_OUT std::vector<float>& scales, CV_OUT std::vector<int>& zeropoints) const;
/**
* @brief Compile Halide layers.
* @param[in] scheduler Path to YAML file with scheduling directives.
* @see setPreferableBackend
*
* Schedule layers that support Halide backend. Then compile them for
* specific target. For layers that not represented in scheduling file
* or if no manual scheduling used at all, automatic scheduling will be applied.
*/
CV_WRAP void setHalideScheduler(const String& scheduler);
/**
* @brief Ask network to use specific computation backend where it supported.
* @param[in] backendId backend identifier.
* @see Backend
*
* If OpenCV is compiled with Intel's Inference Engine library, DNN_BACKEND_DEFAULT
* means DNN_BACKEND_INFERENCE_ENGINE. Otherwise it equals to DNN_BACKEND_OPENCV.
*/
CV_WRAP void setPreferableBackend(int backendId);
/**
* @brief Ask network to make computations on specific target device.
* @param[in] targetId target identifier.
* @see Target
*
* List of supported combinations backend / target:
* | | DNN_BACKEND_OPENCV | DNN_BACKEND_INFERENCE_ENGINE | DNN_BACKEND_HALIDE | DNN_BACKEND_CUDA |
* |------------------------|--------------------|------------------------------|--------------------|-------------------|
* | DNN_TARGET_CPU | + | + | + | |
* | DNN_TARGET_OPENCL | + | + | + | |
* | DNN_TARGET_OPENCL_FP16 | + | + | | |
* | DNN_TARGET_MYRIAD | | + | | |
* | DNN_TARGET_FPGA | | + | | |
* | DNN_TARGET_CUDA | | | | + |
* | DNN_TARGET_CUDA_FP16 | | | | + |
* | DNN_TARGET_HDDL | | + | | |
*/
CV_WRAP void setPreferableTarget(int targetId);
/** @brief Sets the new input value for the network
* @param blob A new blob. Should have CV_32F or CV_8U depth.
* @param name A name of input layer.
* @param scalefactor An optional normalization scale.
* @param mean An optional mean subtraction values.
* @see connect(String, String) to know format of the descriptor.
*
* If scale or mean values are specified, a final input blob is computed
* as:
* \f[input(n,c,h,w) = scalefactor \times (blob(n,c,h,w) - mean_c)\f]
*/
CV_WRAP void setInput(InputArray blob, const String& name = "",
double scalefactor = 1.0, const Scalar& mean = Scalar());
/** @brief Sets the new value for the learned param of the layer.
* @param layer name or id of the layer.
* @param numParam index of the layer parameter in the Layer::blobs array.
* @param blob the new value.
* @see Layer::blobs
* @note If shape of the new blob differs from the previous shape,
* then the following forward pass may fail.
*/
CV_WRAP void setParam(int layer, int numParam, const Mat &blob);
CV_WRAP inline void setParam(const String& layerName, int numParam, const Mat &blob) { return setParam(getLayerId(layerName), numParam, blob); }
/** @brief Returns parameter blob of the layer.
* @param layer name or id of the layer.
* @param numParam index of the layer parameter in the Layer::blobs array.
* @see Layer::blobs
*/
CV_WRAP Mat getParam(int layer, int numParam = 0) const;
CV_WRAP inline Mat getParam(const String& layerName, int numParam = 0) const { return getParam(getLayerId(layerName), numParam); }
/** @brief Returns indexes of layers with unconnected outputs.
*
* FIXIT: Rework API to registerOutput() approach, deprecate this call
*/
CV_WRAP std::vector<int> getUnconnectedOutLayers() const;
/** @brief Returns names of layers with unconnected outputs.
*
* FIXIT: Rework API to registerOutput() approach, deprecate this call
*/
CV_WRAP std::vector<String> getUnconnectedOutLayersNames() const;
/** @brief Returns input and output shapes for all layers in loaded model;
* preliminary inferencing isn't necessary.
* @param netInputShapes shapes for all input blobs in net input layer.
* @param layersIds output parameter for layer IDs.
* @param inLayersShapes output parameter for input layers shapes;
* order is the same as in layersIds
* @param outLayersShapes output parameter for output layers shapes;
* order is the same as in layersIds
*/
CV_WRAP void getLayersShapes(const std::vector<MatShape>& netInputShapes,
CV_OUT std::vector<int>& layersIds,
CV_OUT std::vector<std::vector<MatShape> >& inLayersShapes,
CV_OUT std::vector<std::vector<MatShape> >& outLayersShapes) const;
/** @overload */
CV_WRAP void getLayersShapes(const MatShape& netInputShape,
CV_OUT std::vector<int>& layersIds,
CV_OUT std::vector<std::vector<MatShape> >& inLayersShapes,
CV_OUT std::vector<std::vector<MatShape> >& outLayersShapes) const;
/** @brief Returns input and output shapes for layer with specified
* id in loaded model; preliminary inferencing isn't necessary.
* @param netInputShape shape input blob in net input layer.
* @param layerId id for layer.
* @param inLayerShapes output parameter for input layers shapes;
* order is the same as in layersIds
* @param outLayerShapes output parameter for output layers shapes;
* order is the same as in layersIds
*/
void getLayerShapes(const MatShape& netInputShape,
const int layerId,
CV_OUT std::vector<MatShape>& inLayerShapes,
CV_OUT std::vector<MatShape>& outLayerShapes) const; // FIXIT: CV_WRAP
/** @overload */
void getLayerShapes(const std::vector<MatShape>& netInputShapes,
const int layerId,
CV_OUT std::vector<MatShape>& inLayerShapes,
CV_OUT std::vector<MatShape>& outLayerShapes) const; // FIXIT: CV_WRAP
/** @brief Computes FLOP for whole loaded model with specified input shapes.
* @param netInputShapes vector of shapes for all net inputs.
* @returns computed FLOP.
*/
CV_WRAP int64 getFLOPS(const std::vector<MatShape>& netInputShapes) const;
/** @overload */
CV_WRAP int64 getFLOPS(const MatShape& netInputShape) const;
/** @overload */
CV_WRAP int64 getFLOPS(const int layerId,
const std::vector<MatShape>& netInputShapes) const;
/** @overload */
CV_WRAP int64 getFLOPS(const int layerId,
const MatShape& netInputShape) const;
/** @brief Returns list of types for layer used in model.
* @param layersTypes output parameter for returning types.
*/
CV_WRAP void getLayerTypes(CV_OUT std::vector<String>& layersTypes) const;
/** @brief Returns count of layers of specified type.
* @param layerType type.
* @returns count of layers
*/
CV_WRAP int getLayersCount(const String& layerType) const;
/** @brief Computes bytes number which are required to store
* all weights and intermediate blobs for model.
* @param netInputShapes vector of shapes for all net inputs.
* @param weights output parameter to store resulting bytes for weights.
* @param blobs output parameter to store resulting bytes for intermediate blobs.
*/
void getMemoryConsumption(const std::vector<MatShape>& netInputShapes,
CV_OUT size_t& weights, CV_OUT size_t& blobs) const; // FIXIT: CV_WRAP
/** @overload */
CV_WRAP void getMemoryConsumption(const MatShape& netInputShape,
CV_OUT size_t& weights, CV_OUT size_t& blobs) const;
/** @overload */
CV_WRAP void getMemoryConsumption(const int layerId,
const std::vector<MatShape>& netInputShapes,
CV_OUT size_t& weights, CV_OUT size_t& blobs) const;
/** @overload */
CV_WRAP void getMemoryConsumption(const int layerId,
const MatShape& netInputShape,
CV_OUT size_t& weights, CV_OUT size_t& blobs) const;
/** @brief Computes bytes number which are required to store
* all weights and intermediate blobs for each layer.
* @param netInputShapes vector of shapes for all net inputs.
* @param layerIds output vector to save layer IDs.
* @param weights output parameter to store resulting bytes for weights.
* @param blobs output parameter to store resulting bytes for intermediate blobs.
*/
void getMemoryConsumption(const std::vector<MatShape>& netInputShapes,
CV_OUT std::vector<int>& layerIds,
CV_OUT std::vector<size_t>& weights,
CV_OUT std::vector<size_t>& blobs) const; // FIXIT: CV_WRAP
/** @overload */
void getMemoryConsumption(const MatShape& netInputShape,
CV_OUT std::vector<int>& layerIds,
CV_OUT std::vector<size_t>& weights,
CV_OUT std::vector<size_t>& blobs) const; // FIXIT: CV_WRAP
/** @brief Enables or disables layer fusion in the network.
* @param fusion true to enable the fusion, false to disable. The fusion is enabled by default.
*/
CV_WRAP void enableFusion(bool fusion);
/** @brief Returns overall time for inference and timings (in ticks) for layers.
*
* Indexes in returned vector correspond to layers ids. Some layers can be fused with others,
* in this case zero ticks count will be return for that skipped layers. Supported by DNN_BACKEND_OPENCV on DNN_TARGET_CPU only.
*
* @param[out] timings vector for tick timings for all layers.
* @return overall ticks for model inference.
*/
CV_WRAP int64 getPerfProfile(CV_OUT std::vector<double>& timings);
private:
struct Impl;
Ptr<Impl> impl;
};
/** @brief Reads a network model stored in <a href="https://pjreddie.com/darknet/">Darknet</a> model files.
* @param cfgFile path to the .cfg file with text description of the network architecture.
* @param darknetModel path to the .weights file with learned network.
* @returns Network object that ready to do forward, throw an exception in failure cases.
* @returns Net object.
*/
CV_EXPORTS_W Net readNetFromDarknet(const String &cfgFile, const String &darknetModel = String());
/** @brief Reads a network model stored in <a href="https://pjreddie.com/darknet/">Darknet</a> model files.
* @param bufferCfg A buffer contains a content of .cfg file with text description of the network architecture.
* @param bufferModel A buffer contains a content of .weights file with learned network.
* @returns Net object.
*/
CV_EXPORTS_W Net readNetFromDarknet(const std::vector<uchar>& bufferCfg,
const std::vector<uchar>& bufferModel = std::vector<uchar>());
/** @brief Reads a network model stored in <a href="https://pjreddie.com/darknet/">Darknet</a> model files.
* @param bufferCfg A buffer contains a content of .cfg file with text description of the network architecture.
* @param lenCfg Number of bytes to read from bufferCfg
* @param bufferModel A buffer contains a content of .weights file with learned network.
* @param lenModel Number of bytes to read from bufferModel
* @returns Net object.
*/
CV_EXPORTS Net readNetFromDarknet(const char *bufferCfg, size_t lenCfg,
const char *bufferModel = NULL, size_t lenModel = 0);
/** @brief Reads a network model stored in <a href="http://caffe.berkeleyvision.org">Caffe</a> framework's format.
* @param prototxt path to the .prototxt file with text description of the network architecture.
* @param caffeModel path to the .caffemodel file with learned network.
* @returns Net object.
*/
CV_EXPORTS_W Net readNetFromCaffe(const String &prototxt, const String &caffeModel = String());
/** @brief Reads a network model stored in Caffe model in memory.
* @param bufferProto buffer containing the content of the .prototxt file
* @param bufferModel buffer containing the content of the .caffemodel file
* @returns Net object.
*/
CV_EXPORTS_W Net readNetFromCaffe(const std::vector<uchar>& bufferProto,
const std::vector<uchar>& bufferModel = std::vector<uchar>());
/** @brief Reads a network model stored in Caffe model in memory.
* @details This is an overloaded member function, provided for convenience.
* It differs from the above function only in what argument(s) it accepts.
* @param bufferProto buffer containing the content of the .prototxt file
* @param lenProto length of bufferProto
* @param bufferModel buffer containing the content of the .caffemodel file
* @param lenModel length of bufferModel
* @returns Net object.
*/
CV_EXPORTS Net readNetFromCaffe(const char *bufferProto, size_t lenProto,
const char *bufferModel = NULL, size_t lenModel = 0);
/** @brief Reads a network model stored in <a href="https://www.tensorflow.org/">TensorFlow</a> framework's format.
* @param model path to the .pb file with binary protobuf description of the network architecture
* @param config path to the .pbtxt file that contains text graph definition in protobuf format.
* Resulting Net object is built by text graph using weights from a binary one that
* let us make it more flexible.
* @returns Net object.
*/
CV_EXPORTS_W Net readNetFromTensorflow(const String &model, const String &config = String());
/** @brief Reads a network model stored in <a href="https://www.tensorflow.org/">TensorFlow</a> framework's format.
* @param bufferModel buffer containing the content of the pb file
* @param bufferConfig buffer containing the content of the pbtxt file
* @returns Net object.
*/
CV_EXPORTS_W Net readNetFromTensorflow(const std::vector<uchar>& bufferModel,
const std::vector<uchar>& bufferConfig = std::vector<uchar>());
/** @brief Reads a network model stored in <a href="https://www.tensorflow.org/">TensorFlow</a> framework's format.
* @details This is an overloaded member function, provided for convenience.
* It differs from the above function only in what argument(s) it accepts.
* @param bufferModel buffer containing the content of the pb file
* @param lenModel length of bufferModel
* @param bufferConfig buffer containing the content of the pbtxt file
* @param lenConfig length of bufferConfig
*/
CV_EXPORTS Net readNetFromTensorflow(const char *bufferModel, size_t lenModel,
const char *bufferConfig = NULL, size_t lenConfig = 0);
/**
* @brief Reads a network model stored in <a href="http://torch.ch">Torch7</a> framework's format.
* @param model path to the file, dumped from Torch by using torch.save() function.
* @param isBinary specifies whether the network was serialized in ascii mode or binary.
* @param evaluate specifies testing phase of network. If true, it's similar to evaluate() method in Torch.
* @returns Net object.
*
* @note Ascii mode of Torch serializer is more preferable, because binary mode extensively use `long` type of C language,
* which has various bit-length on different systems.
*
* The loading file must contain serialized <a href="https://github.com/torch/nn/blob/master/doc/module.md">nn.Module</a> object
* with importing network. Try to eliminate a custom objects from serialazing data to avoid importing errors.
*
* List of supported layers (i.e. object instances derived from Torch nn.Module class):
* - nn.Sequential
* - nn.Parallel
* - nn.Concat
* - nn.Linear
* - nn.SpatialConvolution
* - nn.SpatialMaxPooling, nn.SpatialAveragePooling
* - nn.ReLU, nn.TanH, nn.Sigmoid
* - nn.Reshape
* - nn.SoftMax, nn.LogSoftMax
*
* Also some equivalents of these classes from cunn, cudnn, and fbcunn may be successfully imported.
*/
CV_EXPORTS_W Net readNetFromTorch(const String &model, bool isBinary = true, bool evaluate = true);
/**
* @brief Read deep learning network represented in one of the supported formats.
* @param[in] model Binary file contains trained weights. The following file
* extensions are expected for models from different frameworks:
* * `*.caffemodel` (Caffe, http://caffe.berkeleyvision.org/)
* * `*.pb` (TensorFlow, https://www.tensorflow.org/)
* * `*.t7` | `*.net` (Torch, http://torch.ch/)
* * `*.weights` (Darknet, https://pjreddie.com/darknet/)
* * `*.bin` (DLDT, https://software.intel.com/openvino-toolkit)
* * `*.onnx` (ONNX, https://onnx.ai/)
* @param[in] config Text file contains network configuration. It could be a
* file with the following extensions:
* * `*.prototxt` (Caffe, http://caffe.berkeleyvision.org/)
* * `*.pbtxt` (TensorFlow, https://www.tensorflow.org/)
* * `*.cfg` (Darknet, https://pjreddie.com/darknet/)
* * `*.xml` (DLDT, https://software.intel.com/openvino-toolkit)
* @param[in] framework Explicit framework name tag to determine a format.
* @returns Net object.
*
* This function automatically detects an origin framework of trained model
* and calls an appropriate function such @ref readNetFromCaffe, @ref readNetFromTensorflow,
* @ref readNetFromTorch or @ref readNetFromDarknet. An order of @p model and @p config
* arguments does not matter.
*/
CV_EXPORTS_W Net readNet(const String& model, const String& config = "", const String& framework = "");
/**
* @brief Read deep learning network represented in one of the supported formats.
* @details This is an overloaded member function, provided for convenience.
* It differs from the above function only in what argument(s) it accepts.
* @param[in] framework Name of origin framework.
* @param[in] bufferModel A buffer with a content of binary file with weights
* @param[in] bufferConfig A buffer with a content of text file contains network configuration.
* @returns Net object.
*/
CV_EXPORTS_W Net readNet(const String& framework, const std::vector<uchar>& bufferModel,
const std::vector<uchar>& bufferConfig = std::vector<uchar>());
/** @brief Loads blob which was serialized as torch.Tensor object of Torch7 framework.
* @warning This function has the same limitations as readNetFromTorch().
*/
CV_EXPORTS_W Mat readTorchBlob(const String &filename, bool isBinary = true);
/** @brief Load a network from Intel's Model Optimizer intermediate representation.
* @param[in] xml XML configuration file with network's topology.
* @param[in] bin Binary file with trained weights.
* @returns Net object.
* Networks imported from Intel's Model Optimizer are launched in Intel's Inference Engine
* backend.
*/
CV_EXPORTS_W
Net readNetFromModelOptimizer(const String &xml, const String &bin);
/** @brief Load a network from Intel's Model Optimizer intermediate representation.
* @param[in] bufferModelConfig Buffer contains XML configuration with network's topology.
* @param[in] bufferWeights Buffer contains binary data with trained weights.
* @returns Net object.
* Networks imported from Intel's Model Optimizer are launched in Intel's Inference Engine
* backend.
*/
CV_EXPORTS_W