feat: add predict from video

jolibrain · Jan 26, 2022 · 02872eb · 02872eb
1 parent 07644b4
commit 02872eb
Show file tree

Hide file tree

Showing 34 changed files with 1,395 additions and 217 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -573,9 +573,9 @@ if (USE_TF)
 endif() # USE_TF
 
 # OpenCV
-set(OPENCV_MODULES core imgproc highgui imgcodecs)
+set(OPENCV_MODULES core imgproc highgui imgcodecs videoio)
 if (USE_CUDA_CV)
-  list(APPEND OPENCV_MODULES cudaimgproc cudaarithm cudawarping)
+  list(APPEND OPENCV_MODULES cudaimgproc cudaarithm cudawarping cudacodec)
 endif()
 
 if (USE_OPENCV_VERSION STREQUAL "")

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -22,7 +22,7 @@ set(ddetect_SOURCES deepdetect.h deepdetect.cc mllibstrategy.h mlmodel.h
     csvinputfileconn.cc csvtsinputfileconn.h csvtsinputfileconn.cc
     svminputfileconn.h svminputfileconn.cc txtinputfileconn.h
     txtinputfileconn.cc apidata.h apidata.cc chain_actions.h chain_actions.cc
-    service_stats.h service_stats.cc chain.h chain.cc ext/rmustache/mustache.h ext/rmustache/mustache.cc
+    service_stats.h service_stats.cc chain.h chain.cc resources.cc ext/rmustache/mustache.h ext/rmustache/mustache.cc
     utils/oatpp.cc dto/ddtypes.cc utils/db.cpp utils/db_lmdb.cpp ${CMAKE_BINARY_DIR}/src/caffe.pb.cc)
 
 if (USE_JSON_API)

diff --git a/src/allconnectors.hpp b/src/allconnectors.hpp
@@ -0,0 +1,36 @@
+
+#ifdef USE_CAFFE
+#include "caffeinputconns.h"
+#endif
+
+#ifdef USE_TF
+#include "backends/tf/tfinputconns.h"
+#endif
+
+#ifdef USE_DLIB
+#include "backends/dlib/dlibinputconns.h"
+#endif
+
+#ifdef USE_NCNN
+#include "backends/ncnn/ncnninputconns.h"
+#endif
+
+#ifdef USE_CAFFE2
+#include "backends/caffe2/caffe2inputconns.h"
+#endif
+
+#ifdef USE_TENSORRT
+#include "backends/tensorrt/tensorrtinputconns.h"
+#endif
+
+#ifdef USE_TORCH
+#include "backends/torch/torchinputconns.h"
+#endif
+
+#ifdef USE_XGBOOST
+#include "backends/xgb/xgbinputconns.h"
+#endif
+
+#ifdef USE_TSNE
+#include "backends/tsne/tsneinputconns.h"
+#endif
diff --git a/src/backends/caffe/caffelib.cc b/src/backends/caffe/caffelib.cc
@@ -20,7 +20,7 @@
  */
 
 #include "caffelib.h"
-#include "imginputfileconn.h"
+#include "allconnectors.hpp"
 #include "outputconnectorstrategy.h"
 #include "generators/net_caffe.h"
 #include "generators/net_caffe_convnet.h"

diff --git a/src/backends/torch/native/native_factory.cc b/src/backends/torch/native/native_factory.cc
@@ -100,6 +100,16 @@ namespace dd
     return nullptr;
   }
 
+  template <>
+  NativeModule *NativeFactory::from_template<VideoTorchInputFileConn>(
+      const std::string tdef, const APIData &template_params,
+      const VideoTorchInputFileConn &inputc,
+      const std::shared_ptr<spdlog::logger> &logger)
+  {
+    return from_template<ImgTorchInputFileConn>(tdef, template_params, inputc,
+                                                logger);
+  }
+
   template NativeModule *
   NativeFactory::from_template(const std::string tdef,
                                const APIData &template_params,

diff --git a/src/backends/torch/torchdataset.cc b/src/backends/torch/torchdataset.cc
@@ -402,7 +402,7 @@ namespace dd
         if (!_lfiles.empty()) // prefetch batch from file list
           {
             ImgTorchInputFileConn *inputc
-                = reinterpret_cast<ImgTorchInputFileConn *>(_inputc);
+                = dynamic_cast<ImgTorchInputFileConn *>(_inputc);
             bool first_iter = true;
 
             for (int64_t id : ids)
@@ -526,7 +526,7 @@ namespace dd
             else
               {
                 ImgTorchInputFileConn *inputc
-                    = reinterpret_cast<ImgTorchInputFileConn *>(_inputc);
+                    = dynamic_cast<ImgTorchInputFileConn *>(_inputc);
 
                 cv::Mat bgr, bw_target;
                 read_image_from_db(datas, targets, bgr, t, bw_target,
@@ -652,7 +652,7 @@ namespace dd
                                     const bool &target)
   {
     ImgTorchInputFileConn *inputc
-        = reinterpret_cast<ImgTorchInputFileConn *>(_inputc);
+        = dynamic_cast<ImgTorchInputFileConn *>(_inputc);
 
     DDImg dimg;
     inputc->copy_parameters_to(dimg);
@@ -730,7 +730,7 @@ namespace dd
   {
     // read image before reading bboxes to get the size of the image
     ImgTorchInputFileConn *inputc
-        = reinterpret_cast<ImgTorchInputFileConn *>(_inputc);
+        = dynamic_cast<ImgTorchInputFileConn *>(_inputc);
 
     DDImg dimg;
     inputc->copy_parameters_to(dimg);
@@ -798,7 +798,7 @@ namespace dd
                                            const bool &target)
   {
     ImgTorchInputFileConn *inputc
-        = reinterpret_cast<ImgTorchInputFileConn *>(_inputc);
+        = dynamic_cast<ImgTorchInputFileConn *>(_inputc);
 
     std::vector<int64_t> sizes{ height, width, bgr.channels() };
     at::TensorOptions options(at::ScalarType::Byte);

diff --git a/src/backends/torch/torchinputconns.h b/src/backends/torch/torchinputconns.h
@@ -33,6 +33,7 @@
 #include "imginputfileconn.h"
 #include "txtinputfileconn.h"
 #include "csvtsinputfileconn.h"
+#include "videoinputfileconn.h"
 #include "torchdataset.h"
 #include "torchutils.h"
 
@@ -207,7 +208,7 @@ namespace dd
   /**
    * \brief image connector to torch backend
    */
-  class ImgTorchInputFileConn : public ImgInputFileConn,
+  class ImgTorchInputFileConn : virtual public ImgInputFileConn,
                                 public TorchInputInterface
   {
   public:
@@ -336,6 +337,28 @@ namespace dd
     bool _supports_bw = true;
   };
 
+  class VideoTorchInputFileConn : public ImgTorchInputFileConn,
+                                  public VideoInputFileConn
+  {
+  public:
+    VideoTorchInputFileConn()
+        : ImgInputFileConn(), ImgTorchInputFileConn(), VideoInputFileConn()
+    {
+    }
+
+    /**
+     * \brief copy constructor
+     */
+    VideoTorchInputFileConn(const VideoTorchInputFileConn &i)
+        : ImgInputFileConn(i), ImgTorchInputFileConn(i), VideoInputFileConn(i)
+    {
+    }
+
+    ~VideoTorchInputFileConn()
+    {
+    }
+  };
+
   /**
    * \brief txt input connector
    */

diff --git a/src/backends/torch/torchlib.cc b/src/backends/torch/torchlib.cc
@@ -1812,9 +1812,12 @@ namespace dd
 
     if (ad.has("chain") && ad.get("chain").get<bool>())
       {
-        if (typeid(inputc) == typeid(ImgTorchInputFileConn))
+        if (std::is_same<TInputConnectorStrategy,
+                         ImgTorchInputFileConn>::value)
           {
-            auto *img_ic = reinterpret_cast<ImgTorchInputFileConn *>(&inputc);
+            // can't do reinterpret_cast because virtual inheritance
+            InputConnectorStrategy *inputc_base = &inputc;
+            auto *img_ic = dynamic_cast<ImgTorchInputFileConn *>(inputc_base);
             APIData chain_input;
             if (!img_ic->_orig_images.empty())
               chain_input.add("imgs", img_ic->_orig_images);
@@ -2218,6 +2221,8 @@ namespace dd
   }
 
   template class TorchLib<ImgTorchInputFileConn, SupervisedOutput, TorchModel>;
+  template class TorchLib<VideoTorchInputFileConn, SupervisedOutput,
+                          TorchModel>;
   template class TorchLib<TxtTorchInputFileConn, SupervisedOutput, TorchModel>;
   template class TorchLib<CSVTSTorchInputFileConn, SupervisedOutput,
                           TorchModel>;

diff --git a/src/backends/torch/torchmodule.cc b/src/backends/torch/torchmodule.cc
@@ -585,6 +585,21 @@ namespace dd
       const ImgTorchInputFileConn &inputc, const TorchModel &tmodel,
       const torch::Device &device, const APIData &ad);
 
+  template void TorchModule::post_transform(
+      const std::string tmpl, const APIData &template_params,
+      const VideoTorchInputFileConn &inputc, const TorchModel &tmodel,
+      const torch::Device &device);
+
+  template void TorchModule::post_transform_train(
+      const std::string tmpl, const APIData &template_params,
+      const VideoTorchInputFileConn &inputc, const TorchModel &tmodel,
+      const torch::Device &device);
+
+  template void TorchModule::post_transform_predict(
+      const std::string tmpl, const APIData &template_params,
+      const VideoTorchInputFileConn &inputc, const TorchModel &tmodel,
+      const torch::Device &device, const APIData &ad);
+
   template void TorchModule::post_transform(
       const std::string tmpl, const APIData &template_params,
       const TxtTorchInputFileConn &inputc, const TorchModel &tmodel,

diff --git a/src/backends/tsne/tsnelib.cc b/src/backends/tsne/tsnelib.cc
@@ -20,7 +20,7 @@
  */
 
 #include "tsnelib.h"
-#include "csvinputfileconn.h"
+#include "allconnectors.hpp"
 #include "outputconnectorstrategy.h"
 #include <thread>
 #include "utils/utils.hpp"

diff --git a/src/backends/xgb/xgblib.cc b/src/backends/xgb/xgblib.cc
@@ -20,7 +20,7 @@
  */
 
 #include "xgblib.h"
-#include "csvinputfileconn.h"
+#include "allconnectors.hpp"
 #include "outputconnectorstrategy.h"
 #include <iomanip>
 #include <iostream>

diff --git a/src/csvinputfileconn.h b/src/csvinputfileconn.h
@@ -964,12 +964,4 @@ namespace dd
   };
 }
 
-#ifdef USE_XGBOOST
-#include "backends/xgb/xgbinputconns.h"
-#endif
-
-#ifdef USE_TSNE
-#include "backends/tsne/tsneinputconns.h"
-#endif
-
 #endif
diff --git a/src/dto/chain.hpp b/src/dto/chain.hpp
@@ -219,12 +219,11 @@ namespace dd
       DTO_FIELD(Float64, time);
     };
 
-    class ChainResponse : public oatpp::DTO
+    class ChainResponse : public GenericResponse
     {
-      DTO_INIT(ChainResponse, DTO)
+      DTO_INIT(ChainResponse, GenericResponse)
 
       DTO_FIELD(String, dd_msg);
-      DTO_FIELD(Object<Status>, status);
       DTO_FIELD(Object<ChainHead>, head);
       DTO_FIELD(Object<ChainBody>, body);
     };