automated model installation from compressed archive at service creation

jolibrain · Nov 16, 2018 · 3755a12 · 3755a12
1 parent f0e3155
commit 3755a12
Show file tree

Hide file tree

Showing 17 changed files with 198 additions and 28 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -519,7 +519,7 @@ set(COMMON_LINK_DIRS
   ${DLIB_LIB_DIR})
 if (USE_HDF5)
   set(COMMON_LINK_LIBS
-    ddetect ${CUDA_LIB_DEPS} glog gflags ${OpenCV_LIBS} curlpp curl hdf5_cpp ${Boost_LIBRARIES}
+    ddetect ${CUDA_LIB_DEPS} glog gflags ${OpenCV_LIBS} curlpp curl hdf5_cpp ${Boost_LIBRARIES} archive
     ${CAFFE_LIB_DEPS}
     ${CAFFE2_LIB_DEPS}
     ${TF_LIB_DEPS}

diff --git a/README.md b/README.md
@@ -242,7 +242,7 @@ Below are instructions for Ubuntu 14.04 LTS and 16.04 LTS. For other Linux and U
 
 Beware of dependencies, typically on Debian/Ubuntu Linux, do:
 ```
-sudo apt-get install build-essential libgoogle-glog-dev libgflags-dev libeigen3-dev libopencv-dev libcppnetlib-dev libboost-dev libboost-iostreams-dev libcurlpp-dev libcurl4-openssl-dev protobuf-compiler libopenblas-dev libhdf5-dev libprotobuf-dev libleveldb-dev libsnappy-dev liblmdb-dev libutfcpp-dev cmake libgoogle-perftools-dev unzip python-setuptools python-dev libspdlog-dev python-six python-enum34
+sudo apt-get install build-essential libgoogle-glog-dev libgflags-dev libeigen3-dev libopencv-dev libcppnetlib-dev libboost-dev libboost-iostreams-dev libcurlpp-dev libcurl4-openssl-dev protobuf-compiler libopenblas-dev libhdf5-dev libprotobuf-dev libleveldb-dev libsnappy-dev liblmdb-dev libutfcpp-dev cmake libgoogle-perftools-dev unzip python-setuptools python-dev libspdlog-dev python-six python-enum34 libarchive-dev
 ```
 
 #### Default build with Caffe

diff --git a/src/backends/caffe/caffemodel.cc b/src/backends/caffe/caffemodel.cc
@@ -29,8 +29,34 @@
 
 namespace dd
 {
+  CaffeModel::CaffeModel(const APIData &ad, APIData &adg,
+			 const std::shared_ptr<spdlog::logger> &logger)
+    :MLModel(ad,adg,logger)
+  {
+    if (ad.has("templates"))
+      this->_mlmodel_template_repo = ad.get("templates").get<std::string>();
+    else this->_mlmodel_template_repo += "caffe/"; // default
+
+    if (ad.has("def"))
+      _def = ad.get("def").get<std::string>();
+    if (ad.has("trainf"))
+      _trainf = ad.get("trainf").get<std::string>();
+    if (ad.has("weights"))
+      _weights = ad.get("weights").get<std::string>();
+    if (ad.has("corresp"))
+      _corresp = ad.get("corresp").get<std::string>();
+    if (ad.has("solver"))
+      _solver = ad.get("solver").get<std::string>();
+    if (ad.has("repository"))
+      {
+       	if (read_from_repository(ad.get("repository").get<std::string>(),spdlog::get("api")))
+	  throw MLLibBadParamException("error reading or listing Caffe models in repository " + _repo);
+      }
+    read_corresp_file();
+  }
+
   CaffeModel::CaffeModel(const APIData &ad)
-    :MLModel(ad)
+      :MLModel(ad)
   {
     if (ad.has("templates"))
       this->_mlmodel_template_repo = ad.get("templates").get<std::string>();

diff --git a/src/backends/caffe/caffemodel.h b/src/backends/caffe/caffemodel.h
@@ -34,6 +34,8 @@ namespace dd
   public:
   CaffeModel(): MLModel() {}
     CaffeModel(const APIData &ad);
+    CaffeModel(const APIData &ad, APIData &adg,
+	       const std::shared_ptr<spdlog::logger> &logger);
   CaffeModel(const APIData &ad, const std::string &repo)
     :MLModel(ad, repo) {}
     ~CaffeModel() {};

diff --git a/src/backends/caffe2/caffe2model.cc b/src/backends/caffe2/caffe2model.cc
@@ -25,8 +25,9 @@
 
 namespace dd {
 
-  Caffe2Model::Caffe2Model(const APIData &ad)
-    :MLModel()
+  Caffe2Model::Caffe2Model(const APIData &ad, APIData &adg,
+			   const std::shared_ptr<spdlog::logger> &logger)
+    :MLModel(ad, adg, logger)
   {
     std::map<std::string, std::string *> names =
       {

diff --git a/src/backends/caffe2/caffe2model.h b/src/backends/caffe2/caffe2model.h
@@ -33,7 +33,8 @@ namespace dd {
   class Caffe2Model : public MLModel {
   public:
     Caffe2Model():MLModel() {}
-    Caffe2Model(const APIData &ad);
+    Caffe2Model(const APIData &ad, APIData &adg,
+		const std::shared_ptr<spdlog::logger> &logger);
     Caffe2Model(const std::string &repo)
       :MLModel(repo) {}
     ~Caffe2Model() {};

diff --git a/src/backends/dlib/dlibmodel.cc b/src/backends/dlib/dlibmodel.cc
@@ -24,7 +24,10 @@
 #include <string>
 
 namespace dd {
-    DlibModel::DlibModel(const APIData &ad) {
+     DlibModel::DlibModel(const APIData &ad, APIData &adg,
+		       const std::shared_ptr<spdlog::logger> &logger)
+       :MLModel(ad,adg,logger)
+     {
         if (ad.has("repository")) {
             read_from_repository(ad.get("repository").get<std::string>(),
                                  spdlog::get("api")); // XXX: beware, error not caught

diff --git a/src/backends/dlib/dlibmodel.h b/src/backends/dlib/dlibmodel.h
@@ -32,7 +32,8 @@ namespace dd {
     public:
         DlibModel() : MLModel() {}
 
-        DlibModel(const APIData &ad);
+        DlibModel(const APIData &ad, APIData &adg,
+		  const std::shared_ptr<spdlog::logger> &logger);
 
         DlibModel(const std::string &repo)
                 : MLModel(repo) {}

diff --git a/src/backends/tf/tfmodel.cc b/src/backends/tf/tfmodel.cc
@@ -24,7 +24,9 @@
 #include <string>
 namespace dd
 {
-  TFModel::TFModel(const APIData &ad)
+  TFModel::TFModel(const APIData &ad, APIDat &adg,
+		   const std::shared_ptr<spdlog::logger> &logger)
+    :MLModel(ad,adg,logger)
   {
    if (ad.has("repository"))
       {

diff --git a/src/backends/tf/tfmodel.h b/src/backends/tf/tfmodel.h
@@ -33,7 +33,8 @@ namespace dd
   {
   public:
     TFModel():MLModel() {}
-    TFModel(const APIData &ad);
+    TFModel(const APIData &ad, APIData &adg,
+	    const std::shared_ptr<spdlog::logger> &logger);
     TFModel(const std::string &repo)
       :MLModel(repo) {}
     ~TFModel() {}

diff --git a/src/backends/tsne/tsnemodel.h b/src/backends/tsne/tsnemodel.h
@@ -33,8 +33,9 @@ namespace dd
   {
   public:
     TSNEModel():MLModel() {}
-    TSNEModel(const APIData &ad)
-      :MLModel()
+  TSNEModel(const APIData &ad,APIData &adg,
+	    const std::shared_ptr<spdlog::logger> &logger)
+    :MLModel(ad,adg,logger)
       {
 	if (ad.has("repository"))
 	  this->_repo = ad.get("repository").get<std::string>();

diff --git a/src/backends/xgb/xgbmodel.cc b/src/backends/xgb/xgbmodel.cc
@@ -26,8 +26,9 @@
 namespace dd
 {
 
-  XGBModel::XGBModel(const APIData &ad)
-    :MLModel(ad)
+  XGBModel::XGBModel(const APIData &ad, APIData &adg,
+		     const std::shared_ptr<spdlog::logger> &logger)
+    :MLModel(ad,adg,logger)
   {
     if (ad.has("repository"))
       this->_repo = ad.get("repository").get<std::string>();

diff --git a/src/backends/xgb/xgbmodel.h b/src/backends/xgb/xgbmodel.h
@@ -34,7 +34,8 @@ namespace dd
   {
   public:
     XGBModel():MLModel() {}
-    XGBModel(const APIData &ad);
+    XGBModel(const APIData &ad, APIData &adg,
+	     const std::shared_ptr<spdlog::logger> &logger);
     XGBModel(const std::string &repo)
       :MLModel(repo) {}
     ~XGBModel() {}

diff --git a/src/commandlineapi.cc b/src/commandlineapi.cc
@@ -58,6 +58,7 @@ namespace dd
 	  {
 	    APIData model_ad;
 	    model_ad.add("repository",FLAGS_model_repo);
+	    APIData adg;
 	    CaffeModel cmodel(model_ad);
 	    add_service(FLAGS_service,std::move(MLService<CaffeLib,ImgCaffeInputFileConn,SupervisedOutput,CaffeModel>(FLAGS_service,cmodel)));
 	  }
@@ -87,6 +88,7 @@ namespace dd
 	  {
 	    APIData model_ad;
 	    model_ad.add("repository",FLAGS_model_repo);
+	    APIData adg;
 	    CaffeModel cmodel(model_ad);
 	    add_service(FLAGS_service,std::move(MLService<CaffeLib,ImgCaffeInputFileConn,SupervisedOutput,CaffeModel>(FLAGS_service,cmodel)));
 	    APIData ad, out;

diff --git a/src/jsonapi.cc b/src/jsonapi.cc
@@ -409,7 +409,7 @@ namespace dd
       {
 	if (mllib == "caffe")
 	  {
-	    CaffeModel cmodel(ad_model);
+	    CaffeModel cmodel(ad_model,ad,_logger);
 	    if (type == "supervised")
 	      {
 		if (input == "image")
@@ -453,7 +453,7 @@ namespace dd
 #ifdef USE_CAFFE2
 
 	else if (mllib == "caffe2") {
-	  Caffe2Model c2model(ad_model);
+	  Caffe2Model c2model(ad_model,ad,_logger);
 	  if (type == "supervised") {
 
 	    if (input == "image")
@@ -489,7 +489,7 @@ namespace dd
 #ifdef USE_TF
 	else if (mllib == "tensorflow" || mllib == "tf")
 	  {
-	    TFModel tfmodel(ad_model);
+	    TFModel tfmodel(ad_model,ad,_logger);
 	    if (type == "supervised")
 	      {
 		if (input == "image")
@@ -518,7 +518,7 @@ namespace dd
 #endif
 #ifdef USE_DLIB
     else if (mllib == "dlib") {
-	    DlibModel dlibmodel(ad_model);
+            DlibModel dlibmodel(ad_model,ad,_logger);
 	    if (type == "supervised") {
 	        if (input == "image") {
 	            add_service(sname, std::move(MLService<DlibLib, ImgDlibInputFileConn, SupervisedOutput, DlibModel>(sname, dlibmodel, description)), ad);
@@ -539,7 +539,7 @@ namespace dd
 #ifdef USE_XGBOOST
 	else if (mllib == "xgboost")
 	  {
-	    XGBModel xmodel(ad_model);
+	    XGBModel xmodel(ad_model,ad,_logger);
 	    if (input == "csv")
 	      add_service(sname,std::move(MLService<XGBLib,CSVXGBInputFileConn,SupervisedOutput,XGBModel>(sname,xmodel,description)),ad);
 	    else if (input == "svm")
@@ -557,7 +557,7 @@ namespace dd
 #ifdef USE_TSNE
 	else if (mllib == "tsne")
 	  {
-	    TSNEModel tmodel(ad_model);
+	    TSNEModel tmodel(ad_model,ad,_logger);
 	    if (input == "csv")
 	      add_service(sname,std::move(MLService<TSNELib,CSVTSNEInputFileConn,UnsupervisedOutput,TSNEModel>(sname,tmodel,description)),ad);
 	    else if (input == "txt")

diff --git a/src/mlmodel.h b/src/mlmodel.h
@@ -40,15 +40,22 @@ namespace dd
   public:
     MLModel() {}
 
-    MLModel(const APIData &ad)  {
+    MLModel(const APIData &ad, APIData &adg,
+	    const std::shared_ptr<spdlog::logger> &logger)  {
       init_repo_dir(ad);
+      read_config_json(adg,logger);
     }
 
+    MLModel(const APIData &ad)
+      {
+	init_repo_dir(ad);
+      }
+
     MLModel(const std::string &repo)
       :_repo(repo) {}
 
 
-    MLModel(const APIData &ad, const std::string &repo)
+  MLModel(const APIData &ad, const std::string &repo)
     :_repo(repo)
     {
       init_repo_dir(ad);
@@ -146,17 +153,55 @@ namespace dd
   private:
     void init_repo_dir(const APIData &ad)
     {
-      std::string repo =  ad.get("repository").get<std::string>();
+      // auto-creation of model directory
+      _repo =  ad.get("repository").get<std::string>();
       bool create = ad.has("create_repository") && ad.get("create_repository").get<bool>();
       bool isDir;
-      bool exists = fileops::file_exists(repo, isDir);
+      bool exists = fileops::file_exists(_repo, isDir);
       if (exists && !isDir)
         throw MLLibBadParamException("file exists with same name as repository");
       if (!exists && create)
-        fileops::create_dir(repo,0775);
+        fileops::create_dir(_repo,0775);
 #ifdef USE_SIMSEARCH
       _index_preload = ad.has("index_preload") && ad.get("index_preload").get<bool>();
 #endif
+      // auto-install from model archive
+      if (ad.has("init"))
+	{
+	  std::string compressedf = ad.get("init").get<std::string>();
+	  if (fileops::uncompress(compressedf,_repo))
+	    throw MLLibBadParamException("failed installing model from archive, check 'init' argument to model");
+	}
+    }
+
+    void read_config_json(APIData &adg,
+			  const std::shared_ptr<spdlog::logger> &logger)
+    {
+      const std::string cf = _repo + "/config.json";
+      if (!fileops::file_exists(cf))
+	return;
+      std::ifstream is(cf);
+      std::stringstream jbuf;
+      jbuf << is.rdbuf();
+      rapidjson::Document d;
+      d.Parse(jbuf.str().c_str());
+      if (d.HasParseError())
+	{
+	  logger->error("config.json parsing error on string: {}",jbuf.str());
+	  throw MLLibBadParamException("Failed parsing config file " + cf);
+	}
+      APIData adcj;
+      try
+	{
+	  adcj = APIData(d);
+	}
+      catch(RapidjsonException &e)
+	{
+	  logger->error("JSON error {}",e.what());
+	  throw MLLibBadParamException("Failed converting JSON file to internal data format");
+	}
+      APIData adcj_parameters = adcj.getobj("parameters");
+      adg.add("parameters",adcj_parameters);
     }
   };
 }