5.0b1 merge fixes (#625)

* Fix bad merges from the other 5.0b1 feature PRs
* Bump version to 5.0b1
* Bump dependency versions to the tested versions
* Add support for Vision Feature Print in the image deep feature
  extractor
* Includes table printer fixes
* Includes some performance fixes
* Includes some Python 3 compat fixes

CMakeLists.txt

@@ -43,6 +43,7 @@ if(${TC_BUILD_PYTHON})
 
   include(UseCython)
   add_definitions(-DTC_HAS_PYTHON)
+  add_definitions(-DTC_BUILD_VISUALIZATION_CLIENT)
 else()
   message("Skipping python libraries.")
 endif()
@@ -183,7 +184,7 @@ message(WARNING "OpenMP Libraries were not found")
 #**************************************************************************/
 if (CLANG)
 set(CPP11_FLAGS "-std=c++11 -stdlib=libc++ -Wno-deprecated-register -Wno-enum-compare -Wno-conversion-null -Wno-constant-logical-operand -Wno-parentheses-equality -ftemplate-depth=900" CACHE STRING "C++11 enabling flags")
-set(WERROR_FLAGS "-Werror -Wno-error=tautological-undefined-compare -Wno-error=reorder -Wno-error=exceptions -Wno-error=switch -Wno-error=sometimes-uninitialized -Wno-error=unused-lambda-capture -Wno-error=missing-braces -Wno-error=absolute-value -Wno-error=potentially-evaluated-expression -Wno-error=null-arithmetic -Wno-error=format -Wno-error=pessimizing-move -Wno-error=comment -Wno-error=main -Wno-error=constant-conversion -Wno-error=deprecated-declarations -Wno-error=return-type -Wno-error=inconsistent-missing-override -Wno-error=overloaded-virtual -Wno-error=unused-private-field -Wno-error=unused-variable -Wno-error=unused-local-typedef")
+set(WERROR_FLAGS "-Werror -Wno-error=tautological-undefined-compare -Wno-error=reorder -Wno-error=exceptions -Wno-error=switch -Wno-error=sometimes-uninitialized -Wno-error=unused-lambda-capture -Wno-error=missing-braces -Wno-error=absolute-value -Wno-error=potentially-evaluated-expression -Wno-error=null-arithmetic -Wno-error=format -Wno-error=pessimizing-move -Wno-error=comment -Wno-error=main -Wno-error=constant-conversion -Wno-error=deprecated-declarations -Wno-error=return-type -Wno-error=inconsistent-missing-override -Wno-error=overloaded-virtual -Wno-error=unused-private-field -Wno-error=unused-variable -Wno-error=unused-local-typedef -Wno-error=unguarded-availability-new")
 else()
 set(CPP11_FLAGS "-std=c++11 -Wno-enum-compare -Wno-conversion-null -ftemplate-depth=900" CACHE STRING "C++11 enabling flags")
 set(WERROR_FLAGS "")
@@ -411,21 +412,6 @@ include(copy_file)
 include(CMakeParseArguments)
 include(eval)
 
-add_custom_target(external_dependencies)
-
-include(ExternalProject)
-file(GLOB packages "${DEPS_CMAKE}/ExternalProject*.cmake")
-foreach(package ${packages})
-        message(STATUS "We found local package: ${package}")
-        get_filename_component(packagename "${package}" NAME_WE)
-        #package is of the form ExternalProjectXXX"
-        include(${package})
-        STRING(SUBSTRING "${packagename}" 15 -1 depname)
-        message(STATUS "We found local package definition: ${depname}")
-        string(TOLOWER ${depname} depname)
-        set(package_${depname} requires_${depname} CACHE STRING "Package map")
-        add_dependencies(external_dependencies ex_${depname})
-endforeach()
 
 # This is an internal function and should not be used
 # Usage:
@@ -738,7 +724,6 @@ function (make_boost_test NAME)
 endfunction()
 
 
-# Core ML is only present on macOS 10.13 or higher
 if(APPLE)
   EXEC_PROGRAM(xcrun ARGS --show-sdk-version OUTPUT_VARIABLE mac_version RETURN_VALUE _xcrun_ret)
 
@@ -878,6 +863,21 @@ endmacro()
 include(SharedLibraryFromStatic)
 include(MergeStaticLibraries)
 
+add_custom_target(external_dependencies)
+
+include(ExternalProject)
+file(GLOB packages "${DEPS_CMAKE}/ExternalProject*.cmake")
+foreach(package ${packages})
+        message(STATUS "We found local package: ${package}")
+        get_filename_component(packagename "${package}" NAME_WE)
+        #package is of the form ExternalProjectXXX"
+        include(${package})
+        STRING(SUBSTRING "${packagename}" 15 -1 depname)
+        message(STATUS "We found local package definition: ${depname}")
+        string(TOLOWER ${depname} depname)
+        set(package_${depname} requires_${depname} CACHE STRING "Package map")
+        add_dependencies(external_dependencies ex_${depname})
+endforeach()
 
 include_directories(src)
 include_directories(SYSTEM src/external)

build_capi.sh

@@ -141,15 +141,14 @@ function build_capi {
     echo "Stripping local and debug symbols."
     strip -S -x ${install_dir}/lib*.* || echo "Non-fatal error stripping symbols."
   fi
-
 }
 
 function build_capi_framework {
   echo
   echo "Building C-API as macOS/iOS Framework"
   echo
 
-  run_configure --with-capi-framework ${ios_flag} --no-python --no-visualization -D TC_CAPI_FRAMEWORK_PATH=\"${framework_path}\" || exit 1
+  run_configure --with-capi-framework ${ios_flag} --no-python --no-visualization -D TC_CAPI_FRAMEWORK_PATH=\"${framework_path}\" --release-opt-for-size || exit 1
   mkdir -p ${target_dir}
   cd ${build_dir}/src/capi || exit 1
   make -j ${jobs} || exit 1

build_python_wheel.sh

@@ -110,14 +110,5 @@ rm -rf ${target_dir}/python
 mkdir -p ${target_dir}/python
 
 bash scripts/make_wheel.sh --skip_test --skip_cpp_test --build_number="$build_number" --num_procs=${jobs} --${build_mode} --target-dir="${install_dir}"
-pushd ${build_mode}/src
-
-if [[ $apple -eq 1 ]]; then
-  find . -type f -name '*.dylib' -o -name '*.so' | xargs strip -x -
-else
-  find . -type f -name '*.so' | xargs strip -s
-fi
-
-find . -type f -name '*.dylib' -o -name '*.so' | xargs tar cvzf ${install_dir}/shared_objects.tar.gz
 
 

gitlab_scripts/use_ccache.sh

@@ -1,4 +1,2 @@
-export CCACHE_COMPILERCHECK=content
-export CCACHE_DIR=/var/ccache
 ccache -M 100.0G
 ccache -s

scenario-tests/additional_requirements.txt

@@ -1,6 +1,5 @@
 beautifulsoup4
 certifi==2015.04.28
-gensim==0.12.2
 nltk==3.2
 pyscreenshot==0.4
 python-swiftclient

scripts/make_wheel.sh

@@ -272,11 +272,13 @@ package_wheel() {
     temp=`echo $WHEEL_PATH | perl -ne 'print m/(^.*-).*$/'`
     temp=${temp/-cpdarwin-/-cp35m-}
 
-    platform_tag="macosx_10_13_intel.macosx_10_13_x86_64"
-    mac_version=`sw_vers -productVersion`
-    if [[ $mac_version =~ ^10\.12(.\d+)? ]]; then
-        platform_tag="macosx_10_12_intel.macosx_10_12_x86_64"
-    fi
+    platform_tag="macosx_10_12_intel.macosx_10_12_x86_64.macosx_10_13_intel.macosx_10_13_x86_64.macosx_10_14_intel.macosx_10_14_x86_64"
+    #  sdk_version=`xcrun --show-sdk-version`
+    #  if [[ $sdk_version =~ ^10\.13 ]]; then
+    #      platform_tag="macosx_10_13_intel.macosx_10_12_x86_64"
+    #  elif [[ $sdk_version =~ ^10\.12 ]]; then
+    #      platform_tag="macosx_10_12_intel.macosx_10_12_x86_64"
+    #  fi
 
     NEW_WHEEL_PATH=${temp}${platform_tag}".whl"
     mv ${WHEEL_PATH} ${NEW_WHEEL_PATH}

scripts/requirements.txt

@@ -1,4 +1,4 @@
-coremltools==0.8
+coremltools==2.0b1
 scipy==0.19.1
 numpy==1.11.1
 cython==0.24
@@ -16,4 +16,4 @@ scikit-learn==0.17.1
 six==1.10.0
 statsmodels==0.8.0
 wheel==0.29.0
-mxnet==0.11
+mxnet==1.1.0

src/fileio/fileio_constants.hpp

@@ -85,6 +85,12 @@ extern size_t FILEIO_WRITER_BUFFER_SIZE;
  */
 extern std::string S3_ENDPOINT;
 
+/**
+ * \ingroup fileio
+ * The number of GPUs.
+ */
+extern int64_t NUM_GPUS;
+
 /**
  * \ingroup fileio
  * Gets the alternative ssl certificate file and directory.

src/fileio/fs_utils.cpp

@@ -155,14 +155,10 @@ EXPORT file_status get_file_status(const std::string& path) {
       return file_status::MISSING;
     }
   } else if (boost::starts_with(path, "s3://")) {
-#ifdef TC_BUILD_CAPI_IOS
-    log_and_throw("s3:// URLs not supported.");
-#else
     std::pair<bool, bool> ret = is_directory(path);
     if (ret.first == false) return file_status::MISSING;
     else if (ret.second == false) return file_status::REGULAR_FILE;
     else if (ret.second == true) return file_status::DIRECTORY;
-#endif
   } else if (is_web_protocol(get_protocol(path))) {
     return file_status::REGULAR_FILE;
     // some other web protocol?
@@ -208,9 +204,6 @@ get_directory_listing(const std::string& path) {
       // failure for some reason. return with nothing
     }
   } else if (boost::starts_with(path, "s3://")) {
-#ifdef TC_BUILD_CAPI_IOS
-    log_and_throw("s3:// URLs not supported.");
-#else
     list_objects_response response = list_directory(path);
     for (auto dir: response.directories) {
       ret.push_back({dir, file_status::DIRECTORY});
@@ -253,9 +246,6 @@ EXPORT bool create_directory(const std::string& path) {
 #ifdef TC_ENABLE_REMOTEFS
   } else if(boost::starts_with(path, "hdfs://")) {
     // hdfs
-#ifdef TC_BUILD_CAPI_IOS
-    log_and_throw("hdfs:// URLs not supported.");
-#else
     std::string host, port, hdfspath;
     std::tie(host, port, hdfspath) = parse_hdfs_url(path);
     try {
@@ -342,9 +332,6 @@ bool delete_path_impl(const std::string& path,
       return false;
     }
   } else if (boost::starts_with(path, "s3://")) {
-#ifdef TC_BUILD_CAPI_IOS
-    log_and_throw("s3:// URLs not supported.");
-#else
     return delete_object(path).empty();
 #endif
   } else {
@@ -373,9 +360,6 @@ EXPORT bool delete_path_recursive(const std::string& path) {
 #ifdef TC_ENABLE_REMOTEFS
   } else if(boost::starts_with(path, "hdfs://")) {
     // hdfs
-#ifdef TC_BUILD_CAPI_IOS
-    log_and_throw("hdfs:// URLs not supported.");
-#else
     std::string host, port, hdfspath;
     std::tie(host, port, hdfspath) = parse_hdfs_url(path);
     try {
@@ -387,11 +371,7 @@ EXPORT bool delete_path_recursive(const std::string& path) {
       // failure for some reason. return with nothing
       return false;
     }
-#endif
   } else if(boost::starts_with(path, "s3://")) {
-#ifdef TC_BUILD_CAPI_IOS
-    log_and_throw("s3:// URLs not supported.");
-#else
     return delete_prefix(path).empty();
 #endif
   } else {
@@ -683,9 +663,6 @@ bool change_file_mode(const std::string path, short mode) {
   }
 
   if(boost::starts_with(path, "hdfs://")) {
-#ifdef TC_BUILD_CAPI_IOS
-    log_and_throw("hdfs:// URLs not supported.");
-#else
 #ifdef HAS_HADOOP
     // hdfs
     std::string host, port, hdfspath;
@@ -701,7 +678,6 @@ bool change_file_mode(const std::string path, short mode) {
     }
 #else
       return false;
-#endif
 #endif
   } else if (boost::starts_with(path, fileio::get_cache_prefix())) {
     // this is a cache file. There is no filesystem.

src/fileio/union_fstream.cpp

@@ -52,9 +52,6 @@ union_fstream::union_fstream(std::string url,
 #ifdef TC_ENABLE_REMOTEFS
   } else if(boost::starts_with(url, "hdfs://")) {
     // HDFS file type
-#ifdef TC_BUILD_CAPI_IOS
-    log_and_throw("hdfs:// URLs not supported.");
-#else
     type = HDFS;
     std::string host, port, path;
     std::tie(host, port, path) = fileio::parse_hdfs_url(url);
@@ -78,9 +75,6 @@ union_fstream::union_fstream(std::string url,
   } else if (boost::starts_with(url, "s3://")) {
     // the S3 file type currently works by download/uploading a local file
     // i.e. the s3_stream simply remaps a local file stream
-#ifdef TC_BUILD_CAPI_IOS
-    log_and_throw_io_failure("Not implemented: compiled without support for s3:// URLs.");
-#else
     type = STD;
     if (is_output_stream) {
       output_stream = std::make_shared<s3_fstream>(url, true);

src/serialization/oarchive.hpp

@@ -128,7 +128,7 @@ namespace turi {
     inline void direct_assign(const T& t) {
       if (out == NULL) {
         expand_buf(sizeof(T));
-        (*reinterpret_cast<T*>(buf + off)) = t;
+        std::memcpy(buf + off, &t, sizeof(T));
         off += sizeof(T);
       }
       else {

src/serialization/vector.hpp

@@ -56,7 +56,7 @@ namespace turi {
     struct vector_serialize_impl<OutArcType, ValueType, true > {
       static void exec(OutArcType& oarc, const std::vector<ValueType>& vec) {
         oarc << size_t(vec.size());
-        serialize(oarc, &(vec[0]),sizeof(ValueType)*vec.size());
+        serialize(oarc, vec.data(),sizeof(ValueType)*vec.size());
       }
     };
 
@@ -80,7 +80,7 @@ namespace turi {
         size_t len;
         iarc >> len;
         vec.clear(); vec.resize(len);
-        deserialize(iarc, &(vec[0]), sizeof(ValueType)*vec.size());
+        deserialize(iarc, vec.data(), sizeof(ValueType)*vec.size());
       }
     };
 

src/toolkits/image_deep_feature_extractor/image_deep_feature_extractor_toolkit.cpp

@@ -25,8 +25,8 @@ void image_deep_feature_extractor_toolkit::init_options(const std::map<std::stri
 
 gl_sarray image_deep_feature_extractor_toolkit::extract_features(gl_sframe data,
                                                                  const std::string& column_name,
-                                                                 bool verbose) const {
-  return m_feature_extractor->extract_features(data[column_name]);
+                                                                 bool verbose, size_t batch_size) const {
+  return m_feature_extractor->extract_features(data[column_name], verbose, batch_size);
 }
 
 }  // image_deep_feature_extractor

src/toolkits/image_deep_feature_extractor/image_deep_feature_extractor_toolkit.hpp

@@ -25,7 +25,7 @@ class EXPORT image_deep_feature_extractor_toolkit : public ml_model_base {
 
   void init_options(const std::map<std::string, flexible_type>& options);
 
-  gl_sarray extract_features(gl_sframe data, const std::string& column_name, bool verbose) const;
+  gl_sarray extract_features(gl_sframe data, const std::string& column_name, bool verbose, size_t batch_size) const;
 
   inline size_t get_version() const { return -1; }
 
@@ -41,7 +41,7 @@ class EXPORT image_deep_feature_extractor_toolkit : public ml_model_base {
 
   REGISTER_CLASS_MEMBER_FUNCTION(image_deep_feature_extractor_toolkit::init_options, "options");
 
-  REGISTER_CLASS_MEMBER_FUNCTION(image_deep_feature_extractor_toolkit::extract_features, "data",  "column_name", "verbose");
+  REGISTER_CLASS_MEMBER_FUNCTION(image_deep_feature_extractor_toolkit::extract_features, "data",  "column_name", "verbose", "batch_size");
 
   END_CLASS_MEMBER_REGISTRATION
 

src/toolkits/image_deep_feature_extractor/image_feature_extractor.hpp

@@ -30,7 +30,7 @@ class image_feature_extractor {
   // free to perform this computation in a more optimized fashion. The input
   // SArray may also contain flex_string values, in which case each string is
   // interpreted as a URL from which the image can be loaded.
-  virtual gl_sarray extract_features(gl_sarray images) const = 0;
+  virtual gl_sarray extract_features(gl_sarray images, bool verbose, size_t batch_size) const = 0;
 };
 
 }  // image_deep_feature_extractor

src/toolkits/image_deep_feature_extractor/mlmodel_image_feature_extractor.hpp

@@ -27,7 +27,7 @@ class mlmodel_image_feature_extractor: public image_feature_extractor {
 
   // image_feature_extractor interface
   const CoreML::Specification::Model& coreml_spec() const override;
-  gl_sarray extract_features(gl_sarray images) const override;
+  gl_sarray extract_features(gl_sarray images, bool verbose, size_t batch_size) const override;
 
 private:
   // Use PIMPL pattern to hide Objective C from this C++ header.