Add tag based serialization

CMakeLists.txt

@@ -88,6 +88,22 @@ IF(NOT HAVE_IF_INIT)
 	MESSAGE(FATAL_ERROR "Compiling shogun requires a compiler with if init-statements support!")
 ENDIF()
 
+CHECK_CXX_SOURCE_COMPILES("
+#include <string>
+#include <string_view>
+
+int main()
+{
+  char array[3] = {'B', 'a', 'r'};
+  std::string_view array_v(array, std::size(array));
+  return 0;
+}
+" HAVE_STD_STRING_VIEW)
+
+IF(NOT HAVE_STD_STRING_VIEW)
+	MESSAGE(FATAL_ERROR "Compiling shogun requires a compiler with string_view support!")
+ENDIF()
+
 ############# minimum library versions ###################
 SET(VIENNACL_VERSION_MINIMUM 1.5.0)
 

cmake/ShogunUtils.cmake

@@ -144,34 +144,47 @@ macro(ADD_LIBRARY_DEPENDENCY)
 	set(oneValueArgs LIBRARY CONFIG_FLAG VERSION SCOPE)
 	set(multiValueArgs TARGETS)
 	cmake_parse_arguments(ADD_LIBRARY_DEPENDENCY "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
-	STRING(TOUPPER ${ADD_LIBRARY_DEPENDENCY_LIBRARY} LIBRARY_PREFIX)
-	OPTION(ENABLE_${LIBRARY_PREFIX} "Use ${LIBRARY_PREFIX}" ON)
+	SET(LIBRARY_PREFIX ${ADD_LIBRARY_DEPENDENCY_LIBRARY})
+	STRING(TOUPPER ${ADD_LIBRARY_DEPENDENCY_LIBRARY} LIBRARY_PREFIX_UPPER)
+	OPTION(ENABLE_${LIBRARY_PREFIX_UPPER} "Use ${LIBRARY_PREFIX}" ON)
 	if (${ADD_LIBRARY_DEPENDENCY_REQUIRED})
 		find_package(${ADD_LIBRARY_DEPENDENCY_LIBRARY} REQUIRED ${ADD_LIBRARY_DEPENDENCY_VERSION})
 	else()
 		find_package(${ADD_LIBRARY_DEPENDENCY_LIBRARY} ${ADD_LIBRARY_DEPENDENCY_VERSION})
 	endif()
-	if (${LIBRARY_PREFIX}_FOUND AND ENABLE_${LIBRARY_PREFIX})
+	if ((${LIBRARY_PREFIX}_FOUND OR ${LIBRARY_PREFIX_UPPER}_FOUND) AND ENABLE_${LIBRARY_PREFIX_UPPER})
 		if (${LIBRARY_PREFIX}_INCLUDE_DIR)
 			set(LIBRARY_HEADER ${${LIBRARY_PREFIX}_INCLUDE_DIR})
+		elseif (${LIBRARY_PREFIX_UPPER}_INCLUDE_DIR)
+			set(LIBRARY_HEADER ${${LIBRARY_PREFIX_UPPER}_INCLUDE_DIR})
 		elseif (${LIBRARY_PREFIX}_INCLUDE_DIRS)
 			set(LIBRARY_HEADER ${${LIBRARY_PREFIX}_INCLUDE_DIRS})
+		elseif (${LIBRARY_PREFIX_UPPER}_INCLUDE_DIRS)
+			set(LIBRARY_HEADER ${${LIBRARY_PREFIX_UPPER}_INCLUDE_DIRS})
 		else ()
 			message(${${LIBRARY_PREFIX}_INCLUDE_DIR})
 			message(FATAL_ERROR "Found ${ADD_LIBRARY_DEPENDENCY_LIBRARY}, but not it's headers!")
 		endif()
 
 		set(${ADD_LIBRARY_DEPENDENCY_CONFIG_FLAG} ON CACHE BOOL "Use ${LIBRARY_PREFIX}" FORCE)
 
+		if (${LIBRARY_PREFIX}_LIBRARIES)
+			set(LIBRARY_LIBS ${${LIBRARY_PREFIX}_LIBRARIES})
+		elseif(${LIBRARY_PREFIX_UPPER}_LIBRARIES)
+			set(LIBRARY_LIBS ${${LIBRARY_PREFIX_UPPER}_LIBRARIES})
+		else()
+			message(FATAL_ERROR "Found ${ADD_LIBRARY_DEPENDENCY_LIBRARY}, but not it's libraries!")
+		endif()
+
 		ForEach (element ${ADD_LIBRARY_DEPENDENCY_TARGETS})
 			if (TARGET ${element})
 				get_target_property(TARGET_TYPE ${element} TYPE)
 				if (${TARGET_TYPE} STREQUAL INTERFACE_LIBRARY)
 					target_include_directories(${element} INTERFACE ${LIBRARY_HEADER})
-					target_link_libraries(${element} INTERFACE ${${LIBRARY_PREFIX}_LIBRARIES})
+					target_link_libraries(${element} INTERFACE ${LIBRARY_LIBS})
 				else()
 					if (NOT ${TARGET_TYPE} STREQUAL OBJECT_LIBRARY)
-						target_link_libraries(${element} ${ADD_LIBRARY_DEPENDENCY_SCOPE} ${${LIBRARY_PREFIX}_LIBRARIES})
+						target_link_libraries(${element} ${ADD_LIBRARY_DEPENDENCY_SCOPE} ${LIBRARY_LIBS})
 					endif()
 					target_include_directories(${element} ${ADD_LIBRARY_DEPENDENCY_SCOPE} ${LIBRARY_HEADER})
 				endif()

cmake/external/RapidJSON.cmake

@@ -0,0 +1,16 @@
+set(RAPIDJSON_PREFIX ${CMAKE_BINARY_DIR}/RapidJSON)
+set(RAPIDJSON_INCLUDE_DIR "${RAPIDJSON_PREFIX}/src/RapidJSON/include")
+include(ExternalProject)
+ExternalProject_Add(
+	RapidJSON
+	PREFIX ${RAPIDJSON_PREFIX}
+	DOWNLOAD_DIR ${THIRD_PARTY_DIR}/RapidJSON
+	URL https://github.com/Tencent/rapidjson/archive/v1.1.0.tar.gz
+	URL_MD5 badd12c511e081fec6c89c43a7027bce
+	CMAKE_ARGS -DRAPIDJSON_BUILD_DOC:BOOL=OFF
+		-DRAPIDJSON_BUILD_EXAMPLES:BOOL=OFF
+		-DRAPIDJSON_BUILD_TESTS:BOOL=OFF
+    INSTALL_COMMAND ""
+	)
+
+LIST(APPEND SHOGUN_DEPENDS RapidJSON)

cmake/external/bitsery.cmake

@@ -0,0 +1,14 @@
+set(BITSERY_PREFIX ${CMAKE_BINARY_DIR}/bitsery)
+set(BITSERY_INCLUDE_DIR "${BITSERY_PREFIX}/src/bitsery/include")
+
+include(ExternalProject)
+ExternalProject_Add(
+	bitsery
+	PREFIX ${BITSERY_PREFIX}
+	DOWNLOAD_DIR ${THIRD_PARTY_DIR}/bitsery
+	URL https://github.com/fraillt/bitsery/archive/v4.6.0.tar.gz
+	URL_MD5 accb462f98a59ed2bc4ffa7de374c24b
+	INSTALL_COMMAND ""
+)
+
+LIST(APPEND SHOGUN_DEPENDS bitsery)

configs/shogun-sdk/Dockerfile

@@ -8,14 +8,14 @@ RUN printf "deb http://ppa.launchpad.net/timsc/swig-3.0.12/ubuntu xenial main\nd
 
 RUN apt-get update -qq && apt-get upgrade -y && \
     apt-get install -qq --force-yes --no-install-recommends make gcc-7 g++-7 swig \
-    libc6-dev libbz2-dev libjson-c-dev ccache libarpack2-dev libatlas-base-dev \
+    libc6-dev libbz2-dev ccache libarpack2-dev libatlas-base-dev \
     libblas-dev libglpk-dev libhdf5-serial-dev zlib1g-dev liblapacke-dev \
-    libnlopt-dev liblpsolve55-dev libxml2-dev libsnappy-dev liblzo2-dev \
+    libnlopt-dev liblpsolve55-dev libsnappy-dev liblzo2-dev \
     liblzma-dev libeigen3-dev python3-dev python3-numpy python3-matplotlib python3-scipy \
     python3-jinja2 python3-setuptools git-core wget jblas mono-devel mono-dmcs cli-common-dev \
     lua5.1 liblua5.1-0-dev octave liboctave-dev r-base-core clang-6.0 \
     openjdk-8-jdk ruby ruby-dev python3-ply sphinx-doc python3-pip \
-    exuberant-ctags clang-format-3.8 libcolpack-dev lcov \
+    exuberant-ctags clang-format-3.8 libcolpack-dev rapidjson-dev lcov \
     protobuf-compiler libprotobuf-dev googletest
 RUN apt-get -t stretch-backports install -qq --force-yes --no-install-recommends cmake
 

examples/meta/generator/targets/cpp.json

@@ -1,5 +1,5 @@
 {
-    "Program": "#include <shogun/base/init.h>\n#include <shogun/base/some.h>\n${dependencies}\n\nusing namespace shogun;\n\nint main(int, char*[])\n{\ninit_shogun_with_defaults();\n\n$program\nexit_shogun();\nreturn 0;\n}\n",
+    "Program": "#include <shogun/base/init.h>\n#include <shogun/base/some.h>\n${dependencies}\n\nusing namespace shogun;\nusing namespace shogun::io;\n\nint main(int, char*[])\n{\ninit_shogun_with_defaults();\n\n$program\nexit_shogun();\nreturn 0;\n}\n",
     "Dependencies": {
         "IncludeAllClasses": true,
         "IncludeEnums": true,

examples/meta/generator/translate.py

@@ -261,32 +261,20 @@ def injectVarsStoring(self, statementList, programName, varsToStore):
             storing
         """
         storage = "sg_storage"
-        storageFile = "sg_storage_file"
+        storageFormat = "sg_serializer"
 
-        # TODO: handle directories
-        storageFilename = {
-            "Expr": {"StringLiteral": "{}.dat".format(programName)}
-        }
-        # 'w'
-        storageFilemode = {"Expr": {"CharLiteral": 'w'}}
         storageComment = {"Comment": " Serialize output for integration testing (automatically generated)"}
         storageInit = {"Init": [{"ObjectType": "DynamicObjectArray"},
                                 {"Identifier": storage},
                                 {"ArgumentList": []}]}
-        storageFileInit = {
-            "Init": [{"ObjectType": "SerializableAsciiFile"},
-                     {"Identifier": storageFile},
-                     {"ArgumentList": [storageFilename, storageFilemode]}]
-        }
 
         statementList.append({"Statement": "\n"})
         statementList.append(storageComment)
         statementList.append({"Statement": storageInit})
-        statementList.append({"Statement": storageFileInit})
 
         for vartypeAST, varname in varsToStore:
             # avoid storing itself
-            if varname in (storage, storageFile):
+            if varname in (storage, storageFormat):
                 continue
 
             varnameExpr = {"Expr": {"StringLiteral": varname}}
@@ -316,13 +304,30 @@ def injectVarsStoring(self, statementList, programName, varsToStore):
             expression = {"Expr": methodCall}
             statementList.append({"Statement": expression})
 
+
+        storageFormatInit = {
+            "Init": [{"ObjectType": "JsonSerializer"},
+                     {"Identifier": storageFormat},
+                     {"ArgumentList": []}]
+        }
+
+        # TODO: handle directories
+        storageFilename = {
+            "Expr": {"StringLiteral": "{}.dat".format(programName)}
+        }
+
+        # serialize(storageFilename, storage, serializer);
         storageSerialize = {
-            "Expr": {"MethodCall": [
-                {"Identifier": storage},
-                {"Identifier": "save_serializable"},
-                {"ArgumentList": [{"Expr": {"Identifier": storageFile}}]}
+            "Expr": {"GlobalCall": [
+                {"Identifier": "serialize"},
+                {"ArgumentList": [
+                    storageFilename,
+                    {"Expr": {"Identifier": storage}},
+                    {"Expr": {"Identifier": storageFormat}}
+                ]}
             ]}
         }
+        statementList.append({"Statement": storageFormatInit})
         statementList.append({"Statement": storageSerialize})
 
     def dependenciesString(self, allClasses, interfacedClasses, enums,