From 2bcf37d531a3977fdc18e39201d34dc30ec9a285 Mon Sep 17 00:00:00 2001 From: Viktor Gal Date: Tue, 25 Oct 2016 14:41:20 -0700 Subject: [PATCH] remove all static interfaces and integration tests --- .gitignore | 3 - CMakeLists.txt | 55 +- debian | 2 +- src/interfaces/cmdline_static/CMakeLists.txt | 17 - .../cmdline_static/CmdLineInterface.cpp | 794 -------------- .../cmdline_static/CmdLineInterface.h | 184 ---- src/interfaces/cmdline_static/cmdline.cpp | 20 - src/interfaces/elwms_static/elwms.cpp | 15 - src/interfaces/matlab_static/CMakeLists.txt | 13 - .../matlab_static/MatlabInterface.cpp | 703 ------------ .../matlab_static/MatlabInterface.h | 188 ---- src/interfaces/octave_static/CMakeLists.txt | 12 - .../octave_static/OctaveInterface.cpp | 842 --------------- .../octave_static/OctaveInterface.h | 208 ---- src/interfaces/python_static/CMakeLists.txt | 12 - .../python_static/PythonInterface.cpp | 996 ------------------ .../python_static/PythonInterface.h | 237 ----- src/interfaces/r_static/CMakeLists.txt | 23 - src/interfaces/r_static/RInterface.cpp | 782 -------------- src/interfaces/r_static/RInterface.h | 180 ---- tests/integration/CMakeLists.txt | 22 - tests/integration/README | 165 --- tests/integration/blacklist | 12 - tests/integration/cmdline_static/test.sg | 35 - tests/integration/cmdline_static/test_all.sh | 36 - tests/integration/data | 1 - tests/integration/generate_testdata.py | 6 - tests/integration/generator/__init__.py | 66 -- tests/integration/generator/category.py | 54 - tests/integration/generator/classifier.py | 475 --------- tests/integration/generator/clustering.py | 64 -- tests/integration/generator/dataop.py | 195 ---- tests/integration/generator/distance.py | 114 -- tests/integration/generator/distribution.py | 120 --- tests/integration/generator/featop.py | 145 --- tests/integration/generator/fileop.py | 424 -------- tests/integration/generator/kernel.py | 607 ----------- tests/integration/generator/preproc.py | 107 -- tests/integration/generator/regression.py | 125 --- tests/integration/libshogun/test_all.sh | 36 - tests/integration/lua_modular/generator.lua | 57 - .../matlab_and_octave/classifier.m | 138 --- .../matlab_and_octave/clustering.m | 59 -- .../integration/matlab_and_octave/distance.m | 24 - .../matlab_and_octave/distribution.m | 28 - tests/integration/matlab_and_octave/kernel.m | 27 - .../matlab_and_octave/preprocessor.m | 34 - .../matlab_and_octave/regression.m | 60 -- .../integration/matlab_and_octave/test_all.sh | 65 -- .../integration/matlab_and_octave/test_one.sh | 13 - .../matlab_and_octave/util/check_accuracy.m | 27 - .../util/convert_features_and_add_preproc.m | 34 - .../util/fix_classifier_name_inconsistency.m | 15 - .../util/fix_clustering_name_inconsistency.m | 2 - .../util/fix_distance_name_inconsistency.m | 14 - .../util/fix_kernel_name_inconsistency.m | 32 - .../util/fix_normalizer_name_inconsistency.m | 10 - .../util/fix_preproc_name_inconsistency.m | 2 - .../util/fix_regression_name_inconsistency.m | 2 - .../matlab_and_octave/util/globals.m | 178 ---- .../matlab_and_octave/util/set_distance.m | 19 - .../matlab_and_octave/util/set_features.m | 115 -- .../matlab_and_octave/util/set_kernel.m | 208 ---- .../matlab_and_octave/util/tobool.m | 2 - tests/integration/matlab_static | 1 - tests/integration/octave_modular/classifier.m | 164 --- tests/integration/octave_modular/clustering.m | 46 - tests/integration/octave_modular/distance.m | 23 - .../integration/octave_modular/distribution.m | 52 - tests/integration/octave_modular/kernel.m | 42 - .../integration/octave_modular/preprocessor.m | 44 - tests/integration/octave_modular/regression.m | 70 -- tests/integration/octave_modular/test_all.sh | 51 - tests/integration/octave_modular/test_one.sh | 15 - .../octave_modular/util/check_accuracy.m | 27 - .../util/convert_features_and_add_preproc.m | 48 - .../integration/octave_modular/util/globals.m | 179 ---- .../octave_modular/util/set_distance.m | 81 -- .../octave_modular/util/set_features.m | 189 ---- .../octave_modular/util/set_kernel.m | 325 ------ .../integration/octave_modular/util/set_pie.m | 13 - .../octave_modular/util/set_pos_and_neg.m | 59 -- .../integration/octave_modular/util/tobool.m | 2 - tests/integration/octave_static | 1 - tests/integration/perl_modular/README | 13 - tests/integration/perl_modular/classifier.pm | 215 ---- tests/integration/perl_modular/clustering.pm | 56 - tests/integration/perl_modular/distance.pm | 43 - .../integration/perl_modular/distribution.pm | 81 -- tests/integration/perl_modular/generator.pm | 128 --- tests/integration/perl_modular/kernel.pm | 327 ------ .../integration/perl_modular/preprocessor.pm | 44 - tests/integration/perl_modular/regression.pm | 74 -- tests/integration/perl_modular/test_all.sh | 52 - tests/integration/perl_modular/test_one.pl | 171 --- .../perl_modular/test_svm_kernel_multiple.pl | 47 - tests/integration/perl_modular/tester.pl | 272 ----- tests/integration/perl_modular/util.pm | 254 ----- .../integration/python_modular/CMakeLists.txt | 41 - tests/integration/python_modular/README | 8 - .../integration/python_modular/classifier.py | 172 --- .../integration/python_modular/clustering.py | 50 - tests/integration/python_modular/distance.py | 34 - .../python_modular/distribution.py | 62 -- tests/integration/python_modular/generator.py | 114 -- tests/integration/python_modular/kernel.py | 227 ---- .../python_modular/preprocessor.py | 42 - .../integration/python_modular/regression.py | 74 -- tests/integration/python_modular/test_all.sh | 52 - tests/integration/python_modular/test_one.py | 119 --- .../test_svm_kernel_multiple.py | 46 - tests/integration/python_modular/tester.py | 216 ---- tests/integration/python_modular/tests | 1 - tests/integration/python_modular/util.py | 194 ---- tests/integration/python_static/classifier.py | 140 --- tests/integration/python_static/clustering.py | 72 -- tests/integration/python_static/distance.py | 35 - .../integration/python_static/distribution.py | 58 - tests/integration/python_static/kernel.py | 203 ---- .../integration/python_static/preprocessor.py | 46 - tests/integration/python_static/regression.py | 81 -- tests/integration/python_static/test_all.sh | 50 - tests/integration/python_static/test_one.py | 113 -- .../python_static/test_svm_kernel_multiple.py | 46 - tests/integration/python_static/util.py | 219 ---- tests/integration/r_modular/classifier.R | 156 --- tests/integration/r_modular/clustering.R | 44 - tests/integration/r_modular/distance.R | 25 - tests/integration/r_modular/distribution.R | 46 - tests/integration/r_modular/kernel.R | 24 - tests/integration/r_modular/preproc.R | 46 - tests/integration/r_modular/regression.R | 66 -- tests/integration/r_modular/test_all.sh | 51 - tests/integration/r_modular/test_one.R | 22 - tests/integration/r_modular/test_one.sh | 7 - .../r_modular/util/check_accuracy.R | 29 - .../util/convert_features_and_add_preproc.R | 35 - .../integration/r_modular/util/get_distance.R | 21 - .../integration/r_modular/util/get_features.R | 137 --- tests/integration/r_modular/util/get_kernel.R | 275 ----- tests/integration/r_modular/util/get_pie.R | 9 - tests/integration/r_modular/util/get_posneg.R | 45 - tests/integration/r_modular/util/read_mfile.R | 35 - tests/integration/r_modular/util/tobool.R | 9 - tests/integration/r_static/classifier.R | 115 -- tests/integration/r_static/clustering.R | 58 - tests/integration/r_static/distance.R | 22 - tests/integration/r_static/distribution.R | 23 - tests/integration/r_static/kernel.R | 24 - tests/integration/r_static/preprocessor.R | 34 - tests/integration/r_static/regression.R | 57 - tests/integration/r_static/test_all.sh | 52 - tests/integration/r_static/test_one.R | 22 - tests/integration/r_static/test_one.sh | 7 - .../r_static/util/check_accuracy.R | 26 - .../util/convert_features_and_add_preproc.R | 28 - .../util/fix_classifier_name_inconsistency.R | 12 - .../util/fix_clustering_name_inconsistency.R | 3 - .../util/fix_distance_name_inconsistency.R | 12 - .../util/fix_kernel_name_inconsistency.R | 38 - .../util/fix_normalizer_name_inconsistency.R | 13 - .../util/fix_preproc_name_inconsistency.R | 3 - .../util/fix_regression_name_inconsistency.R | 3 - tests/integration/r_static/util/read_mfile.R | 35 - .../integration/r_static/util/set_distance.R | 17 - .../integration/r_static/util/set_features.R | 96 -- tests/integration/r_static/util/set_kernel.R | 164 --- tests/integration/r_static/util/tobool.R | 9 - tests/integration/test_configure.py | 36 - 169 files changed, 12 insertions(+), 16963 deletions(-) delete mode 100644 src/interfaces/cmdline_static/CMakeLists.txt delete mode 100644 src/interfaces/cmdline_static/CmdLineInterface.cpp delete mode 100644 src/interfaces/cmdline_static/CmdLineInterface.h delete mode 100644 src/interfaces/cmdline_static/cmdline.cpp delete mode 100644 src/interfaces/elwms_static/elwms.cpp delete mode 100644 src/interfaces/matlab_static/CMakeLists.txt delete mode 100644 src/interfaces/matlab_static/MatlabInterface.cpp delete mode 100644 src/interfaces/matlab_static/MatlabInterface.h delete mode 100644 src/interfaces/octave_static/CMakeLists.txt delete mode 100644 src/interfaces/octave_static/OctaveInterface.cpp delete mode 100644 src/interfaces/octave_static/OctaveInterface.h delete mode 100644 src/interfaces/python_static/CMakeLists.txt delete mode 100644 src/interfaces/python_static/PythonInterface.cpp delete mode 100644 src/interfaces/python_static/PythonInterface.h delete mode 100644 src/interfaces/r_static/CMakeLists.txt delete mode 100644 src/interfaces/r_static/RInterface.cpp delete mode 100644 src/interfaces/r_static/RInterface.h delete mode 100644 tests/integration/CMakeLists.txt delete mode 100644 tests/integration/README delete mode 100644 tests/integration/blacklist delete mode 100644 tests/integration/cmdline_static/test.sg delete mode 100755 tests/integration/cmdline_static/test_all.sh delete mode 120000 tests/integration/data delete mode 100755 tests/integration/generate_testdata.py delete mode 100644 tests/integration/generator/__init__.py delete mode 100644 tests/integration/generator/category.py delete mode 100644 tests/integration/generator/classifier.py delete mode 100644 tests/integration/generator/clustering.py delete mode 100644 tests/integration/generator/dataop.py delete mode 100644 tests/integration/generator/distance.py delete mode 100644 tests/integration/generator/distribution.py delete mode 100644 tests/integration/generator/featop.py delete mode 100644 tests/integration/generator/fileop.py delete mode 100644 tests/integration/generator/kernel.py delete mode 100644 tests/integration/generator/preproc.py delete mode 100644 tests/integration/generator/regression.py delete mode 100755 tests/integration/libshogun/test_all.sh delete mode 100644 tests/integration/lua_modular/generator.lua delete mode 100644 tests/integration/matlab_and_octave/classifier.m delete mode 100644 tests/integration/matlab_and_octave/clustering.m delete mode 100644 tests/integration/matlab_and_octave/distance.m delete mode 100644 tests/integration/matlab_and_octave/distribution.m delete mode 100644 tests/integration/matlab_and_octave/kernel.m delete mode 100644 tests/integration/matlab_and_octave/preprocessor.m delete mode 100644 tests/integration/matlab_and_octave/regression.m delete mode 100755 tests/integration/matlab_and_octave/test_all.sh delete mode 100755 tests/integration/matlab_and_octave/test_one.sh delete mode 100644 tests/integration/matlab_and_octave/util/check_accuracy.m delete mode 100644 tests/integration/matlab_and_octave/util/convert_features_and_add_preproc.m delete mode 100644 tests/integration/matlab_and_octave/util/fix_classifier_name_inconsistency.m delete mode 100644 tests/integration/matlab_and_octave/util/fix_clustering_name_inconsistency.m delete mode 100644 tests/integration/matlab_and_octave/util/fix_distance_name_inconsistency.m delete mode 100644 tests/integration/matlab_and_octave/util/fix_kernel_name_inconsistency.m delete mode 100644 tests/integration/matlab_and_octave/util/fix_normalizer_name_inconsistency.m delete mode 100644 tests/integration/matlab_and_octave/util/fix_preproc_name_inconsistency.m delete mode 100644 tests/integration/matlab_and_octave/util/fix_regression_name_inconsistency.m delete mode 100644 tests/integration/matlab_and_octave/util/globals.m delete mode 100644 tests/integration/matlab_and_octave/util/set_distance.m delete mode 100644 tests/integration/matlab_and_octave/util/set_features.m delete mode 100644 tests/integration/matlab_and_octave/util/set_kernel.m delete mode 100644 tests/integration/matlab_and_octave/util/tobool.m delete mode 120000 tests/integration/matlab_static delete mode 100644 tests/integration/octave_modular/classifier.m delete mode 100644 tests/integration/octave_modular/clustering.m delete mode 100644 tests/integration/octave_modular/distance.m delete mode 100644 tests/integration/octave_modular/distribution.m delete mode 100644 tests/integration/octave_modular/kernel.m delete mode 100644 tests/integration/octave_modular/preprocessor.m delete mode 100644 tests/integration/octave_modular/regression.m delete mode 100755 tests/integration/octave_modular/test_all.sh delete mode 100755 tests/integration/octave_modular/test_one.sh delete mode 100644 tests/integration/octave_modular/util/check_accuracy.m delete mode 100644 tests/integration/octave_modular/util/convert_features_and_add_preproc.m delete mode 100644 tests/integration/octave_modular/util/globals.m delete mode 100644 tests/integration/octave_modular/util/set_distance.m delete mode 100644 tests/integration/octave_modular/util/set_features.m delete mode 100644 tests/integration/octave_modular/util/set_kernel.m delete mode 100644 tests/integration/octave_modular/util/set_pie.m delete mode 100644 tests/integration/octave_modular/util/set_pos_and_neg.m delete mode 100644 tests/integration/octave_modular/util/tobool.m delete mode 120000 tests/integration/octave_static delete mode 100644 tests/integration/perl_modular/README delete mode 100644 tests/integration/perl_modular/classifier.pm delete mode 100644 tests/integration/perl_modular/clustering.pm delete mode 100644 tests/integration/perl_modular/distance.pm delete mode 100644 tests/integration/perl_modular/distribution.pm delete mode 100644 tests/integration/perl_modular/generator.pm delete mode 100644 tests/integration/perl_modular/kernel.pm delete mode 100644 tests/integration/perl_modular/preprocessor.pm delete mode 100644 tests/integration/perl_modular/regression.pm delete mode 100755 tests/integration/perl_modular/test_all.sh delete mode 100755 tests/integration/perl_modular/test_one.pl delete mode 100755 tests/integration/perl_modular/test_svm_kernel_multiple.pl delete mode 100755 tests/integration/perl_modular/tester.pl delete mode 100644 tests/integration/perl_modular/util.pm delete mode 100644 tests/integration/python_modular/CMakeLists.txt delete mode 100644 tests/integration/python_modular/README delete mode 100644 tests/integration/python_modular/classifier.py delete mode 100644 tests/integration/python_modular/clustering.py delete mode 100644 tests/integration/python_modular/distance.py delete mode 100644 tests/integration/python_modular/distribution.py delete mode 100755 tests/integration/python_modular/generator.py delete mode 100644 tests/integration/python_modular/kernel.py delete mode 100644 tests/integration/python_modular/preprocessor.py delete mode 100644 tests/integration/python_modular/regression.py delete mode 100755 tests/integration/python_modular/test_all.sh delete mode 100755 tests/integration/python_modular/test_one.py delete mode 100644 tests/integration/python_modular/test_svm_kernel_multiple.py delete mode 100755 tests/integration/python_modular/tester.py delete mode 120000 tests/integration/python_modular/tests delete mode 100644 tests/integration/python_modular/util.py delete mode 100644 tests/integration/python_static/classifier.py delete mode 100644 tests/integration/python_static/clustering.py delete mode 100644 tests/integration/python_static/distance.py delete mode 100644 tests/integration/python_static/distribution.py delete mode 100644 tests/integration/python_static/kernel.py delete mode 100644 tests/integration/python_static/preprocessor.py delete mode 100644 tests/integration/python_static/regression.py delete mode 100755 tests/integration/python_static/test_all.sh delete mode 100755 tests/integration/python_static/test_one.py delete mode 100644 tests/integration/python_static/test_svm_kernel_multiple.py delete mode 100644 tests/integration/python_static/util.py delete mode 100644 tests/integration/r_modular/classifier.R delete mode 100644 tests/integration/r_modular/clustering.R delete mode 100644 tests/integration/r_modular/distance.R delete mode 100644 tests/integration/r_modular/distribution.R delete mode 100644 tests/integration/r_modular/kernel.R delete mode 100644 tests/integration/r_modular/preproc.R delete mode 100644 tests/integration/r_modular/regression.R delete mode 100755 tests/integration/r_modular/test_all.sh delete mode 100644 tests/integration/r_modular/test_one.R delete mode 100755 tests/integration/r_modular/test_one.sh delete mode 100644 tests/integration/r_modular/util/check_accuracy.R delete mode 100644 tests/integration/r_modular/util/convert_features_and_add_preproc.R delete mode 100644 tests/integration/r_modular/util/get_distance.R delete mode 100644 tests/integration/r_modular/util/get_features.R delete mode 100644 tests/integration/r_modular/util/get_kernel.R delete mode 100644 tests/integration/r_modular/util/get_pie.R delete mode 100644 tests/integration/r_modular/util/get_posneg.R delete mode 100644 tests/integration/r_modular/util/read_mfile.R delete mode 100644 tests/integration/r_modular/util/tobool.R delete mode 100644 tests/integration/r_static/classifier.R delete mode 100644 tests/integration/r_static/clustering.R delete mode 100644 tests/integration/r_static/distance.R delete mode 100644 tests/integration/r_static/distribution.R delete mode 100644 tests/integration/r_static/kernel.R delete mode 100644 tests/integration/r_static/preprocessor.R delete mode 100644 tests/integration/r_static/regression.R delete mode 100755 tests/integration/r_static/test_all.sh delete mode 100644 tests/integration/r_static/test_one.R delete mode 100755 tests/integration/r_static/test_one.sh delete mode 100644 tests/integration/r_static/util/check_accuracy.R delete mode 100644 tests/integration/r_static/util/convert_features_and_add_preproc.R delete mode 100644 tests/integration/r_static/util/fix_classifier_name_inconsistency.R delete mode 100644 tests/integration/r_static/util/fix_clustering_name_inconsistency.R delete mode 100644 tests/integration/r_static/util/fix_distance_name_inconsistency.R delete mode 100644 tests/integration/r_static/util/fix_kernel_name_inconsistency.R delete mode 100644 tests/integration/r_static/util/fix_normalizer_name_inconsistency.R delete mode 100644 tests/integration/r_static/util/fix_preproc_name_inconsistency.R delete mode 100644 tests/integration/r_static/util/fix_regression_name_inconsistency.R delete mode 100644 tests/integration/r_static/util/read_mfile.R delete mode 100644 tests/integration/r_static/util/set_distance.R delete mode 100644 tests/integration/r_static/util/set_features.R delete mode 100644 tests/integration/r_static/util/set_kernel.R delete mode 100644 tests/integration/r_static/util/tobool.R delete mode 100644 tests/integration/test_configure.py diff --git a/.gitignore b/.gitignore index 74ed8e478dd..794e8d2ee77 100644 --- a/.gitignore +++ b/.gitignore @@ -55,9 +55,6 @@ cpplint.py /src/configure-*-*.c* /src/build-local -# static interfaces -/src/interfaces/cmdline_static/shogun - # modular interfaces /src/interfaces/*_modular/*.doxy /src/interfaces/*_modular/Evaluation.i diff --git a/CMakeLists.txt b/CMakeLists.txt index c54e74da927..7eaf43bf972 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -263,7 +263,7 @@ ENDIF() # this has been only fixed in swig 2.0.12 or later. IF (NOT ((CYGWIN AND ENABLE_TESTING) OR (DARWIN AND COMPILE_MODULAR_INTERFACE - AND SWIG_VERSION VERSION_LESS "2.0.12") OR MatlabStatic)) + AND SWIG_VERSION VERSION_LESS "2.0.12"))) INCLUDE(CheckCXX11Features) IF(_HAS_CXX11_FLAG) @@ -384,6 +384,13 @@ OPTION(PerlModular "Perl Modular" OFF) OPTION(RubyModular "Ruby Modular" OFF) OPTION(CSharpModular "CSharp Modular" OFF) +SET(STATIC_INTERFACES PythonStatic;OctaveStatic;RStatic;MatlabStatic;CmdLineStatic) +FOREACH(static_interface ${STATIC_INTERFACES}) + if(${static_interface}) + MESSAGE(FATAL_ERROR "Static interfaces are deprecated, use the corresponding modular interface!") + endif() +ENDFOREACH() + # Debugging Python-interface with CTest OPTION(ENABLE_PYTHON_DEBUG "Enable Python-interface-debugging with CTest" OFF) @@ -958,7 +965,7 @@ IF(REDUCE_SWIG_DEBUG) ENDIF(REDUCE_SWIG_DEBUG) # python modular -IF (PythonModular OR PythonStatic) +IF (PythonModular) # find python library for the found python interpreter # i.e. versions should match! FIND_PACKAGE(PythonLibs ${PYTHON_VERSION_STRING} EXACT REQUIRED) @@ -999,12 +1006,6 @@ IF (PythonModular OR PythonStatic) add_subdirectory(${CMAKE_SOURCE_DIR}/src/interfaces/python_modular) ENDIF() ENDIF() - - IF(PythonStatic) - IF(EXISTS ${CMAKE_SOURCE_DIR}/src/interfaces/python_static) - add_subdirectory(${CMAKE_SOURCE_DIR}/src/interfaces/python_static) - ENDIF() - ENDIF() ENDIF() # lua modular @@ -1055,7 +1056,7 @@ IF (RubyModular) ENDIF() # octave modular -IF (OctaveModular OR OctaveStatic) +IF (OctaveModular) FIND_PACKAGE(Octave 3.6 REQUIRED) SET(OCTAVE_APIVERSION ${OCTAVE_API_VERSION}) @@ -1065,12 +1066,6 @@ IF (OctaveModular OR OctaveStatic) add_subdirectory(${CMAKE_SOURCE_DIR}/src/interfaces/octave_modular) ENDIF() ENDIF() - - IF(OctaveStatic) - IF(EXISTS ${CMAKE_SOURCE_DIR}/src/interfaces/octave_static) - add_subdirectory(${CMAKE_SOURCE_DIR}/src/interfaces/octave_static) - ENDIF() - ENDIF() ENDIF() # csharp modular @@ -1083,7 +1078,7 @@ IF (CSharpModular) ENDIF() # r modular -IF (RModular OR RStatic) +IF (RModular) FIND_PACKAGE(R REQUIRED) IF (RModular) @@ -1092,12 +1087,6 @@ IF (RModular OR RStatic) add_subdirectory(${CMAKE_SOURCE_DIR}/src/interfaces/r_modular) ENDIF() ENDIF() - - IF (RStatic) - IF(EXISTS ${CMAKE_SOURCE_DIR}/src/interfaces/r_static) - add_subdirectory(${CMAKE_SOURCE_DIR}/src/interfaces/r_static) - ENDIF() - ENDIF() ENDIF() # perl modular @@ -1109,23 +1098,6 @@ IF (PerlModular) ENDIF() ENDIF() -IF (MatlabStatic) - FIND_PACKAGE(Matlab REQUIRED) - IF(MATLAB_FOUND) - IF(EXISTS ${CMAKE_SOURCE_DIR}/src/interfaces/matlab_static) - add_subdirectory(${CMAKE_SOURCE_DIR}/src/interfaces/matlab_static) - ENDIF() - ELSE() - MESSAGE(FATAL_ERROR "Could not find Matlab, which is required for compiling matlab_static interface. Try setting MATLAB_ROOT enviroment variable to the right path") - ENDIF() -ENDIF() - -IF (CmdLineStatic) - IF(EXISTS ${CMAKE_SOURCE_DIR}/src/interfaces/cmdline_static) - add_subdirectory(${CMAKE_SOURCE_DIR}/src/interfaces/cmdline_static) - ENDIF() -ENDIF() - IF (SVMLight) MESSAGE(STATUS ${SVMLightWarning}) ENDIF() @@ -1143,7 +1115,6 @@ ENDIF() OPTION(ENABLE_TESTING "Enable testing" OFF) OPTION(TRAVIS_DISABLE_UNIT_TESTS "Disable unit testing to speed up jobs on travis-ci" OFF) OPTION(TRAVIS_DISABLE_LIBSHOGUN_TESTS "Disable libshogun tests to speed up jobs on travis-ci" OFF) -OPTION(INTEGRATION_TESTS_ENABLED "Enable integration testing (legacy)" OFF) OPTION(TRAVIS_DISABLE_META_CPP "Disable cpp meta examples and integration testing to speed up jobs on travis-ci" OFF) OPTION(DISABLE_META_INTEGRATION_TESTS "Disable meta integration testing to speed up build" OFF) @@ -1155,10 +1126,6 @@ IF(ENABLE_TESTING) ENDIF() IF(EXISTS ${CMAKE_SOURCE_DIR}/tests) - if (INTEGRATION_TESTS_ENABLED AND EXISTS ${CMAKE_SOURCE_DIR}/tests/integration) - add_subdirectory(${CMAKE_SOURCE_DIR}/tests/integration) - ENDIF() - IF (NOT TRAVIS_DISABLE_UNIT_TESTS AND EXISTS ${CMAKE_SOURCE_DIR}/tests/unit) add_subdirectory(${CMAKE_SOURCE_DIR}/tests/unit) ENDIF() diff --git a/debian b/debian index d5c98311e0c..2ad8c961c8f 160000 --- a/debian +++ b/debian @@ -1 +1 @@ -Subproject commit d5c98311e0c390380e0b1b01071914c300d31167 +Subproject commit 2ad8c961c8f29a1ac1bdaf77cbad3b0eff3bce32 diff --git a/src/interfaces/cmdline_static/CMakeLists.txt b/src/interfaces/cmdline_static/CMakeLists.txt deleted file mode 100644 index 35ce1e6d9a2..00000000000 --- a/src/interfaces/cmdline_static/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -include_directories(${SYSTEM_INCLUDES}) -SET(CMDLINE_STATIC_EXEC "${CMAKE_CURRENT_BINARY_DIR}/cmdline_static" PARENT_SCOPE) -add_executable(cmdline_static cmdline.cpp CmdLineInterface.cpp) -target_link_libraries(cmdline_static shogun) - -# Add static-interface to the dependencies of modular interfaces to make -# sure nothing will infer with them being build single-threaded. -IF(SWIG_SINGLE_THREADED) - FOREACH(SG_MODULAR_INTERFACE_TARGET ${SG_MODULAR_INTERFACE_TARGETS}) - ADD_DEPENDENCIES(${SG_MODULAR_INTERFACE_TARGET} cmdline_static) - ENDFOREACH(SG_MODULAR_INTERFACE_TARGET ${SG_MODULAR_INTERFACE_TARGETS}) -ENDIF(SWIG_SINGLE_THREADED) - -INSTALL(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/cmdline_static - DESTINATION bin - COMPONENT cmdline - RENAME shogun) diff --git a/src/interfaces/cmdline_static/CmdLineInterface.cpp b/src/interfaces/cmdline_static/CmdLineInterface.cpp deleted file mode 100644 index be26ca92517..00000000000 --- a/src/interfaces/cmdline_static/CmdLineInterface.cpp +++ /dev/null @@ -1,794 +0,0 @@ -#include "CmdLineInterface.h" - -#include -#include -#include -#include -#include -#include -#include - -#ifdef HAVE_READLINE -#include -#include -#endif - -#ifndef WIN32 -#include -#include -#include -#endif - -#include -#include - -void cmdline_print_message(FILE* target, const char* str) -{ - fprintf(target, "%s", str); -} - -void cmdline_print_warning(FILE* target, const char* str) -{ - fprintf(target, "%s", str); -} - -void cmdline_print_error(FILE* target, const char* str) -{ - fprintf(target, "%s", str); -} - -void cmdline_cancel_computations(bool &delayed, bool &immediately) -{ -} - -using namespace shogun; - -const int32_t READLINE_BUFFER_SIZE = 10000; -extern CSGInterface* interface; -extern CSGInterfaceMethod sg_methods[]; - -CCmdLineInterface::CCmdLineInterface() -: CSGInterface(), m_lhs(NULL), m_rhs(NULL) -{ - reset(); -} - -CCmdLineInterface::~CCmdLineInterface() -{ - delete m_lhs; - delete m_rhs; -} - -void CCmdLineInterface::reset(const char* line) -{ - CSGInterface::reset(); - - if (!line) - return; - - char* element=NULL; - const char delim_equal[]="="; - const char delim_lhs[]="=, \t\n"; - const char delim_rhs[]=" \t\n"; - - delete m_lhs; - m_lhs=NULL; - delete m_rhs; - m_rhs=NULL; - - /* split lhs from rhs - * for some reason strstr on sunos and newer libc's - * requires a char* haystack - */ - char* equal_sign=strstr((char*) line, delim_equal); - if (equal_sign) - { -#ifdef DEBUG_CMDLINEIF - SG_PRINT("has lhs!\n"); -#endif - element=strtok((char*) line, delim_lhs); - if (element) - { - m_lhs=new DynArray(); - m_lhs->append_element(element); - m_nlhs++; - while ((element=strtok(NULL, delim_lhs))) - { - if (element>equal_sign) // on rhs now - break; - - m_lhs->append_element(element); - m_nlhs++; - } - } - } - else - element=strtok((char*) line, delim_rhs); - - if (element) - { - m_rhs=new DynArray(); - m_rhs->append_element(element); - m_nrhs++; - while ((element=strtok(NULL, delim_rhs))) - { - m_rhs->append_element(element); - m_nrhs++; - } - } - else - m_rhs=NULL; - -#ifdef DEBUG_CMDLINEIF - SG_PRINT("nlhs=%d nrhs=%d\n", m_nlhs, m_nrhs); - if (m_lhs) - { - for (int32_t i=0; iget_num_elements(); i++) - SG_PRINT("element lhs %i %s\n", i, m_lhs->get_element(i)); - } - - if (m_rhs) - { - for (int32_t i=0; iget_num_elements(); i++) - SG_PRINT("element rhs %i %s\n", i, m_rhs->get_element(i)); - } -#endif -} - - -/** get functions - to pass data from the target interface to shogun */ - -/** determine argument type - * - * a signature is read from a data file. this signature contains the argument - * type. - * - * currently, the signature must be in the beginning of the file, - * consists of a line starting with 3 hash signs, 1 space and then - * the argument type. e.g.: - * - * ### SHOGUN V0 STRING_CHAR - * ACGTGCAAAAGC - * AGTCDTCD - */ -IFType CCmdLineInterface::get_argument_type() -{ - const int32_t len=1024; - IFType argtype=UNDEFINED; - const char* filename=m_rhs->get_element(m_rhs_counter); - - // read the first 1024 of the file and heuristically decide about its - // content - FILE* fh=fopen((char*) filename, "r"); - if (!fh) - SG_ERROR("Could not find file %s.\n", filename); - - char* chunk=SG_MALLOC(char, len+1); - memset(chunk, 0, sizeof(char)*(len+1)); - size_t nread=fread(chunk, sizeof(char), len, fh); - fclose(fh); - if (nread<=0) - SG_ERROR("Could not read data from %s.\n"); - - char* signature=SG_MALLOC(char, len+1); - int32_t num=sscanf(chunk, "### SHOGUN V0 %s\n", signature); - - // if file has valid shogun signature use it to determine file type - if (num==1) - { - SG_DEBUG("read signature: %s\n", signature); - - if (strncmp(signature, "STRING_CHAR", 11)==0) - argtype=STRING_CHAR; - else if (strncmp(signature, "STRING_BYTE", 11)==0) - argtype=STRING_BYTE; - else if (strncmp(signature, "DENSE_REAL", 10)==0) - argtype=DENSE_REAL; - else if (strncmp(signature, "SPARSE_REAL", 11)==0) - argtype=SPARSE_REAL; - } - else - { - SG_DEBUG("could not find signature in file %s guessing file type.\n", filename); - - // special for cubes - if (strspn(chunk, "123456\n")==nread) - { - argtype=STRING_CHAR; - SG_DEBUG("guessing STRING_CHAR\n"); - } - else if (strspn(chunk, "0123456789.e+- \t\n")==nread) - { - argtype=DENSE_REAL; - SG_DEBUG("guessing DENSE_REAL\n"); - } - else if (strspn(chunk, "0123456789:.e+- \t\n")==nread) - { - argtype=SPARSE_REAL; - SG_DEBUG("guessing SPARSE_REAL\n"); - } - else - { - argtype=STRING_CHAR; - SG_DEBUG("guessing STRING_CHAR\n"); - } - } - - SG_FREE(signature); - SG_FREE(chunk); - return argtype; -} - - -int32_t CCmdLineInterface::get_int() -{ - const char* i=get_arg_increment(); - if (!i) - SG_ERROR("Expected Scalar Integer as argument %d\n", m_rhs_counter); - - int32_t value=-1; - int32_t num=sscanf(i, "%d", &value); - if (num!=1) - SG_ERROR("Expected Scalar Integer as argument %d\n", m_rhs_counter); - - return value; -} - -float64_t CCmdLineInterface::get_real() -{ - const char* r=get_arg_increment(); - if (!r) - SG_ERROR("Expected Scalar Real as argument %d\n", m_rhs_counter); - - float64_t value=-1; - int32_t num=sscanf(r, "%lf", &value); - if (num!=1) - SG_ERROR("Expected Scalar Real as argument %d\n", m_rhs_counter); - - return value; -} - -bool CCmdLineInterface::get_bool() -{ - const char* b=get_arg_increment(); - if (!b) - SG_ERROR("Expected Scalar Bool as argument %d\n", m_rhs_counter); - - int32_t value=-1; - int32_t num=sscanf(b, "%i", &value); - if (num!=1) - SG_ERROR("Expected Scalar Bool as argument %d\n", m_rhs_counter); - - return (value!=0); -} - - -char* CCmdLineInterface::get_string(int32_t& len) -{ - const char* s=get_arg_increment(); - if (!s) - SG_ERROR("Expected 1 String as argument %d.\n", m_rhs_counter); - - len=strlen(s); - ASSERT(len>0); - - char* result=SG_MALLOC(char, len+1); - memcpy(result, s, len*sizeof(char)); - result[len]='\0'; - - return result; -} - -#define GET_VECTOR(fname, sg_type) \ -void CCmdLineInterface::fname(sg_type*& vec, int32_t& len) \ -{ \ - const char* filename=get_arg_increment(); \ - if (!filename) \ - SG_ERROR("No filename given to read vector.\n"); \ - \ - CCSVFile f((char*) filename, 'r'); \ - \ - try \ - { \ - f.fname(vec, len); \ - } \ - catch (...) \ - { \ - SG_ERROR("Could not read data from %s.\n", filename); \ - } \ -} -GET_VECTOR(get_vector, uint8_t) -GET_VECTOR(get_vector, char) -GET_VECTOR(get_vector, int32_t) -GET_VECTOR(get_vector, float32_t) -GET_VECTOR(get_vector, float64_t) -GET_VECTOR(get_vector, int16_t) -GET_VECTOR(get_vector, uint16_t) -#undef GET_VECTOR - -#define GET_MATRIX(fname, sg_type) \ -void CCmdLineInterface::fname(sg_type*& matrix, int32_t& num_feat, int32_t& num_vec) \ -{ \ - const char* filename=get_arg_increment(); \ - if (!filename) \ - SG_ERROR("No filename given to read matrix.\n"); \ - \ - CCSVFile f((char*) filename, 'r'); \ - \ - try \ - { \ - f.fname(matrix, num_feat, num_vec); \ - } \ - catch (...) \ - { \ - SG_ERROR("Could not read data from %s.\n", filename); \ - } \ -} -GET_MATRIX(get_matrix, uint8_t) -GET_MATRIX(get_matrix, char) -GET_MATRIX(get_matrix, int32_t) -GET_MATRIX(get_matrix, float32_t) -GET_MATRIX(get_matrix, float64_t) -GET_MATRIX(get_matrix, int16_t) -GET_MATRIX(get_matrix, uint16_t) -#undef GET_MATRIX - -void CCmdLineInterface::get_ndarray(uint8_t*& array, int32_t*& dims, int32_t& num_dims) -{ -} - -void CCmdLineInterface::get_ndarray(char*& array, int32_t*& dims, int32_t& num_dims) -{ -} - -void CCmdLineInterface::get_ndarray(int32_t*& array, int32_t*& dims, int32_t& num_dims) -{ -} - -void CCmdLineInterface::get_ndarray(float32_t*& array, int32_t*& dims, int32_t& num_dims) -{ -} - -void CCmdLineInterface::get_ndarray(float64_t*& array, int32_t*& dims, int32_t& num_dims) -{ -} - -void CCmdLineInterface::get_ndarray(int16_t*& array, int32_t*& dims, int32_t& num_dims) -{ -} - -void CCmdLineInterface::get_ndarray(uint16_t*& array, int32_t*& dims, int32_t& num_dims) -{ -} - -void CCmdLineInterface::get_sparse_matrix(SGSparseVector*& matrix, int32_t& num_feat, int32_t& num_vec) -{ - const char* filename=get_arg_increment(); - if (!filename) - SG_ERROR("No filename given to read SPARSE REAL matrix.\n"); - - CLibSVMFile f((char*) filename, 'r'); - f.get_sparse_matrix(matrix, num_feat, num_vec); -} - - -void CCmdLineInterface::get_string_list(SGString*& strings, int32_t& num_str, int32_t& max_string_len) -{ - strings=NULL; - num_str=0; - max_string_len=0; -} - -void CCmdLineInterface::get_string_list(SGString*& strings, int32_t& num_str, int32_t& max_string_len) -{ - const char* filename=get_arg_increment(); - if (!filename) - SG_ERROR("No filename given to read CHAR string list.\n"); - - CCSVFile f((char*) filename, 'r'); - f.get_string_list(strings, num_str, max_string_len); -} - -void CCmdLineInterface::get_string_list(SGString*& strings, int32_t& num_str, int32_t& max_string_len) -{ - strings=NULL; - num_str=0; - max_string_len=0; -} - -void CCmdLineInterface::get_string_list(SGString*& strings, int32_t& num_str, int32_t& max_string_len) -{ - strings=NULL; - num_str=0; - max_string_len=0; -} - -void CCmdLineInterface::get_string_list(SGString*& strings, int32_t& num_str, int32_t& max_string_len) -{ - strings=NULL; - num_str=0; - max_string_len=0; -} - - -void CCmdLineInterface::get_attribute_struct(const CDynamicArray* &attrs) -{ - attrs=NULL; -} - -/** set functions - to pass data from shogun to the target interface */ -bool CCmdLineInterface::create_return_values(int32_t num) -{ - if (num==m_nlhs) - return true; - - return false; -} - -void* CCmdLineInterface::get_return_values() -{ - return NULL; -} - - -/** set functions - to pass data from shogun to the target interface */ - -#define SET_SCALAR(fname, mfname, sg_type) \ -void CCmdLineInterface::fname(sg_type scalar) \ -{ \ - const char* filename=set_arg_increment(); \ - CCSVFile f((char*) filename, 'w'); \ - f.mfname(&scalar, 1, 1); \ -} -SET_SCALAR(set_int, set_matrix, int32_t) -SET_SCALAR(set_real, set_matrix, float64_t) -SET_SCALAR(set_bool, CFile::set_matrix, bool) -#undef SET_SCALAR - -#define SET_VECTOR(fname, mfname, sg_type) \ -void CCmdLineInterface::fname(const sg_type* vec, int32_t len) \ -{ \ - const char* filename=set_arg_increment(); \ - if (!filename) \ - SG_ERROR("No filename given to write vector.\n"); \ - \ - CCSVFile f((char*) filename, 'w'); \ - f.mfname(vec, len, 1); \ -} -SET_VECTOR(set_vector, set_matrix, uint8_t) -SET_VECTOR(set_vector, set_matrix, char) -SET_VECTOR(set_vector, set_matrix, int32_t) -SET_VECTOR(set_vector, set_matrix, float32_t) -SET_VECTOR(set_vector, set_matrix, float64_t) -SET_VECTOR(set_vector, set_matrix, int16_t) -SET_VECTOR(set_vector, set_matrix, uint16_t) -#undef SET_VECTOR - -#define SET_MATRIX(fname, sg_type) \ -void CCmdLineInterface::fname(const sg_type* matrix, int32_t num_feat, int32_t num_vec) \ -{ \ - const char* filename=set_arg_increment(); \ - if (!filename) \ - SG_ERROR("No filename given to write matrix.\n"); \ - \ - CCSVFile f((char*) filename, 'w'); \ - f.fname(matrix, num_feat, num_vec); \ -} -SET_MATRIX(set_matrix, uint8_t) -SET_MATRIX(set_matrix, char) -SET_MATRIX(set_matrix, int32_t) -SET_MATRIX(set_matrix, float32_t) -SET_MATRIX(set_matrix, float64_t) -SET_MATRIX(set_matrix, int16_t) -SET_MATRIX(set_matrix, uint16_t) - -void CCmdLineInterface::set_sparse_matrix(const SGSparseVector* matrix, int32_t num_feat, int32_t num_vec, int64_t nnz) -{ - const char* filename=set_arg_increment(); - if (!filename) - SG_ERROR("No filename given to write SPARSE REAL matrix.\n"); - - CCSVFile f((char*) filename, 'w'); - f.set_sparse_matrix(matrix, num_feat, num_vec); -} - -#define SET_STRING_LIST(fname, sg_type) \ -void CCmdLineInterface::fname(const SGString* strings, int32_t num_str) \ -{ \ - const char* filename=set_arg_increment(); \ - if (!filename) \ - SG_ERROR("No filename given to write CHAR string list.\n"); \ - \ - CCSVFile f((char*) filename, 'w'); \ - f.fname(strings, num_str); \ -} -SET_STRING_LIST(set_string_list, uint8_t) -SET_STRING_LIST(set_string_list, char) -SET_STRING_LIST(set_string_list, int32_t) -SET_STRING_LIST(set_string_list, int16_t) -SET_STRING_LIST(set_string_list, uint16_t) -#undef SET_STRING_LIST - -void CCmdLineInterface::set_attribute_struct(const CDynamicArray* attrs) -{ -} - -bool CCmdLineInterface::skip_line(const char* line) -{ - if (!line) - return true; - - if (line[0]=='\n' || - (line[0]=='\r' && line[1]=='\n') || - int(line[0])==0) - { - return true; - } - - //SG_PRINT("ascii(0) %d, %c\n", int(line[0]), line[0]); - - char* skipped=SGIO::skip_blanks((char*) line); - if (skipped[0]==CMDLINE_COMMENT0 || skipped[0]==CMDLINE_COMMENT1) - return true; - - return false; -} - -void CCmdLineInterface::print_prompt() -{ - SG_PRINT( "\033[1;34mshogun\033[0m >> "); - //SG_PRINT("shogun >> "); -} - - -char* CCmdLineInterface::get_line(FILE* infile, bool interactive_mode) -{ - char* in=NULL; - memset(input, 0, sizeof(input)); - - if (feof(infile)) - return NULL; - -#ifdef HAVE_READLINE - if (interactive_mode) - { - in=readline("\033[1;34mshogun\033[0m >> "); - if (in) - { - strncpy(input, in, sizeof(input)); - add_history(in); - free(in); - } - } - else - { - if (fgets(input, sizeof(input), infile)==NULL) - return NULL; - in=input; - } -#else - if (interactive_mode) - print_prompt(); - if (fgets(input, sizeof(input), infile)==NULL) - return NULL; - in=input; -#endif - - if (in==NULL) - return NULL; - else - return input; -} - -bool CCmdLineInterface::parse_line(char* line) -{ - if (!line) - return false; - - if (skip_line(line)) - return true; - else - { - ((CCmdLineInterface*) interface)->reset(line); - return interface->handle(); - } -} - -#ifdef HAVE_READLINE -char* command_generator(const char *text, int state) -{ - static int list_index, len; - const char *name; - - /* If this is a new word to complete, initialize now. This - * includes saving the length of TEXT for efficiency, and - * initializing the index variable to 0. */ - if (!state) - { - list_index = 0; - len = strlen (text); - } - - /* Return the next name which partially matches from the - * command list. */ - while ((name = sg_methods[list_index].command)) - { - list_index++; - - if (strncmp (name, text, len) == 0) - return (strdup(name)); - } - - /* If no names matched, then return NULL. */ - return NULL; -} - -/* Attempt to complete on the contents of TEXT. START and END - * bound the region of rl_line_buffer that contains the word to - * complete. TEXT is the word to complete. We can use the entire - * contents of rl_line_buffer in case we want to do some simple - * parsing. Returnthe array of matches, or NULL if there aren't - * any. */ -char** shogun_completion (const char *text, int start, int end) -{ - char **matches; - - matches = (char **)NULL; - - /* If this word is at the start of the line, then it is a command - * to complete. Otherwise it is the name of a file in the - * current - * directory. */ - if (start == 0) - matches = rl_completion_matches (text, command_generator); - - return (matches); -} -#endif //HAVE_READLINE - -int main(int argc, char* argv[]) -{ -#ifdef HAVE_READLINE - rl_readline_name = "shogun"; - rl_attempted_completion_function = shogun_completion; -#endif //HAVE_READLINE - - init_shogun(&cmdline_print_message, &cmdline_print_warning, - &cmdline_print_error, &cmdline_cancel_computations); - try - { - interface=new CCmdLineInterface(); - CCmdLineInterface* intf=(CCmdLineInterface*) interface; - - // interactive - if (argc<=1) - { - while (true) - { - char* line=intf->get_line(); - - if (!line) - break; - - try - { - intf->parse_line(line); - } - catch (ShogunException e) { } - - } - - SG_UNREF(interface); - exit_shogun(); - return 0; - } - - // help - if ( argc>2 || ((argc==2) && - ( !strcmp(argv[1], "-h") || !strcmp(argv[1], "/?") || !strcmp(argv[1], "--help")) ) - ) - { - SG_SPRINT("\n\n"); - SG_SPRINT("usage: shogun [ -h | --help | /? | -i | filename ]\n\n"); - SG_SPRINT("if no options are given shogun enters interactive mode\n"); - SG_SPRINT("if filename is specified the commands will be read and executed from file\n"); - SG_SPRINT("if -i is specified shogun will listen on port 7367 from file\n"); - SG_SPRINT("==hex(sg), *dangerous* as commands from any source are accepted\n\n"); - - SG_UNREF(interface); - exit_shogun(); - return 1; - } - -#ifndef CYGWIN - // from tcp - if ( argc==2 && !strcmp(argv[1], "-i")) - { - int s=socket(AF_INET, SOCK_STREAM, 0); - struct sockaddr_in sa; - sa.sin_family=AF_INET; - sa.sin_port=htons(7367); - sa.sin_addr.s_addr=INADDR_ANY; - bzero(&(sa.sin_zero), 8); - - bind(s, (sockaddr*) (&sa), sizeof(sockaddr_in)); - listen(s, 1); - int s2=accept(s, NULL, NULL); - SG_SINFO( "accepting connection\n"); - - char input[READLINE_BUFFER_SIZE]; - do - { - bzero(input, sizeof(input)); - int length=read(s2, input, sizeof(input)); - if (length>0 && length<(int) sizeof(input)) - input[length]='\0'; - else - { - SG_SERROR( "error reading cmdline\n"); - return 1; - } - } - while (intf->parse_line(input)); - SG_UNREF(interface); - exit_shogun(); - return 0; - } -#endif - - // from file - if (argc==2) - { - FILE* file=fopen(argv[1], "r"); - - if (!file) - { - SG_SERROR( "error opening/reading file: \"%s\"",argv[1]); - SG_UNREF(interface); - exit_shogun(); - return 1; - } - else - { - try - { - while(!feof(file) && intf->parse_line(intf->get_line(file, false))); - } - catch (ShogunException e) - { - fclose(file); - SG_UNREF(interface); - exit_shogun(); - return 1; - } - - fclose(file); - SG_UNREF(interface); - exit_shogun(); - return 0; - } - } - - } - catch (std::bad_alloc) - { - SG_SPRINT("Out of memory error.\n"); - SG_UNREF(interface); - exit_shogun(); - return 2; - } - catch (ShogunException e) - { - SG_SPRINT("%s", e.get_exception_string()); - SG_UNREF(interface); - exit_shogun(); - return 3; - } - catch (...) - { - SG_SPRINT("Returning from SHOGUN unknown error."); - SG_UNREF(interface); - exit_shogun(); - return 4; - } -} diff --git a/src/interfaces/cmdline_static/CmdLineInterface.h b/src/interfaces/cmdline_static/CmdLineInterface.h deleted file mode 100644 index 5a245eb686b..00000000000 --- a/src/interfaces/cmdline_static/CmdLineInterface.h +++ /dev/null @@ -1,184 +0,0 @@ -#ifndef __CMDLINEINTERFACE__H_ -#define __CMDLINEINTERFACE__H_ - -#include - -#include -#include - -#include - -namespace shogun -{ -#define CMDLINE_COMMENT0 '#' -#define CMDLINE_COMMENT1 '%' - -class CCmdLineInterface : public CSGInterface -{ - public: - CCmdLineInterface(); - ~CCmdLineInterface(); - - /// reset to clean state - virtual void reset(const char* line=NULL); - - /** get functions - to pass data from the target interface to shogun */ - - /// get type of current argument (does not increment argument counter) - virtual IFType get_argument_type(); - - virtual int32_t get_int(); - virtual float64_t get_real(); - virtual bool get_bool(); - - virtual char* get_string(int32_t& len); - - virtual void get_vector(uint8_t*& vec, int32_t& len); - virtual void get_vector(char*& vec, int32_t& len); - virtual void get_vector(int32_t*& vec, int32_t& len); - virtual void get_vector(float32_t*& vec, int32_t& len); - virtual void get_vector(float64_t*& vec, int32_t& len); - virtual void get_vector(int16_t*& vec, int32_t& len); - virtual void get_vector(uint16_t*& vec, int32_t& len); - - virtual void get_matrix( - uint8_t*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - char*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - int32_t*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - float32_t*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - float64_t*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - int16_t*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - uint16_t*& matrix, int32_t& num_feat, int32_t& num_vec); - - virtual void get_ndarray( - uint8_t*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - char*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - int32_t*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - float32_t*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - float64_t*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - int16_t*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - uint16_t*& array, int32_t*& dims, int32_t& num_dims); - - virtual void get_sparse_matrix( - SGSparseVector*& matrix, int32_t& num_feat, int32_t& num_vec); - - virtual void get_string_list( - SGString*& strings, int32_t& num_str, - int32_t& max_string_len); - virtual void get_string_list( - SGString*& strings, int32_t& num_str, - int32_t& max_string_len); - virtual void get_string_list( - SGString*& strings, int32_t& num_str, - int32_t& max_string_len); - virtual void get_string_list( - SGString*& strings, int32_t& num_str, - int32_t& max_string_len); - virtual void get_string_list( - SGString*& strings, int32_t& num_str, - int32_t& max_string_len); - - virtual void get_attribute_struct( - const CDynamicArray* &attrs); - - /** set functions - to pass data from shogun to the target interface */ - virtual void set_int(int32_t scalar); - virtual void set_real(float64_t scalar); - virtual void set_bool(bool scalar); - - virtual bool create_return_values(int32_t num_val); - virtual void set_vector(const uint8_t* vec, int32_t len); - virtual void set_vector(const char* vec, int32_t len); - virtual void set_vector(const int32_t* vec, int32_t len); - virtual void set_vector(const float32_t* vec, int32_t len); - virtual void set_vector(const float64_t* vec, int32_t len); - virtual void set_vector(const int16_t* vec, int32_t len); - virtual void set_vector(const uint16_t* vec, int32_t len); - - virtual void set_matrix( - const uint8_t* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const char* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const int32_t* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const float32_t* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const float64_t* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const int16_t* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const uint16_t* matrix, int32_t num_feat, int32_t num_vec); - - virtual void set_sparse_matrix( - const SGSparseVector* matrix, int32_t num_feat, - int32_t num_vec, int64_t nnz); - - virtual void set_string_list( - const SGString* strings, int32_t num_str); - virtual void set_string_list( - const SGString* strings, int32_t num_str); - virtual void set_string_list( - const SGString* strings, int32_t num_str); - virtual void set_string_list( - const SGString* strings, int32_t num_str); - virtual void set_string_list( - const SGString* strings, int32_t num_str); - - virtual void set_attribute_struct( - const CDynamicArray* attrs); - - void* get_return_values(); - - /** determine if given line is a comment or empty */ - bool skip_line(const char* line=NULL); - - /// get line from file or stdin or... - char* get_line(FILE* infile=stdin, bool interactive_mode=true); - - /// parse a single line - bool parse_line(char* line); - - /// print interactive prompt - void print_prompt(); - - private: - const char* get_arg_increment() - { - ASSERT(m_rhs_counter>=0 && m_rhs_counterget_element(m_rhs_counter); - m_rhs_counter++; - - return element; - } - - const char* set_arg_increment() - { - ASSERT(m_lhs_counter>=0 && m_lhs_counterget_element(m_lhs_counter); - m_lhs_counter++; - - return element; - } - - /** @return object name */ - inline virtual const char* get_name() const { return "CmdLineInterface"; } - - private: - DynArray* m_lhs; - DynArray* m_rhs; -}; -} -#endif // __CMDLINEINTERFACE__H_ diff --git a/src/interfaces/cmdline_static/cmdline.cpp b/src/interfaces/cmdline_static/cmdline.cpp deleted file mode 100644 index 2062308a979..00000000000 --- a/src/interfaces/cmdline_static/cmdline.cpp +++ /dev/null @@ -1,20 +0,0 @@ -#include - -void sg_print_message(FILE* target, const char* str) -{ - fprintf(target, "%s", str); -} - -void sg_print_warning(FILE* target, const char* str) -{ - fprintf(target, "%s", str); -} - -void sg_print_error(FILE* target, const char* str) -{ - fprintf(target, "%s", str); -} - -void sg_cancel_computations(bool &delayed, bool &immediately) -{ -} diff --git a/src/interfaces/elwms_static/elwms.cpp b/src/interfaces/elwms_static/elwms.cpp deleted file mode 100644 index fbcd9cf4ff0..00000000000 --- a/src/interfaces/elwms_static/elwms.cpp +++ /dev/null @@ -1,15 +0,0 @@ -#ifdef HAVE_PYTHON -#include "../python_static/PythonInterface.cpp" -#endif - -#ifdef HAVE_MATLAB -#include "../matlab_static/MatlabInterface.cpp" -#endif - -#ifdef HAVE_OCTAVE -#include "../octave_static/OctaveInterface.cpp" -#endif - -#ifdef HAVE_R -#include "../r_static/RInterface.cpp" -#endif diff --git a/src/interfaces/matlab_static/CMakeLists.txt b/src/interfaces/matlab_static/CMakeLists.txt deleted file mode 100644 index 8dbe9df01c2..00000000000 --- a/src/interfaces/matlab_static/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -ADD_CUSTOM_TARGET(matlab_static ALL - COMMAND "${MATLAB_ROOT}/bin/mex" -o sg -I${CMAKE_SOURCE_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR}/MatlabInterface.cpp $ - DEPENDS shogun - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - COMMENT "Compiling MATLAB static interface") - -# Add static-interface to the dependencies of modular interfaces to make -# sure nothing will infer with them being build single-threaded. -IF(SWIG_SINGLE_THREADED) - FOREACH(SG_MODULAR_INTERFACE_TARGET ${SG_MODULAR_INTERFACE_TARGETS}) - ADD_DEPENDENCIES(${SG_MODULAR_INTERFACE_TARGET} matlab_static) - ENDFOREACH(SG_MODULAR_INTERFACE_TARGET ${SG_MODULAR_INTERFACE_TARGETS}) -ENDIF(SWIG_SINGLE_THREADED) diff --git a/src/interfaces/matlab_static/MatlabInterface.cpp b/src/interfaces/matlab_static/MatlabInterface.cpp deleted file mode 100644 index 3d8e07cea44..00000000000 --- a/src/interfaces/matlab_static/MatlabInterface.cpp +++ /dev/null @@ -1,703 +0,0 @@ -// somehow this has to depend on the matlab version -// comment it out for now -// -// #include -// - -#include "MatlabInterface.h" - -#include -#include -#include -#include -#include - -#ifdef HAVE_PYTHON -#include "../python/PythonInterface.h" -#endif - -#ifdef HAVE_OCTAVE -#include "../octave/OctaveInterface.h" -#endif - -#ifdef HAVE_R -#include "../r/RInterface.h" -#endif - -void matlab_print_message(FILE* target, const char* str) -{ - if (target==stdout) - mexPrintf("%s", str); - else - fprintf(target, "%s", str); -} - -void matlab_print_warning(FILE* target, const char* str) -{ - if (target==stdout) - mexWarnMsgTxt(str); - else - fprintf(target, "%s", str); -} - -void matlab_print_error(FILE* target, const char* str) -{ - fprintf(target, "%s", str); -} - -void matlab_cancel_computations(bool &delayed, bool &immediately) -{ -} - -extern CSGInterface* interface; - -CMatlabInterface::CMatlabInterface( - int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) -: CSGInterface() -{ - reset(nlhs, plhs, nrhs, prhs); - -} - -CMatlabInterface::~CMatlabInterface() -{ -} - -void CMatlabInterface::reset( - int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) -{ - CSGInterface::reset(); - - m_nlhs=nlhs; - m_nrhs=nrhs; - m_lhs=plhs; - m_rhs=prhs; -} - - -/** get functions - to pass data from the target interface to shogun */ - -/// get type of current argument (does not increment argument counter) -IFType CMatlabInterface::get_argument_type() -{ - const mxArray* arg=m_rhs[m_rhs_counter]; - ASSERT(arg); - - if (mxIsSparse(arg)) - { - if (mxIsUint8(arg)) - return SPARSE_BYTE; - if (mxIsChar(arg)) - return SPARSE_CHAR; - if (mxIsInt32(arg)) - return SPARSE_INT; - if (mxIsDouble(arg)) - return SPARSE_REAL; - if (mxIsInt16(arg)) - return SPARSE_SHORT; - if (mxIsSingle(arg)) - return SPARSE_SHORTREAL; - if (mxIsUint16(arg)) - return SPARSE_WORD; - - return UNDEFINED; - } - - if (mxIsInt32(arg)) - return DENSE_INT; - if (mxIsDouble(arg)) - return DENSE_REAL; - if (mxIsInt16(arg)) - return DENSE_SHORT; - if (mxIsSingle(arg)) - return DENSE_SHORTREAL; - if (mxIsUint16(arg)) - return DENSE_WORD; - - if (mxIsChar(arg)) - return STRING_CHAR; - if (mxIsUint8(arg)) - return STRING_BYTE; - - if (mxIsCell(arg)) - { - const mxArray* cell=mxGetCell(arg, 0); - if (cell && mxGetM(cell)==1) - { - if (mxIsUint8(cell)) - return STRING_BYTE; - if (mxIsChar(cell)) - return STRING_CHAR; - if (mxIsInt32(cell)) - return STRING_INT; - if (mxIsInt16(cell)) - return STRING_SHORT; - if (mxIsUint16(cell)) - return STRING_WORD; - } - } - - return UNDEFINED; -} - - -int32_t CMatlabInterface::get_int() -{ - const mxArray* i=get_arg_increment(); - if (!i || !mxIsNumeric(i) || mxGetN(i)!=1 || mxGetM(i)!=1) - SG_ERROR("Expected Scalar Integer as argument %d\n", m_rhs_counter); - - double s=mxGetScalar(i); - if (s-CMath::floor(s)!=0) - SG_ERROR("Expected Integer as argument %d\n", m_rhs_counter); - - return int32_t(s); -} - -float64_t CMatlabInterface::get_real() -{ - const mxArray* f=get_arg_increment(); - if (!f || !mxIsNumeric(f) || mxGetN(f)!=1 || mxGetM(f)!=1) - SG_ERROR("Expected Scalar Float as argument %d\n", m_rhs_counter); - - return mxGetScalar(f); -} - -bool CMatlabInterface::get_bool() -{ - const mxArray* b=get_arg_increment(); - if (mxIsLogical(b) && mxGetN(b)==1 && mxGetM(b)==1) - return mxGetLogicals(b)[0]; - else if (mxIsNumeric(b)) - return (mxGetScalar(b)!=0); - else - SG_ERROR("Expected Scalar Boolean as argument %d\n", m_rhs_counter); - - return false; -} - - -char* CMatlabInterface::get_string(int32_t& len) -{ - bool zero_terminate=true; - const mxArray* s=get_arg_increment(); - - if ( !(mxIsChar(s)) || (mxGetM(s)!=1) ) - SG_ERROR("Expected String as argument %d\n", m_rhs_counter); - - len=mxGetN(s); - char* string=NULL; - if (zero_terminate) - string=SG_MALLOC(char, len+1); - else - string=SG_MALLOC(char, len); - mxChar* c=mxGetChars(s); - ASSERT(c); - for (int32_t i=0; i*& matrix, int32_t& num_feat, int32_t& num_vec) \ -{ \ - const mxArray* mx_mat=get_arg_increment(); \ - if (!mx_mat || !mxIsSparse(mx_mat)) \ - SG_ERROR("Expected Sparse Matrix as argument %d\n", m_rhs_counter); \ - \ - if (!mxIsClass(mx_mat,mx_type)) \ - SG_ERROR("Expected " error_string " Matrix, got class %s as argument %d\n", \ - mxGetClassName(mx_mat), m_rhs_counter); \ - \ - num_vec=mxGetN(mx_mat); \ - num_feat=mxGetM(mx_mat); \ - matrix=SG_MALLOC(SGSparseVector, num_vec); \ - if_type* data=(if_type*) mxGetData(mx_mat); \ - \ - int64_t nzmax=mxGetNzmax(mx_mat); \ - mwIndex* ir=mxGetIr(mx_mat); \ - mwIndex* jc=mxGetJc(mx_mat); \ - int64_t offset=0; \ - for (int32_t i=0; i0) \ - { \ - matrix[i].features=SG_MALLOC(SGSparseVectorEntry, len); \ - \ - for (int32_t j=0; j*& strings, int32_t& num_str, int32_t& max_string_len) \ -{ \ - const mxArray* mx_str=get_arg_increment(); \ - if (!mx_str) \ - SG_ERROR("Expected Stringlist as argument (none given).\n"); \ - \ - if (mxIsCell(mx_str)) \ - { \ - num_str=mxGetNumberOfElements(mx_str); \ - ASSERT(num_str>=1); \ - \ - strings=SG_MALLOC(SGString, num_str); \ - for (int32_t i=0; i0) \ - { \ - if_type* data=(if_type*) mxGetData(str); \ - strings[i].slen=len; /* all must have same length in matlab */ \ - strings[i].string=SG_MALLOC(sg_type, len+1); /* not zero terminated in matlab */ \ - int32_t j; \ - for (j=0; j, num_str); \ - \ - for (int32_t i=0; i0) \ - { \ - strings[i].slen=len; /* all must have same length in matlab */ \ - strings[i].string=SG_MALLOC(sg_type, len+1); /* not zero terminated in matlab */ \ - int32_t j; \ - for (j=0; j* &attrs) -{ - attrs=NULL; -} - -/** set functions - to pass data from shogun to the target interface */ - -void CMatlabInterface::set_int(int32_t scalar) -{ - mxArray* o=mxCreateNumericMatrix(1, 1, mxINT32_CLASS, mxREAL); - if (!o) - SG_ERROR("Couldn't create Integer.\n"); - - int* data=(int*) mxGetData(o); - data[0]=scalar; - - set_arg_increment(o); -} - -void CMatlabInterface::set_real(float64_t scalar) -{ - mxArray* o=mxCreateNumericMatrix(1, 1, mxDOUBLE_CLASS, mxREAL); - if (!o) - SG_ERROR("Couldn't create Double.\n"); - - double* data=(double*) mxGetData(o); - data[0]=scalar; - - set_arg_increment(o); -} - -void CMatlabInterface::set_bool(bool scalar) -{ - mxArray* o=mxCreateLogicalMatrix(1, 1); - if (!o) - SG_ERROR("Couldn't create Logical.\n"); - - bool* data=(bool*) mxGetData(o); - data[0]=scalar; - - set_arg_increment(o); -} - - -#define SET_VECTOR(function_name, mx_type, sg_type, if_type, error_string) \ -void CMatlabInterface::function_name(const sg_type* vector, int32_t len) \ -{ \ - if (!vector) \ - SG_ERROR("Given vector is invalid.\n"); \ - \ - mxArray* mx_vec=mxCreateNumericMatrix(1, len, mx_type, mxREAL); \ - if (!mx_vec) \ - SG_ERROR("Couldn't create " error_string " Vector of length %d.\n", len); \ - \ - if_type* data=(if_type*) mxGetData(mx_vec); \ - \ - for (int32_t i=0; i* matrix, int32_t num_feat, int32_t num_vec, int64_t nnz) \ -{ \ - if (!matrix) \ - SG_ERROR("Given matrix is invalid.\n"); \ - \ - mxArray* mx_mat=mxCreateSparse(num_feat, num_vec, nnz, mxREAL); \ - if (!mx_mat) \ - SG_ERROR("Couldn't create Sparse Matrix of %d rows and %d cols.\n", num_feat, num_vec); \ - \ - if_type* data=(if_type*) mxGetData(mx_mat); \ - \ - mwIndex* ir=mxGetIr(mx_mat); \ - mwIndex* jc=mxGetJc(mx_mat); \ - int64_t offset=0; \ - for (int32_t i=0; i* strings, int32_t num_str) \ -{ \ - if (!strings) \ - SG_ERROR("Given strings are invalid.\n"); \ - \ - mxArray* mx_str= mxCreateCellMatrix(num_str, 1); \ - if (!mx_str) \ - SG_ERROR("Couldn't create Cell Array of %d strings.\n", num_str); \ - \ - for (int32_t i=0; i0) \ - { \ - mxArray* str=mxCreateNumericMatrix(1, len, mx_type, mxREAL); \ - if (!str) \ - SG_ERROR("Couldn't create " error_string " String %d of length %d.\n", i, len); \ - \ - if_type* data=(if_type*) mxGetData(str); \ - \ - for (int32_t j=0; j* attrs) -{ -} - -//////////////////////////////////////////////////////////////////// - -const mxArray* CMatlabInterface::get_arg_increment() -{ - const mxArray* retval; - ASSERT(m_rhs_counter>=0 && m_rhs_counter=0 && m_lhs_counterreset(nlhs, plhs, nrhs, prhs); - - if (!interface->handle()) - SG_SERROR("Unknown command.\n"); - } - catch (std::bad_alloc) - { - mexErrMsgTxt("Out of memory error."); - } - catch (ShogunException e) - { - mexErrMsgTxt(e.get_exception_string()); - } - catch (...) - { - mexErrMsgTxt("Returning from SHOGUN in error."); - } -} - -/* to be run on exiting matlab ... does not seem to be possible right now: - * run_octave_exit() - * run_python_exit() - * run_r_exit() - */ diff --git a/src/interfaces/matlab_static/MatlabInterface.h b/src/interfaces/matlab_static/MatlabInterface.h deleted file mode 100644 index 08c1a5c497e..00000000000 --- a/src/interfaces/matlab_static/MatlabInterface.h +++ /dev/null @@ -1,188 +0,0 @@ -#ifndef __MATLABINTERFACE__H_ -#define __MATLABINTERFACE__H_ - -#include - -//use compatibility mode w/ matlab <7.x -#if !defined(MX_API_VER) || MX_API_VER<0x07040000 -#define mwSize int32_t -#define mwIndex int32_t - -#define mxIsLogicalScalar(x) false -#define mxIsLogicalScalarTrue(x) false -#endif - -#include -#include -#include - -using namespace shogun; - -class CMatlabInterface : public CSGInterface -{ - public: - CMatlabInterface( - int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]); - ~CMatlabInterface(); - - /// reset to clean state - virtual void reset( - int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]); - - /** get functions - to pass data from the target interface to shogun */ - - /// get type of current argument (does not increment argument counter) - virtual IFType get_argument_type(); - - virtual int32_t get_int(); - virtual float64_t get_real(); - virtual bool get_bool(); - - virtual char* get_string(int32_t& len); - - virtual void get_vector(uint8_t*& vector, int32_t& len); - virtual void get_vector(char*& vector, int32_t& len); - virtual void get_vector(int32_t*& vector, int32_t& len); - virtual void get_vector(float32_t*& vector, int32_t& len); - virtual void get_vector(float64_t*& vector, int32_t& len); - virtual void get_vector(int16_t*& vector, int32_t& len); - virtual void get_vector(uint16_t*& vector, int32_t& len); - - virtual void get_matrix( - uint8_t*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - char*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - int32_t*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - float32_t*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - float64_t*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - int16_t*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - uint16_t*& matrix, int32_t& num_feat, int32_t& num_vec); - - virtual void get_ndarray( - uint8_t*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - char*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - int32_t*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - float32_t*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - float64_t*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - int16_t*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - uint16_t*& array, int32_t*& dims, int32_t& num_dims); - - virtual void get_sparse_matrix( - SGSparseVector*& matrix, int32_t& num_feat, int32_t& num_vec); - - /* future versions might support types other than float64_t - - virtual void get_sparse_matrix(SGSparseVector*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_sparse_matrix(SGSparseVector*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_sparse_matrix(SGSparseVector*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_sparse_matrix(SGSparseVector*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_sparse_matrix(SGSparseVector*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_sparse_matrix(SGSparseVector*& matrix, int32_t& num_feat, int32_t& num_vec);*/ - - virtual void get_string_list( - SGString*& strings, int32_t& num_str, - int32_t& max_string_len); - virtual void get_string_list( - SGString*& strings, int32_t& num_str, - int32_t& max_string_len); - virtual void get_string_list( - SGString*& strings, int32_t& num_str, - int32_t& max_string_len); - virtual void get_string_list( - SGString*& strings, int32_t& num_str, - int32_t& max_string_len); - virtual void get_string_list( - SGString*& strings, int32_t& num_str, - int32_t& max_string_len); - - virtual void get_attribute_struct( - const CDynamicArray* &attrs); - - /** set functions - to pass data from shogun to the target interface */ - virtual void set_int(int32_t scalar); - virtual void set_real(float64_t scalar); - virtual void set_bool(bool scalar); - - virtual void set_vector(const uint8_t* vec, int32_t len); - virtual void set_vector(const char* vec, int32_t len); - virtual void set_vector(const int32_t* vec, int32_t len); - virtual void set_vector(const float32_t* vec, int32_t len); - virtual void set_vector(const float64_t* vec, int32_t len); - virtual void set_vector(const int16_t* vec, int32_t len); - virtual void set_vector(const uint16_t* vec, int32_t len); - - virtual void set_matrix( - const uint8_t* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const char* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const int32_t* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const float32_t* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const float64_t* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const int16_t* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const uint16_t* matrix, int32_t num_feat, int32_t num_vec); - - virtual void set_sparse_matrix( - const SGSparseVector* matrix, int32_t num_feat, - int32_t num_vec, int64_t nnz); - - /* future versions might support types other than float64_t - - virtual void set_sparse_matrix(const SGSparseVector* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_sparse_matrix(const SGSparseVector* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_sparse_matrix(const SGSparseVector* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_sparse_matrix(const SGSparseVector* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_sparse_matrix(const SGSparseVector* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_sparse_matrix(const SGSparseVector* matrix, int32_t num_feat, int32_t num_vec);*/ - - virtual void set_string_list( - const SGString* strings, int32_t num_str); - virtual void set_string_list( - const SGString* strings, int32_t num_str); - virtual void set_string_list( - const SGString* strings, int32_t num_str); - virtual void set_string_list( - const SGString* strings, int32_t num_str); - virtual void set_string_list( - const SGString* strings, int32_t num_str); - - virtual void set_attribute_struct( - const CDynamicArray* attrs); - - virtual bool create_return_values(int32_t num) - { - return m_nlhs==num; - } - - /** @return object name */ - inline virtual const char* get_name() const { return "MatlabInterface"; } - - virtual bool cmd_run_python(); - virtual bool cmd_run_octave(); - virtual bool cmd_run_r(); - - private: - const mxArray* get_arg_increment(); - void set_arg_increment(mxArray* arg); - - private: - mxArray** m_lhs; - const mxArray** m_rhs; - -}; -#endif // __MATLABINTERFACE__H_ diff --git a/src/interfaces/octave_static/CMakeLists.txt b/src/interfaces/octave_static/CMakeLists.txt deleted file mode 100644 index e408da56e44..00000000000 --- a/src/interfaces/octave_static/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -include_directories(${OCTAVE_INCLUDE_DIRS} ${SYSTEM_INCLUDES} ${INCLUDES}) - -add_library(octave_static SHARED OctaveInterface.cpp) -target_link_libraries(octave_static shogun ${OCTAVE_LIBRARY}) - -# Add static-interface to the dependencies of modular interfaces to make -# sure nothing will infer with them being build single-threaded. -IF(SWIG_SINGLE_THREADED) - FOREACH(SG_MODULAR_INTERFACE_TARGET ${SG_MODULAR_INTERFACE_TARGETS}) - ADD_DEPENDENCIES(${SG_MODULAR_INTERFACE_TARGET} octave_static) - ENDFOREACH(SG_MODULAR_INTERFACE_TARGET ${SG_MODULAR_INTERFACE_TARGETS}) -ENDIF(SWIG_SINGLE_THREADED) diff --git a/src/interfaces/octave_static/OctaveInterface.cpp b/src/interfaces/octave_static/OctaveInterface.cpp deleted file mode 100644 index 830b20e9efa..00000000000 --- a/src/interfaces/octave_static/OctaveInterface.cpp +++ /dev/null @@ -1,842 +0,0 @@ -#include "OctaveInterface.h" - -#undef length - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#ifdef HAVE_PYTHON -#include "../python_static/PythonInterface.h" -#endif - -#ifdef HAVE_R -#include "../r_static/RInterface.h" -#undef length -#endif - -void octave_print_message(FILE* target, const char* str) -{ - fprintf(target, "%s", str); -} - -void octave_print_warning(FILE* target, const char* str) -{ - if (target==stdout) - ::warning(str); - else - fprintf(target, "%s", str); -} - -void octave_print_error(FILE* target, const char* str) -{ - if (target!=stdout) - fprintf(target, "%s", str); -} - -void octave_cancel_computations(bool &delayed, bool &immediately) -{ -} - -extern CSGInterface* interface; - -COctaveInterface::COctaveInterface(octave_value_list prhs, int32_t nlhs, bool verbose) -: CSGInterface(verbose) -{ - reset(prhs, nlhs); -} - -COctaveInterface::~COctaveInterface() -{ -} - -void COctaveInterface::reset(octave_value_list prhs, int32_t nlhs) -{ - CSGInterface::reset(); - - m_nlhs=nlhs; - m_nrhs=prhs.length(); - m_lhs=octave_value_list(); - m_rhs=prhs; -} - -/** get functions - to pass data from the target interface to shogun */ - - -/// get type of current argument (does not increment argument counter) -IFType COctaveInterface::get_argument_type() -{ - octave_value arg=m_rhs(m_rhs_counter); - - if (arg.is_real_scalar()) - return SCALAR_REAL; - if (arg.is_bool_scalar()) - return SCALAR_BOOL; - - if (arg.is_char_matrix()) - return STRING_CHAR; - if (arg.is_uint8_type() && arg.is_matrix_type()) - return STRING_BYTE; - - if (arg.is_sparse_type()) - { - if (arg.is_uint8_type()) - return SPARSE_BYTE; - if (arg.is_char_matrix()) - return SPARSE_CHAR; - if (arg.is_int32_type()) - return SPARSE_INT; - if (arg.is_double_type()) - return SPARSE_REAL; - if (arg.is_int16_type()) - return SPARSE_SHORT; - if (arg.is_single_type()) - return SPARSE_SHORTREAL; - if (arg.is_uint16_type()) - return SPARSE_WORD; - - return UNDEFINED; - } - - if (arg.is_cell()) - { - Cell c = arg.cell_value(); - - if (c.nelem()>0) - { - if (c.elem(0).is_char_matrix() && c.elem(0).rows()==1) - return STRING_CHAR; - if (c.elem(0).is_uint8_type() && c.elem(0).rows()==1) - return STRING_BYTE; - if (c.elem(0).is_int32_type() && c.elem(0).rows()==1) - return STRING_INT; - if (c.elem(0).is_int16_type() && c.elem(0).rows()==1) - return STRING_SHORT; - if (c.elem(0).is_uint16_type() && c.elem(0).rows()==1) - return STRING_WORD; - } - } - - - if (arg.is_matrix_type() && arg.ndims()==1 && arg.rows()==1) - { - if (arg.is_uint32_type()) - return VECTOR_INT; - if (arg.is_double_type()) - return VECTOR_REAL; - if (arg.is_int16_type()) - return VECTOR_SHORT; - if (arg.is_single_type()) - return VECTOR_SHORTREAL; - if (arg.is_uint16_type()) - return VECTOR_WORD; - - return UNDEFINED; - } - - if (arg.is_matrix_type() && arg.ndims()==2) - { - if (arg.is_uint32_type()) - return DENSE_INT; - if (arg.is_double_type()) - return DENSE_REAL; - if (arg.is_int16_type()) - return DENSE_SHORT; - if (arg.is_single_type()) - return DENSE_SHORTREAL; - if (arg.is_uint16_type()) - return DENSE_WORD; - - return UNDEFINED; - } - - if (arg.is_matrix_type() && arg.ndims()>2) - { - if (arg.is_uint8_type()) - return NDARRAY_BYTE; - if (arg.is_uint32_type()) - return NDARRAY_INT; - if (arg.is_double_type()) - return NDARRAY_REAL; - if (arg.is_int16_type()) - return NDARRAY_SHORT; - if (arg.is_single_type()) - return NDARRAY_SHORTREAL; - if (arg.is_uint16_type()) - return NDARRAY_WORD; - - return UNDEFINED; - } - - if (arg.is_map()) - return ATTR_STRUCT; - - return UNDEFINED; -} - - -int32_t COctaveInterface::get_int() -{ - const octave_value i=get_arg_increment(); - if (!i.is_real_scalar()) - SG_ERROR("Expected Scalar Integer as argument %d\n", m_rhs_counter); - - double s=i.double_value(); - if (s-CMath::floor(s)!=0) - SG_ERROR("Expected Integer as argument %d\n", m_rhs_counter); - - return int32_t(s); -} - -float64_t COctaveInterface::get_real() -{ - const octave_value f=get_arg_increment(); - if (!f.is_real_scalar()) - SG_ERROR("Expected Scalar Float as argument %d\n", m_rhs_counter); - - return f.double_value(); -} - -bool COctaveInterface::get_bool() -{ - const octave_value b=get_arg_increment(); - if (b.is_bool_scalar()) - return b.bool_value(); - else if (b.is_real_scalar()) - return (b.double_value()!=0); - else - SG_ERROR("Expected Scalar Boolean as argument %d\n", m_rhs_counter); - - return false; -} - -char* COctaveInterface::get_string(int32_t& len) -{ - const octave_value s=get_arg_increment(); - if (!s.is_string()) - SG_ERROR("Expected String as argument %d\n", m_rhs_counter); - - std::string std_str=s.string_value(); - const char* str= std_str.c_str(); - len=std_str.length(); - ASSERT(str && len>0); - - char* cstr=SG_MALLOC(char, len+1); - memcpy(cstr, str, len+1); - cstr[len]='\0'; - - return cstr; -} - -#define GET_VECTOR(function_name, oct_type_check, oct_type, oct_converter, sg_type, if_type, error_string) \ -void COctaveInterface::function_name(sg_type*& vec, int32_t& len) \ -{ \ - const octave_value mat_feat=get_arg_increment(); \ - if (!mat_feat.oct_type_check()) \ - SG_ERROR("Expected " error_string " Vector as argument %d\n", m_rhs_counter); \ - \ - oct_type m = mat_feat.oct_converter(); \ - \ - if (m.rows()!=1) \ - SG_ERROR("Expected " error_string " (1xN) Vector as argument %d, got vector " \ - "of shape (%dx%d)\n", m_rhs_counter, m.rows(), m.cols()); \ - \ - len = m.cols(); \ - vec=SG_MALLOC(sg_type, len); \ - \ - for (int32_t i=0; i*& matrix, int32_t& num_feat, int32_t& num_vec) -{ - const octave_value mat_feat=get_arg_increment(); - if (!mat_feat.is_sparse_type() || !(mat_feat.is_double_type())) - SG_ERROR("Expected Sparse Double Matrix as argument %d\n", m_rhs_counter); - - SparseMatrix sm = mat_feat.sparse_matrix_value (); - num_vec=sm.cols(); - num_feat=sm.rows(); - int64_t nnz=sm.nelem(); - - matrix=SG_MALLOC(SGSparseVector, num_vec); - - int64_t offset=0; - for (int32_t i=0; i0) - { - matrix[i].features=SG_MALLOC(SGSparseVectorEntry, len); - - for (int32_t j=0; j*& strings, int32_t& num_str, int32_t& max_string_len) \ -{ \ - max_string_len=0; \ - octave_value arg=get_arg_increment(); \ - if (arg.is_cell()) \ - { \ - Cell c = arg.cell_value(); \ - num_str=c.nelem(); \ - ASSERT(num_str>=1); \ - strings=SG_MALLOC(SGString, num_str); \ - \ - for (int32_t i=0; i0) \ - { \ - strings[i].slen=len; /* all must have same length in octave */ \ - strings[i].string=SG_MALLOC(sg_type, len+1); /* not zero terminated in octave */ \ - int32_t j; \ - for (j=0; j, num_str); \ - \ - for (int32_t i=0; i0) \ - { \ - strings[i].slen=len; /* all must have same length in octave */ \ - strings[i].string=SG_MALLOC(sg_type, len+1); /* not zero terminated in octave */ \ - int32_t j; \ - for (j=0; j* &attrs) -{ - attrs=NULL; -} - -/** set functions - to pass data from shogun to Octave */ - -void COctaveInterface::set_int(int32_t scalar) -{ - octave_value o(scalar); - set_arg_increment(o); -} - -void COctaveInterface::set_real(float64_t scalar) -{ - octave_value o(scalar); - set_arg_increment(o); -} - -void COctaveInterface::set_bool(bool scalar) -{ - octave_value o(scalar); - set_arg_increment(o); -} - - -#define SET_VECTOR(function_name, oct_type, sg_type, if_type, error_string) \ -void COctaveInterface::function_name(const sg_type* vec, int32_t len) \ -{ \ - oct_type mat=oct_type(dim_vector(1, len)); \ - \ - for (int32_t i=0; i* matrix, int32_t num_feat, int32_t num_vec, int64_t nnz) -{ - SparseMatrix sm((octave_idx_type) num_feat, (octave_idx_type) num_vec, (octave_idx_type) nnz); - - int64_t offset=0; - for (int32_t i=0; i* strings, int32_t num_str) \ -{ \ - if (!strings) \ - SG_ERROR("Given strings are invalid.\n") \ - \ - Cell c= Cell(dim_vector(1,num_str)); \ - if (c.nelem()!=num_str) \ - SG_ERROR("Couldn't create Cell Array of %d strings.\n", num_str) \ - \ - for (int32_t i=0; i0) \ - { \ - oct_type str(dim_vector(1,len)); \ - if (str.cols()!=len) \ - SG_ERROR("Couldn't create " error_string " String %d of length %d.\n", i, len) \ - \ - for (int32_t j=0; j* attrs) -{ -/* octave_value arg=get_arg_increment(); - if (!arg.is_map()) - SG_ERROR("not a struct"); - attrs = new CDynamicArray();*/ -} - -bool COctaveInterface::cmd_run_python() -{ -#ifdef HAVE_PYTHON - return CPythonInterface::run_python_helper(this); -#else - return false; -#endif -} - -bool COctaveInterface::cmd_run_r() -{ -#ifdef HAVE_R - return CRInterface::run_r_helper(this); -#else - return false; -#endif -} - -void COctaveInterface::recover_from_exception(void) -{ -#if defined(OCTAVE_APIVERSION) && OCTAVE_APIVERSION < 37 - unwind_protect::run_all (); -#endif - can_interrupt = true; - octave_interrupt_immediately = 0; - octave_interrupt_state = 0; - -#if !defined(OCTAVE_APIVERSION) || OCTAVE_APIVERSION >= 37 - octave_exception_state = octave_no_exception; -#else - octave_allocation_error = 0; -#endif - octave_restore_signal_mask (); - octave_catch_interrupts (); -} - -void COctaveInterface::clear_octave_globals() -{ - //string_vector gvars = symbol_table::global_variable_names(); - - //int gcount = gvars.length(); - - //for (int i = 0; i < gcount; i++) - // symbol_table::clear_global(gvars[i]); - int parse_status; - eval_string("clear all", false, parse_status); - // symbol_table::clear_global_pattern ("*"); - //global_sym_tab->clear(); -} - -void COctaveInterface::run_octave_init() -{ - char* name=get_strdup("octave"); - char* opts=get_strdup("-q"); - char* argv[2]={name, opts}; - octave_main(2,argv,1); - free(opts); - free(name); -} - -void COctaveInterface::run_octave_exit() -{ -#if defined(OCTAVE_MAJOR_VERSION) && OCTAVE_MAJOR_VERSION >= 3 && defined(OCTAVE_MINOR_VERSION) && OCTAVE_MINOR_VERSION >= 8 - clean_up_and_exit (0); -#else - do_octave_atexit(); -#endif -} - -bool COctaveInterface::run_octave_helper(CSGInterface* from_if) -{ - SG_OBJ_DEBUG(from_if, "Entering Octave\n"); - octave_save_signal_mask (); - - if (octave_set_current_context) - { -#if defined (USE_EXCEPTIONS_FOR_INTERRUPTS) - panic_impossible (); -#else -#if defined(OCTAVE_APIVERSION) && OCTAVE_APIVERSION < 37 - unwind_protect::run_all (); -#endif - raw_mode (0); - octave_restore_signal_mask (); -#endif - } - - can_interrupt = true; - octave_catch_interrupts (); - octave_initialized = true; - - try - { - int parse_status; - char* octave_code=NULL; - clear_octave_globals(); - - for (int i=0; iget_nrhs(); i++) - { - int len=0; - char* var_name = from_if->get_string(len); - SG_OBJ_DEBUG(from_if, "var_name = '%s'\n", var_name); - if (strmatch(var_name, "octavecode")) - { - len=0; - octave_code=from_if->get_string(len); - SG_OBJ_DEBUG(from_if, "octave_code = '%s'\n", octave_code); - break; - } - else - { - octave_value_list args; - - COctaveInterface* in = new COctaveInterface(args, 1, false); - in->create_return_values(1); - from_if->translate_arg(from_if, in); -#if !defined(OCTAVE_APIVERSION) || OCTAVE_APIVERSION >= 37 - symbol_table::varref (var_name) = in->get_return_values()(0); -#else - set_global_value(var_name, in->get_return_values()(0)); -#endif - SG_FREE(var_name); - SG_UNREF(in); - } - } - -#if !defined(OCTAVE_APIVERSION) || OCTAVE_APIVERSION >= 37 -#else - symbol_table* old=curr_sym_tab; - curr_sym_tab = global_sym_tab; -#endif - reset_error_handler (); - eval_string(octave_code, false, parse_status); - SG_FREE(octave_code); - - int32_t sz=0; - octave_value_list results; - -#if !defined(OCTAVE_APIVERSION) || OCTAVE_APIVERSION >= 37 - if (symbol_table::is_variable("results")) - { - results = symbol_table::varval("results"); - //results = get_global_value("results", false); - sz=results.length(); - } -#else - if (curr_sym_tab->lookup("results")) - { - results = get_global_value("results", false); - sz=results.length(); - } -#endif - - if (sz>0) - { - if (results(0).is_cs_list()) - { - SG_OBJ_DEBUG(from_if, "Found return list of length %d\n", results(0).length()); - results=results(0).list_value(); - sz=results.length(); - } - } - - if (sz>0 && from_if->create_return_values(sz)) - { - SG_OBJ_DEBUG(from_if, "Found %d args\n", sz); - COctaveInterface* out = new COctaveInterface(results, sz, false); - - //process d - for (int32_t i=0; itranslate_arg(out, from_if); - - SG_UNREF(out); - } - else - { - if (sz!=from_if->get_nlhs()) - { - SG_OBJ_ERROR(from_if, "Number of return values (%d) does not match number of expected" - " return values (%d).\n", sz, from_if->get_nlhs()); - } - } - -#if !defined(OCTAVE_APIVERSION) || OCTAVE_APIVERSION >= 37 -#else - curr_sym_tab=old; -#endif - } - catch (octave_interrupt_exception) - { - recover_from_exception (); - SG_SPRINT("%\n"); - } - catch (std::bad_alloc) - { - recover_from_exception (); - SG_SPRINT("%\n"); - } - - octave_restore_signal_mask(); - octave_initialized = false; - - SG_OBJ_DEBUG(from_if, "Leaving Octave.\n"); - return true; -} - -#ifdef HAVE_ELWMS -DEFUN_DLD (elwms, prhs, nlhs, "shogun.") -#else -DEFUN_DLD (sg, prhs, nlhs, "shogun.") -#endif -{ - try - { - if (!interface) - { - // init_shogun has to be called before anything else - // exit_shogun is called upon destruction of the interface (see - // destructor of COctaveInterface - init_shogun(&octave_print_message, &octave_print_warning, - &octave_print_error, &octave_cancel_computations); - interface=new COctaveInterface(prhs, nlhs); -#ifdef HAVE_PYTHON - CPythonInterface::run_python_init(); -#endif -#ifdef HAVE_R - CRInterface::run_r_init(); -#endif - } - else - ((COctaveInterface*) interface)->reset(prhs, nlhs); - - if (!interface->handle()) - SG_SERROR("Unknown command.\n"); - - return ((COctaveInterface*) interface)->get_return_values(); - } - catch (std::bad_alloc) - { - SG_SPRINT("Out of memory error.\n"); - return octave_value_list(); - } - catch (ShogunException e) - { - error("%s", e.get_exception_string()); - return octave_value_list(); - } - catch (...) - { - error("%s", "Returning from SHOGUN in error."); - return octave_value_list(); - } -} - -/* to be run on exiting matlab ... does not seem to be possible right now: - * run_octave_exit() - * run_python_exit() - * run_r_exit() - */ diff --git a/src/interfaces/octave_static/OctaveInterface.h b/src/interfaces/octave_static/OctaveInterface.h deleted file mode 100644 index 2290415fc20..00000000000 --- a/src/interfaces/octave_static/OctaveInterface.h +++ /dev/null @@ -1,208 +0,0 @@ -#ifndef __OCTAVEINTERFACE__H_ -#define __OCTAVEINTERFACE__H_ - -#include -#include - -#include - -#undef HAVE_STAT -#include - -#ifdef HAVE_MATLAB -#define MXARRAY_H -typedef struct mxArray_tag mxArray; -#endif - -#include - -using namespace shogun; - -class COctaveInterface : public CSGInterface -{ - public: - COctaveInterface(octave_value_list prhs, int32_t nlhs, bool verbose=true); - ~COctaveInterface(); - - /// reset to clean state - virtual void reset(octave_value_list prhs, int32_t nlhs); - - /** get functions - to pass data from the target interface to shogun */ - - /// get type of current argument (does not increment argument counter) - virtual IFType get_argument_type(); - - virtual int32_t get_int(); - virtual float64_t get_real(); - virtual bool get_bool(); - - virtual char* get_string(int32_t& len); - - virtual void get_vector(uint8_t*& vec, int32_t& len); - virtual void get_vector(char*& vec, int32_t& len); - virtual void get_vector(int32_t*& vec, int32_t& len); - virtual void get_vector(float32_t*& vec, int32_t& len); - virtual void get_vector(float64_t*& vec, int32_t& len); - virtual void get_vector(int16_t*& vec, int32_t& len); - virtual void get_vector(uint16_t*& vec, int32_t& len); - - virtual void get_matrix( - uint8_t*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - char*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - int32_t*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - float32_t*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - float64_t*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - int16_t*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - uint16_t*& matrix, int32_t& num_feat, int32_t& num_vec); - - virtual void get_ndarray( - uint8_t*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - char*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - int32_t*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - float32_t*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - float64_t*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - int16_t*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - uint16_t*& array, int32_t*& dims, int32_t& num_dims); - - virtual void get_sparse_matrix( - SGSparseVector*& matrix, int32_t& num_feat, int32_t& num_vec); - /*virtual void get_sparse_matrix(SGSparseVector*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_sparse_matrix(SGSparseVector*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_sparse_matrix(SGSparseVector*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_sparse_matrix(SGSparseVector*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_sparse_matrix(SGSparseVector*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_sparse_matrix(SGSparseVector*& matrix, int32_t& num_feat, int32_t& num_vec);*/ - - virtual void get_string_list( - SGString*& strings, int32_t& num_str, - int32_t& max_string_len); - virtual void get_string_list( - SGString*& strings, int32_t& num_str, - int32_t& max_string_len); - virtual void get_string_list( - SGString*& strings, int32_t& num_str, - int32_t& max_string_len); - virtual void get_string_list( - SGString*& strings, int32_t& num_str, - int32_t& max_string_len); - virtual void get_string_list( - SGString*& strings, int32_t& num_str, - int32_t& max_string_len); - - virtual void get_attribute_struct( - const CDynamicArray* &attrs); - - /** set functions - to pass data from shogun to the target interface */ - - virtual void set_int(int32_t scalar); - virtual void set_real(float64_t scalar); - virtual void set_bool(bool scalar); - - virtual void set_vector(const uint8_t* vec, int32_t len); - virtual void set_vector(const char* vec, int32_t len); - virtual void set_vector(const int32_t* vec, int32_t len); - virtual void set_vector(const float32_t* vec, int32_t len); - virtual void set_vector(const float64_t* vec, int32_t len); - virtual void set_vector(const int16_t* vec, int32_t len); - virtual void set_vector(const uint16_t* vec, int32_t len); - - virtual void set_matrix( - const uint8_t* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const char* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const int32_t* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const float32_t* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const float64_t* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const int16_t* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const uint16_t* matrix, int32_t num_feat, int32_t num_vec); - - virtual void set_sparse_matrix( - const SGSparseVector* matrix, int32_t num_feat, - int32_t num_vec, int64_t nnz); - /* - virtual void set_sparse_matrix(const SGSparseVector* matrix, int32_t num_feat, int32_t num_vec, int64_t nnz); - virtual void set_sparse_matrix(const SGSparseVector* matrix, int32_t num_feat, int32_t num_vec, int64_t nnz); - virtual void set_sparse_matrix(const SGSparseVector* matrix, int32_t num_feat, int32_t num_vec, int64_t nnz); - virtual void set_sparse_matrix(const SGSparseVector* matrix, int32_t num_feat, int32_t num_vec, int64_t nnz); - virtual void set_sparse_matrix(const SGSparseVector* matrix, int32_t num_feat, int32_t num_vec, int64_t nnz); - virtual void set_sparse_matrix(const SGSparseVector* matrix, int32_t num_feat, int32_t num_vec, int64_t nnz);*/ - - void set_string_list( - const SGString* strings, int32_t num_str); - void set_string_list( - const SGString* strings, int32_t num_str); - void set_string_list( - const SGString* strings, int32_t num_str); - void set_string_list( - const SGString* strings, int32_t num_str); - void set_string_list( - const SGString* strings, int32_t num_str); - - virtual void set_attribute_struct( - const CDynamicArray* attrs); - - virtual bool create_return_values(int32_t num) - { - return m_nlhs==num; - } - - inline octave_value_list get_return_values() - { - return m_lhs; - } - - virtual bool cmd_run_python(); - virtual bool cmd_run_r(); - static void run_octave_init(); - static void run_octave_exit(); - static bool run_octave_helper(CSGInterface* from_if); - static void recover_from_exception(); - - private: - - static void clear_octave_globals(); - - const octave_value get_arg_increment() - { - octave_value retval; - ASSERT(m_rhs_counter>=0 && m_rhs_counter=0 && m_lhs_counter -#include - -#include -#include -#include -#include - -#ifdef HAVE_OCTAVE -#include "../octave_static/OctaveInterface.h" -#endif - -#ifdef HAVE_R -#include "../r_static/RInterface.h" -#endif - -using namespace shogun; - -void* CPythonInterface::m_pylib=0; - -void python_print_message(FILE* target, const char* str) -{ - fprintf(target, "%s", str); -} - -void python_print_warning(FILE* target, const char* str) -{ - if (target==stdout) - PyErr_Warn(NULL, (char*) str); //the cast seems to be necessary for python 2.4.3 - else - fprintf(target, "%s", str); -} - -void python_print_error(FILE* target, const char* str) -{ - if (target==stdout) - PyErr_SetString(PyExc_RuntimeError, str); - else - fprintf(target, "%s", str); -} - -void python_cancel_computations(bool &delayed, bool &immediately) -{ - if (PyErr_CheckSignals()) - { - SG_SPRINT("\nImmediately return to matlab prompt / Prematurely finish computations / Do nothing (I/P/D)? "); - char answer=fgetc(stdin); - - if (answer == 'I') - immediately=true; - else if (answer == 'P') - { - PyErr_Clear(); - delayed=true; - } - else - SG_SPRINT("\n"); - } -} - -extern CSGInterface* interface; - -CPythonInterface::CPythonInterface(PyObject* args) -: CSGInterface(false) -{ - reset(NULL, args); -} - -CPythonInterface::CPythonInterface(PyObject* self, PyObject* args) -: CSGInterface() -{ - reset(self, args); -} - -CPythonInterface::~CPythonInterface() -{ -} - -void CPythonInterface::reset(PyObject* self, PyObject* args) -{ - CSGInterface::reset(); - - ASSERT(PyTuple_Check(args)); - m_rhs=args; - m_nrhs=PyTuple_GET_SIZE(args); - - m_nlhs=0; - Py_INCREF(Py_None); - m_lhs=Py_None; -} - - -/** get functions - to pass data from the target interface to shogun */ - - -/// get type of current argument (does not increment argument counter) -IFType CPythonInterface::get_argument_type() -{ - PyObject* arg= PyTuple_GetItem(m_rhs, m_rhs_counter); - ASSERT(arg); - - if (PyList_Check(arg) && PyList_Size((PyObject *) arg)>0) - { - PyObject* item= PyList_GetItem((PyObject *) arg, 0); - -#ifdef IS_PYTHON3 - if (PyUnicode_Check(item)) -#else - if (PyString_Check(item)) -#endif - { - return STRING_CHAR; - } - } - else if PyArray_Check(arg) - { - if (PyArray_TYPE(arg)==NPY_CHAR) - return STRING_CHAR; - if (PyArray_TYPE(arg)==NPY_BYTE) - return STRING_BYTE; - if (PyArray_TYPE(arg)==NPY_INT) - return DENSE_INT; - if (PyArray_TYPE(arg)==NPY_DOUBLE) - return DENSE_REAL; - if (PyArray_TYPE(arg)==NPY_SHORT) - return DENSE_SHORT; - if (PyArray_TYPE(arg)==NPY_FLOAT) - return DENSE_SHORTREAL; - if (PyArray_TYPE(arg)==NPY_USHORT) - return DENSE_WORD; - } - return UNDEFINED; -} - - -int32_t CPythonInterface::get_int() -{ - const PyObject* i = get_arg_increment(); - if (!i || !PyInt_Check(i)) - SG_ERROR("Expected Scalar Integer as argument %d\n", m_rhs_counter); - - return PyInt_AS_LONG(const_cast(i)); -} - -float64_t CPythonInterface::get_real() -{ - const PyObject* f=get_arg_increment(); - if (!f || !PyFloat_Check(f)) - SG_ERROR("Expected Scalar Float as argument %d\n", m_rhs_counter); - - return PyFloat_AS_DOUBLE(f); -} - -bool CPythonInterface::get_bool() -{ - const PyObject* b=get_arg_increment(); - if (!b || !PyBool_Check(b)) - SG_ERROR("Expected Scalar Boolean as argument %d\n", m_rhs_counter); - - return PyInt_AS_LONG(const_cast(b)) != 0; -} - -char* CPythonInterface::get_string(int32_t& len) -{ - const PyObject* s=get_arg_increment(); - -#ifdef IS_PYTHON3 - if (!s || !PyUnicode_Check(s)) - SG_ERROR("Expected String as argument %d\n", m_rhs_counter); - - len = PyUnicode_GetSize((PyObject*) s); - char* str = PyBytes_AsString(PyUnicode_AsASCIIString(const_cast(s))); -#else - if (!s || !PyString_Check(s)) - SG_ERROR("Expected String as argument %d\n", m_rhs_counter); - - len = PyString_Size((PyObject*) s); - char* str = PyString_AS_STRING(s); - ASSERT(str && len>0); -#endif - - ASSERT(str && len>0); - - char* cstr=SG_MALLOC(char, len+1); - memcpy(cstr, str, len+1); - cstr[len]='\0'; - - return cstr; -} - -#define GET_VECTOR(function_name, py_type, sg_type, if_type, error_string) \ -void CPythonInterface::function_name(sg_type*& vector, int32_t& len) \ -{ \ - const PyArrayObject* py_vec=(PyArrayObject *) get_arg_increment(); \ - if (!py_vec || !PyArray_Check(py_vec) || py_vec->nd!=1 || \ - PyArray_TYPE(py_vec)!=py_type) \ - { \ - SG_ERROR("Expected " error_string " Vector as argument %d\n", \ - m_rhs_counter); \ - } \ - \ - len=py_vec->dimensions[0]; \ - npy_intp stride_offs= py_vec->strides[0]; \ - vector=SG_MALLOC(sg_type, len); \ - char* data=(char*) py_vec->data; \ - npy_intp offs=0; \ - \ - for (int32_t i=0; ind!=2) \ - { \ - SG_ERROR("Expected " error_string " Matrix as argument %d\n", \ - m_rhs_counter); \ - } \ - \ - num_feat=py_mat->dimensions[0]; \ - num_vec=py_mat->dimensions[1]; \ - matrix=SG_MALLOC(sg_type, num_vec*num_feat); \ - char* data=py_mat->data; \ - npy_intp* strides= py_mat->strides; \ - npy_intp d2_offs=0; \ - for (int32_t i=0; ind; \ - int64_t total_size=0; \ - \ - dims=SG_MALLOC(int32_t, num_dims); \ - for (int32_t d=0; ddimensions[d]; \ - total_size+=dims[d]; \ - } \ - \ - array=SG_MALLOC(sg_type, total_size); \ - \ - char* data=py_mat->data; \ - for (int64_t i=0; i*& matrix, int32_t& num_feat, int32_t& num_vec) \ -{ \ - /* no sparse available yet */ \ - return; \ - \ - /* \ - const PyArray_Object* py_mat=(PyArrayObject *) get_arg_increment(); \ - if (!PyArray_Check(py_mat)) \ - SG_ERROR("Expected Sparse Matrix as argument %d\n", m_rhs_counter); \ - \ - if (!PyArray_TYPE(py_mat)!=py_type) \ - SG_ERROR("Expected " error_string " Matrix as argument %d\n", \ - m_rhs_counter); \ - \ - num_vec=py_mat->dimensions[0]; \ - num_feat=py_mat->nd; \ - matrix=SG_MALLOC(SGSparseVector, num_vec); \ - if_type* data=(if_type*) py_mat->data; \ - \ - int64_t nzmax=mxGetNzmax(mx_mat); \ - mwIndex* ir=mxGetIr(mx_mat); \ - mwIndex* jc=mxGetJc(mx_mat); \ - int64_t offset=0; \ - for (int32_t i=0; i0) \ - { \ - matrix[i].features=SG_MALLOC(SGSparseVectorEntry, len); \ - for (int32_t j=0; j*& strings, int32_t& num_str, int32_t& max_string_len) \ -{ \ - max_string_len=0; \ - const PyObject* py_str= get_arg_increment(); \ - if (!py_str) \ - SG_ERROR("Expected Stringlist as argument (none given).\n"); \ - \ - if (PyList_Check(py_str)) \ - { \ - if (!is_char_str) \ - SG_ERROR("Only Character Strings supported.\n"); \ - \ - num_str=PyList_Size((PyObject*) py_str); \ - ASSERT(num_str>=1); \ - \ - strings=SG_MALLOC(SGString, num_str); \ - ASSERT(strings); \ - \ - for (int32_t i=0; i(o))); \ - \ - strings[i].slen=len; \ - strings[i].string=NULL; \ - max_string_len=CMath::max(max_string_len, len); \ - \ - if (len>0) \ - { \ - strings[i].string=SG_MALLOC(sg_type, len+1); \ - memcpy(strings[i].string, str, len); \ - strings[i].string[len]='\0'; \ - } \ - } \ - else \ - { \ - for (int32_t j=0; jnd==2) \ - { \ - const PyArrayObject* py_array_str=(const PyArrayObject*) py_str; \ - if_type* data=(if_type*) py_array_str->data; \ - num_str=py_array_str->dimensions[0]; \ - int32_t len=py_array_str->dimensions[1]; \ - strings=SG_MALLOC(SGString, num_str); \ - \ - for (int32_t i=0; i0) \ - { \ - strings[i].slen=len; /* all must have same length*/ \ - strings[i].string=SG_MALLOC(sg_type, len+1); /* not zero terminated */ \ - int32_t j; \ - for (j=0; j*& strings, int32_t& num_str, int32_t& max_string_len) \ -{ \ - max_string_len=0; \ - const PyObject* py_str= get_arg_increment(); \ - if (!py_str) \ - SG_ERROR("Expected Stringlist as argument (none given).\n"); \ - \ - if (PyList_Check(py_str)) \ - { \ - if (!is_char_str) \ - SG_ERROR("Only Character Strings supported.\n"); \ - \ - num_str=PyList_Size((PyObject*) py_str); \ - ASSERT(num_str>=1); \ - \ - strings=SG_MALLOC(SGString, num_str); \ - ASSERT(strings); \ - \ - for (int32_t i=0; i0) \ - { \ - strings[i].string=SG_MALLOC(sg_type, len+1); \ - memcpy(strings[i].string, str, len); \ - strings[i].string[len]='\0'; \ - } \ - } \ - else \ - { \ - for (int32_t j=0; jnd==2) \ - { \ - const PyArrayObject* py_array_str=(const PyArrayObject*) py_str; \ - if_type* data=(if_type*) py_array_str->data; \ - num_str=py_array_str->dimensions[0]; \ - int32_t len=py_array_str->dimensions[1]; \ - strings=SG_MALLOC(SGString, num_str); \ - \ - for (int32_t i=0; i0) \ - { \ - strings[i].slen=len; /* all must have same length*/ \ - strings[i].string=SG_MALLOC(sg_type, len+1); /* not zero terminated */ \ - int32_t j; \ - for (j=0; j* &attrs) -{ - attrs=NULL; -} - - -/** set functions - to pass data from shogun to the target interface */ - -void CPythonInterface::set_int(int32_t scalar) -{ - PyObject* o=Py_BuildValue("i", scalar); - if (!o) - SG_ERROR("Could not build an integer.\n"); - - set_arg_increment(o); -} - -void CPythonInterface::set_real(float64_t scalar) -{ - PyObject* o=Py_BuildValue("d", scalar); - if (!o) - SG_ERROR("Could not build a double.\n"); - - set_arg_increment(o); -} - -void CPythonInterface::set_bool(bool scalar) -{ - // bool does not exist in Py_BuildValue, using byte instead - PyObject* o=Py_BuildValue("b", scalar); - if (!o) - SG_ERROR("Could not build a bool.\n"); - - set_arg_increment(o); -} - - -#define SET_VECTOR(function_name, py_type, sg_type, if_type, error_string) \ -void CPythonInterface::function_name(const sg_type* vector, int32_t len) \ -{ \ - if (!vector || len<1) \ - SG_ERROR("Given vector is invalid.\n"); \ - \ - npy_intp dims=len; \ - PyObject* py_vec=PyArray_SimpleNew(1, &dims, py_type); \ - if (!py_vec || !PyArray_Check(py_vec)) \ - SG_ERROR("Couldn't create " error_string " Vector of length %d.\n", \ - len); \ - \ - if_type* data=(if_type*) ((PyArrayObject *) py_vec)->data; \ - \ - for (int32_t i=0; idata; \ - \ - for (int32_t j=0; j* matrix, int32_t num_feat, int32_t num_vec, int64_t nnz) \ -{ \ - /* no sparse available yet */ \ - return; \ - /*\ - if (!matrix) \ - SG_ERROR("Given matrix is invalid.\n"); \ - \ - mxArray* mx_mat=mxCreateSparse(num_feat, num_vec, nnz, mxREAL); \ - if (!mx_mat) \ - SG_ERROR("Couldn't create Sparse Matrix of %d rows and %d cols.\n", num_feat, num_vec); \ - \ - if_type* data=(if_type*) mxGetData(mx_mat); \ - \ - mwIndex* ir=mxGetIr(mx_mat); \ - mwIndex* jc=mxGetJc(mx_mat); \ - int64_t offset=0; \ - for (int32_t i=0; i* strings, int32_t num_str) \ -{ \ - if (!is_char_str) \ - SG_ERROR("Only character strings supported.\n"); \ - \ - if (!strings || num_str<1) \ - SG_ERROR("Given strings are invalid.\n"); \ - \ - PyObject* py_str=PyList_New(num_str); \ - if (!py_str || PyTuple_GET_SIZE(py_str)!=num_str) \ - SG_ERROR("Couldn't create Cell Array of %d strings.\n", num_str); \ - \ - for (int32_t i=0; i0) \ - { \ - PyObject* str=PyUnicode_FromStringAndSize((const char*) strings[i].string, len); \ - if (!str) \ - SG_ERROR("Couldn't create " error_string \ - " String %d of length %d.\n", i, len); \ - \ - PyList_SET_ITEM(py_str, i, str); \ - } \ - } \ - \ - set_arg_increment(py_str); \ -} - -#else - -#define SET_STRINGLIST(function_name, py_type, sg_type, if_type, is_char_str, error_string) \ -void CPythonInterface::function_name(const SGString* strings, int32_t num_str) \ -{ \ - if (!is_char_str) \ - SG_ERROR("Only character strings supported.\n"); \ - \ - if (!strings || num_str<1) \ - SG_ERROR("Given strings are invalid.\n"); \ - \ - PyObject* py_str=PyList_New(num_str); \ - if (!py_str || PyTuple_GET_SIZE(py_str)!=num_str) \ - SG_ERROR("Couldn't create Cell Array of %d strings.\n", num_str); \ - \ - for (int32_t i=0; i0) \ - { \ - PyObject* str=PyString_FromStringAndSize((const char*) strings[i].string, len); \ - if (!str) \ - SG_ERROR("Couldn't create " error_string \ - " String %d of length %d.\n", i, len); \ - \ - PyList_SET_ITEM(py_str, i, str); \ - } \ - } \ - \ - set_arg_increment(py_str); \ -} - -#endif - -SET_STRINGLIST(set_string_list, NPY_BYTE, uint8_t, uint8_t, 0, "Byte") -SET_STRINGLIST(set_string_list, NPY_CHAR, char, char, 1, "Char") -SET_STRINGLIST(set_string_list, NPY_INT, int32_t, int, 0, "Integer") -SET_STRINGLIST(set_string_list, NPY_SHORT, int16_t, short, 0, "Short") -SET_STRINGLIST(set_string_list, NPY_USHORT, uint16_t, unsigned short, 0, "Word") -#undef SET_STRINGLIST - -void CPythonInterface::set_attribute_struct(const CDynamicArray* attrs) -{ -} - -bool CPythonInterface::create_return_values(int32_t num) -{ - if (num<=0) - return true; - - m_lhs=PyTuple_New(num); - ASSERT(m_lhs); - - m_nlhs=num; - return PyTuple_GET_SIZE(m_lhs)==num; -} - - -void CPythonInterface::run_python_init() -{ -#ifdef LIBPYTHON - m_pylib = dlopen(LIBPYTHON, RTLD_NOW | RTLD_GLOBAL); - if (!m_pylib) - SG_SERROR("couldn't open " LIBPYTHON ".so\n"); -#endif - Py_Initialize(); - init_numpy(); -} - -void CPythonInterface::run_python_exit() -{ - Py_Finalize(); -#ifdef LIBPYTHON - dlclose(m_pylib); -#endif -} - -bool CPythonInterface::run_python_helper(CSGInterface* from_if) -{ - SG_OBJ_DEBUG(from_if, "Entering Python\n") - PyObject* globals = PyDict_New(); - PyObject* builtins = PyEval_GetBuiltins(); - PyDict_SetItemString(globals,"__builtins__", builtins); - char* python_code=NULL; - - for (int i=0; iget_nrhs(); i++) - { - int len=0; - char* var_name = from_if->get_string(len); - SG_OBJ_DEBUG(from_if, "var_name = '%s'\n", var_name); - if (strmatch(var_name, "pythoncode")) - { - len=0; - python_code=from_if->get_string(len); - SG_OBJ_DEBUG(from_if, "python_code = '%s'\n", python_code); - break; - } - else - { - PyObject* tuple = PyTuple_New(1); - - CPythonInterface* in = new CPythonInterface(tuple); - in->create_return_values(1); - from_if->translate_arg(from_if, in); - PyDict_SetItemString(globals, var_name, in->get_return_values()); - SG_FREE(var_name); - Py_DECREF(tuple); - SG_UNREF(in); - } - } - - PyObject* python_code_obj = Py_CompileString(python_code, "", Py_file_input); - if (python_code_obj == NULL) - { - PyErr_Print(); - SG_OBJ_ERROR(from_if, "Compiling python code failed."); - } - - SG_FREE(python_code); - -#ifdef IS_PYTHON3 - PyObject* res = PyEval_EvalCode(python_code_obj, globals, NULL); -#else - PyObject* res = PyEval_EvalCode((PyCodeObject*) python_code_obj, globals, NULL); -#endif - - Py_DECREF(python_code_obj); - - if (res == NULL) - { - PyErr_Print(); - SG_OBJ_ERROR(from_if, "Running python code failed.\n"); - } - else - SG_OBJ_DEBUG(from_if, "Successfully executed python code.\n"); - - Py_DECREF(res); - - PyObject* results = PyDict_GetItemString(globals, "results"); - int32_t sz=-1; - - if (results) - { - if (!PyTuple_Check(results)) - SG_OBJ_ERROR(from_if, "results should be a tuple, e.g. results=(1,2,3) or results=tuple([42])") - else - sz=PyTuple_Size(results); - } - - if (sz>0 && from_if->create_return_values(sz)) - { - CPythonInterface* out = new CPythonInterface(results); - - //process d - for (int32_t i=0; itranslate_arg(out, from_if); - - Py_DECREF(results); - SG_UNREF(out); - } - else - { - if (sz>-1 && sz!=from_if->get_nlhs()) - { - SG_OBJ_ERROR(from_if, "Number of return values (%d) does not match number of expected" - " return values (%d).\n", sz, from_if->get_nlhs()) - } - } - - Py_DECREF(globals); - SG_OBJ_DEBUG(from_if, "Leaving Python.\n"); - return true; -} - -bool CPythonInterface::cmd_run_octave() -{ -#ifdef HAVE_OCTAVE - return COctaveInterface::run_octave_helper(this); -#else - return false; -#endif -} - -bool CPythonInterface::cmd_run_r() -{ -#ifdef HAVE_R - return CRInterface::run_r_helper(this); -#else - return false; -#endif -} - - -#ifdef HAVE_ELWMS -PyObject* elwms(PyObject* self, PyObject* args) -#else -PyObject* sg(PyObject* self, PyObject* args) -#endif -{ - try - { - if (!interface) - interface=new CPythonInterface(self, args); - else - ((CPythonInterface*) interface)->reset(self, args); - - if (!interface->handle()) - SG_SERROR("Unknown command.\n"); - } - catch (std::bad_alloc) - { - SG_SPRINT("Out of memory error.\n"); - return NULL; - } - catch (ShogunException e) - { - PyErr_SetString(PyExc_RuntimeError, e.get_exception_string()); - return NULL; - } - catch (...) - { - return NULL; - } - - return ((CPythonInterface*) interface)->get_return_values(); -} - -void exitsg(void) -{ - SG_SINFO("Quitting...\n"); -#ifdef HAVE_OCTAVE - COctaveInterface::run_octave_exit(); -#endif -#ifdef HAVE_R - CRInterface::run_r_exit(); -#endif - exit_shogun(); -} - -static PyMethodDef sg_pythonmethods[] = { - {(char*) -#ifdef HAVE_ELWMS - "elwms", elwms, METH_VARARGS, (char*) "Shogun."}, -#else - "sg", sg, METH_VARARGS, (char*) "Shogun."}, -#endif - {NULL, NULL, 0, NULL} /* Sentinel */ -}; - -#ifdef HAVE_ELWMS -MOD_INIT(elwms) -#else -MOD_INIT(sg) -#endif -{ - PyObject *module; - - // initialize python interpreter - Py_Initialize(); - - // initialize threading (just in case it is needed) - PyEval_InitThreads(); - - // callback to cleanup at exit - Py_AtExit(exitsg); - - // initialize callbacks -#ifdef HAVE_ELWMS - MOD_DEF(module, (char*) "elwms", sg_pythonmethods); -#else - MOD_DEF(module, (char*) "sg", sg_pythonmethods); -#endif - - if (module == NULL) - return MOD_ERROR_VAL; - -#ifdef HAVE_OCTAVE - COctaveInterface::run_octave_init(); -#endif -#ifdef HAVE_R - CRInterface::run_r_init(); -#endif - init_numpy(); - - // init_shogun has to be called before anything else - // exit_shogun is called upon destruction in exitsg() - init_shogun(&python_print_message, &python_print_warning, - &python_print_error, &python_cancel_computations); - - return MOD_SUCCESS_VAL(module); -} diff --git a/src/interfaces/python_static/PythonInterface.h b/src/interfaces/python_static/PythonInterface.h deleted file mode 100644 index e3e26fffef1..00000000000 --- a/src/interfaces/python_static/PythonInterface.h +++ /dev/null @@ -1,237 +0,0 @@ -#ifndef __PYTHONINTERFACE__H_ -#define __PYTHONINTERFACE__H_ - -#undef _POSIX_C_SOURCE -#undef HAVE_STAT - -#include - -extern "C" { -#include -#include -} - -#include - -extern "C" { -#include -} - -#if PY_VERSION_HEX >= 0x03000000 - #define IS_PYTHON3 - - #define PyInt_Check PyLong_Check - #define PyInt_AS_LONG PyLong_AS_LONG -#endif - -// Python2/3 module initialization -#ifdef IS_PYTHON3 - #define MOD_ERROR_VAL NULL - #define MOD_SUCCESS_VAL(val) val - #define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void) - #define MOD_DEF(ob, name, methods) \ - static struct PyModuleDef moduledef = { \ - PyModuleDef_HEAD_INIT, name, NULL, -1, methods, }; \ - ob = PyModule_Create(&moduledef); -#else - #define MOD_ERROR_VAL - #define MOD_SUCCESS_VAL(val) - #define MOD_INIT(name) PyMODINIT_FUNC init##name(void) - #define MOD_DEF(ob, name, methods) \ - ob = Py_InitModule(name, methods); -#endif - -#ifdef IS_PYTHON3 -int init_numpy() -{ - import_array(); -} -#else -void init_numpy() -{ - import_array(); -} -#endif - -namespace shogun -{ -class CPythonInterface : public CSGInterface -{ - public: - CPythonInterface(PyObject* args); - CPythonInterface(PyObject* self, PyObject* args); - ~CPythonInterface(); - - /// reset to clean state - virtual void reset(PyObject* self, PyObject* args); - - /** get functions - to pass data from the target interface to shogun */ - - /// get type of current argument (does not increment argument counter) - virtual IFType get_argument_type(); - - virtual int32_t get_int(); - virtual float64_t get_real(); - virtual bool get_bool(); - - virtual char* get_string(int32_t& len); - - virtual void get_vector(uint8_t*& vector, int32_t& len); - virtual void get_vector(char*& vector, int32_t& len); - virtual void get_vector(int32_t*& vector, int32_t& len); - virtual void get_vector(float64_t*& vector, int32_t& len); - virtual void get_vector(float32_t*& vector, int32_t& len); - virtual void get_vector(int16_t*& vector, int32_t& len); - virtual void get_vector(uint16_t*& vector, int32_t& len); - - virtual void get_matrix( - uint8_t*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - char*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - int32_t*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - float32_t*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - float64_t*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - int16_t*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - uint16_t*& matrix, int32_t& num_feat, int32_t& num_vec); - - virtual void get_sparse_matrix( - SGSparseVector*& matrix, int32_t& num_feat, int32_t& num_vec); - - virtual void get_string_list( - SGString*& strings, int32_t& num_str, - int32_t& max_string_len); - virtual void get_string_list( - SGString*& strings, int32_t& num_str, - int32_t& max_string_len); - virtual void get_string_list( - SGString*& strings, int32_t& num_str, - int32_t& max_string_len); - virtual void get_string_list( - SGString*& strings, int32_t& num_str, - int32_t& max_string_len); - virtual void get_string_list( - SGString*& strings, int32_t& num_str, - int32_t& max_string_len); - - virtual void get_attribute_struct( - const CDynamicArray* &attrs); - - /** set functions - to pass data from shogun to the target interface */ - virtual void set_int(int32_t scalar); - virtual void set_real(float64_t scalar); - virtual void set_bool(bool scalar); - - virtual void set_vector(const uint8_t* vector, int32_t len); - virtual void set_vector(const char* vector, int32_t len); - virtual void set_vector(const int32_t* vector, int32_t len); - virtual void set_vector( - const float32_t* vector, int32_t len); - virtual void set_vector(const float64_t* vector, int32_t len); - virtual void set_vector(const int16_t* vector, int32_t len); - virtual void set_vector(const uint16_t* vector, int32_t len); - - virtual void set_matrix( - const uint8_t* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const char* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const int32_t* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const float32_t* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const float64_t* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const int16_t* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const uint16_t* matrix, int32_t num_feat, int32_t num_vec); - - virtual void get_ndarray( - uint8_t*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - char*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - int32_t*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - float32_t*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - float64_t*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - int16_t*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - uint16_t*& array, int32_t*& dims, int32_t& num_dims); - - virtual void set_sparse_matrix( - const SGSparseVector* matrix, int32_t num_feat, - int32_t num_vec, int64_t nnz); - - virtual void set_string_list( - const SGString* strings, int32_t num_str); - virtual void set_string_list( - const SGString* strings, int32_t num_str); - virtual void set_string_list( - const SGString* strings, int32_t num_str); - virtual void set_string_list( - const SGString* strings, int32_t num_str); - virtual void set_string_list( - const SGString* strings, int32_t num_str); - - virtual void set_attribute_struct( - const CDynamicArray* attrs); - - virtual bool create_return_values(int32_t num); - - PyObject* get_return_values() - { - if (m_nlhs==1) - { - PyObject* retval=PyTuple_GET_ITEM(m_lhs, 0); - Py_INCREF(retval); - Py_DECREF(m_lhs); - m_lhs=retval; - } - return m_lhs; - } - - /** @return object name */ - inline virtual const char* get_name() const { return "PythonInterface"; } - - static void run_python_init(); - static void run_python_exit(); - static bool run_python_helper(CSGInterface* from_if); - virtual bool cmd_run_octave(); - virtual bool cmd_run_r(); - - private: - const PyObject* get_arg_increment() - { - const PyObject* retval; - ASSERT(m_rhs_counter>=0 && m_rhs_counter=0 && m_lhs_counter -#include -#include -#include -#include -#include -#include -#include -} - - -#include -#include -#include -#include -#include -#include -#include - -#ifdef HAVE_PYTHON -#include "../python_static/PythonInterface.h" -#endif - -#ifdef HAVE_OCTAVE -#include "../octave_static/OctaveInterface.h" -#endif - -void r_print_message(FILE* target, const char* str) -{ - if (target==stdout) - Rprintf((char*) "%s", str); - else - fprintf(target, "%s", str); -} - -void r_print_warning(FILE* target, const char* str) -{ - if (target==stdout) - Rprintf((char*) "%s", str); - else - fprintf(target, "%s", str); -} - -void r_print_error(FILE* target, const char* str) -{ - if (target!=stdout) - fprintf(target, "%s", str); -} - -void r_cancel_computations(bool &delayed, bool &immediately) -{ - //R_Suicide((char*) "sg stopped by SIGINT\n"); -} - -extern CSGInterface* interface; - -CRInterface::CRInterface(SEXP prhs, bool skip) -: CSGInterface(skip) -{ - skip_value=skip; - reset(prhs); -} - -CRInterface::~CRInterface() -{ -} - -void CRInterface::reset(SEXP prhs) -{ - CSGInterface::reset(); - - if (skip_value && prhs) - prhs=CDR(prhs); - - m_nlhs=0; - m_nrhs=0; - if (prhs) - m_nrhs=Rf_length(prhs); - if (m_nrhs<0) - m_nrhs=0; - m_lhs=R_NilValue; - m_rhs=prhs; -} - - -/** get functions - to pass data from the target interface to shogun */ - - -/// get type of current argument (does not increment argument counter) -IFType CRInterface::get_argument_type() -{ - if (m_rhs) - { - SEXP arg=CAR(m_rhs); - - switch (TYPEOF(arg)) - { - case INTSXP: - return DENSE_INT; - case REALSXP: - return DENSE_REAL; - case STRSXP: - return STRING_CHAR; - }; - } - return UNDEFINED; -} - - -int32_t CRInterface::get_int() -{ - SEXP i=get_arg_increment(); - - if (i == R_NilValue || nrows(i)!=1 || ncols(i)!=1) - SG_ERROR("Expected Scalar Integer as argument %d\n", m_rhs_counter); - - if (TYPEOF(i) == REALSXP) - { - double d=REAL(i)[0]; - if (d-CMath::floor(d)!=0) - SG_ERROR("Expected Integer as argument %d\n", m_rhs_counter); - return (int32_t) d; - } - - if (TYPEOF(i) != INTSXP) - SG_ERROR("Expected Scalar Integer as argument %d\n", m_rhs_counter); - - return INTEGER(i)[0]; -} - -float64_t CRInterface::get_real() -{ - SEXP f=get_arg_increment(); - if (f == R_NilValue || TYPEOF(f) != REALSXP || nrows(f)!=1 || ncols(f)!=1) - SG_ERROR("Expected Scalar Float as argument %d\n", m_rhs_counter); - - return REAL(f)[0]; -} - -bool CRInterface::get_bool() -{ - SEXP b=get_arg_increment(); - if (b == R_NilValue || TYPEOF(b) != LGLSXP || nrows(b)!=1 || ncols(b)!=1) - SG_ERROR("Expected Scalar Boolean as argument %d\n", m_rhs_counter); - - return INTEGER(b)[0] != 0; -} - - -char* CRInterface::get_string(int32_t& len) -{ - SEXP s=get_arg_increment(); - if (s == R_NilValue || TYPEOF(s) != STRSXP || Rf_length(s)!=1) - SG_ERROR("Expected String as argument %d\n", m_rhs_counter); - - SEXPREC* rstr= STRING_ELT(s,0); - const char* str= CHAR(rstr); - len=LENGTH(rstr); - ASSERT(len>0); - char* res=SG_MALLOC(char, len+1); - memcpy(res, str, len*sizeof(char)); - res[len]='\0'; - return res; -} - -void CRInterface::get_vector(uint8_t*& vec, int32_t& len) -{ - vec=NULL; - len=0; -} - -void CRInterface::get_vector(char*& vec, int32_t& len) -{ - vec=NULL; - len=0; -} - -void CRInterface::get_vector(int32_t*& vec, int32_t& len) -{ - vec=NULL; - len=0; - - SEXP rvec=get_arg_increment(); - if( TYPEOF(rvec) != INTSXP ) - SG_ERROR("Expected Integer Vector as argument %d\n", m_rhs_counter); - - len=LENGTH(rvec); - vec=SG_MALLOC(int32_t, len); - ASSERT(vec); - - for (int32_t i=0; i*& matrix, int32_t& num_feat, int32_t& num_vec) -{ -} - -void CRInterface::get_string_list(SGString*& strings, int32_t& num_str, int32_t& max_string_len) -{ -} - -void CRInterface::get_string_list(SGString*& strings, int32_t& num_str, int32_t& max_string_len) -{ - SEXP strs=get_arg_increment(); - - if (strs == R_NilValue || TYPEOF(strs) != STRSXP) - SG_ERROR("Expected String List as argument %d\n", m_rhs_counter); - - SG_DEBUG("nrows=%d ncols=%d Rf_length=%d\n", nrows(strs), ncols(strs), Rf_length(strs)); - - if (nrows(strs) && ncols(strs)!=1) - { - num_str = ncols(strs); - max_string_len = nrows(strs); - - strings=SG_MALLOC(SGString, num_str); - ASSERT(strings); - - for (int32_t i=0; i, num_str); - ASSERT(strings); - - for (int32_t i=0; i*& strings, int32_t& num_str, int32_t& max_string_len) -{ -} - -void CRInterface::get_string_list(SGString*& strings, int32_t& num_str, int32_t& max_string_len) -{ -} - -void CRInterface::get_string_list(SGString*& strings, int32_t& num_str, int32_t& max_string_len) -{ -} - - -void CRInterface::get_attribute_struct(const CDynamicArray* &attrs) -{ - attrs=NULL; -} - -/** set functions - to pass data from shogun to the target interface */ -bool CRInterface::create_return_values(int32_t num) -{ - if (num<=0) - return true; - - PROTECT(m_lhs=allocVector(VECSXP, num)); - m_nlhs=num; - return Rf_length(m_lhs) == num; -} - -SEXP CRInterface::get_return_values() -{ - if (m_nlhs==1) - { - SEXP arg=VECTOR_ELT(m_lhs, 0); - SET_VECTOR_ELT(m_lhs, 0, R_NilValue); - UNPROTECT(1); - return arg; - } - - if (m_nlhs>0) - UNPROTECT(1); - - return m_lhs; -} - - -/** set functions - to pass data from shogun to the target interface */ - -void CRInterface::set_int(int32_t scalar) -{ - set_arg_increment(ScalarInteger(scalar)); -} - -void CRInterface::set_real(float64_t scalar) -{ - set_arg_increment(ScalarReal(scalar)); -} - -void CRInterface::set_bool(bool scalar) -{ - set_arg_increment(ScalarLogical(scalar)); -} - - -void CRInterface::set_vector(const char* vec, int32_t len) -{ -} - -#undef SET_VECTOR -#define SET_VECTOR(function_name, r_type, r_cast, sg_type, if_type, error_string) \ -void CRInterface::function_name(const sg_type* vec, int32_t len) \ -{ \ - SEXP feat=NULL; \ - PROTECT( feat = allocVector(r_type, len) ); \ - \ - for (int32_t i=0; i* matrix, int32_t num_feat, int32_t num_vec, int64_t nnz) -{ - // R does not support sparse matrices yet -} - -void CRInterface::set_string_list(const SGString* strings, int32_t num_str) -{ -} - //this function will fail for strings containing 0, unclear how to do 'raw' - //strings in R -void CRInterface::set_string_list(const SGString* strings, int32_t num_str) -{ - if (!strings) - SG_ERROR("Given strings are invalid.\n"); - - SEXP feat=NULL; - PROTECT( feat = allocVector(STRSXP, num_str) ); - - for (int32_t i=0; i0) - SET_STRING_ELT(feat, i, mkChar(strings[i].string)); - } - UNPROTECT(1); - set_arg_increment(feat); -} - -void CRInterface::set_string_list(const SGString* strings, int32_t num_str) -{ -} - -void CRInterface::set_string_list(const SGString* strings, int32_t num_str) -{ -} - -void CRInterface::set_string_list(const SGString* strings, int32_t num_str) -{ -} - -void CRInterface::set_attribute_struct(const CDynamicArray* attrs) -{ -} - -bool CRInterface::cmd_run_python() -{ -#ifdef HAVE_PYTHON - return CPythonInterface::run_python_helper(this); -#else - return false; -#endif -} - -bool CRInterface::cmd_run_octave() -{ -#ifdef HAVE_OCTAVE - return COctaveInterface::run_octave_helper(this); -#else - return false; -#endif -} - -void CRInterface::run_r_init() -{ -#ifdef R_HOME_ENV - setenv("R_HOME", R_HOME_ENV, 0); -#endif - char* name=get_strdup("R"); - char* opts=get_strdup("-q"); - char* argv[2]={name, opts}; - Rf_initEmbeddedR(2, argv); - free(opts); - free(name); -} - -void CRInterface::run_r_exit() -{ - //R_dot_Last(); - //R_RunExitFinalizers(); - //R_gc(); - Rf_endEmbeddedR(0); -} - -bool CRInterface::run_r_helper(CSGInterface* from_if) -{ - char* rfile=NULL; - - try - { - for (int i=0; iget_nrhs(); i++) - { - int len=0; - char* var_name = from_if->get_string(len); - SG_OBJ_DEBUG(from_if, "var_name = '%s'\n", var_name); - if (strmatch(var_name, "rfile")) - { - len=0; - rfile=from_if->get_string(len); - SG_OBJ_DEBUG(from_if, "rfile = '%s'\n", rfile); - break; - } - else - { - CRInterface* in = new CRInterface(R_NilValue, false); - in->create_return_values(1); - from_if->translate_arg(from_if, in); - - setVar(install(var_name), in->get_return_values(), R_GlobalEnv); - SG_FREE(var_name); - SG_UNREF(in); - } - } - } - catch (ShogunException e) - { - SG_OBJ_PRINT(from_if, "%s", e.get_exception_string()) - return true; - } - - // Find source function - SEXP src = Rf_findFun(Rf_install("source"), R_GlobalEnv); - PROTECT(src); - - // Make file argument - SEXP file; - PROTECT(file = NEW_CHARACTER(1)); - SET_STRING_ELT(file, 0, COPY_TO_USER_STRING(rfile)); - - // expression source(file,print.eval=p) - SEXP expr; - PROTECT(expr = allocVector(LANGSXP,2)); - SETCAR(expr,src); - SETCAR(CDR(expr),file); - - int err=0; - R_tryEval(expr,NULL,&err); - - if (err) - { - UNPROTECT(3); - SG_OBJ_PRINT(from_if, "Error occurred\n"); - return true; - } - - SEXP results; - PROTECT(results=findVar(install("results"), R_GlobalEnv)); - SG_OBJ_DEBUG(from_if, "Found type %d\n", TYPEOF(results)); - - try - { - if (TYPEOF(results)==LISTSXP) - { - int32_t sz=Rf_length(results); - SG_OBJ_DEBUG(from_if, "Found %d args\n", sz); - - if (sz>0 && from_if->create_return_values(sz)) - { - CRInterface* out = new CRInterface(results, false); - - //process d - for (int32_t i=0; itranslate_arg(out, from_if); - - SG_UNREF(out); - } - else if (sz!=from_if->get_nlhs()) - { - UNPROTECT(4); - SG_OBJ_PRINT(from_if, "Number of return values (%d) does not match " - "number of expected return values (%d).\n", - sz, from_if->get_nlhs()); - return true; - } - } - } - catch (ShogunException e) - { - UNPROTECT(4); - SG_OBJ_PRINT(from_if, "%s", e.get_exception_string()); - } - - UNPROTECT(4); - - return true; -} - -/* The main function of the shogun R interface. All commands from the R command line - * to the shogun backend are passed using the syntax: - * .External("sg", "func", ... ) - * where '...' is a number of arguments passed to the shogun function 'func'. */ - -extern "C" { -/* This method is called by R when the shogun module is loaded into R - * via dyn.load('sg.so'). */ - -SEXP Rsg(SEXP args); - -#ifdef HAVE_ELWMS -void R_init_elwms(DllInfo *info) -#else -void R_init_sg(DllInfo *info) -#endif -{ - /* There are four different external language call mechanisms available in R, namely: - * .C - * .Call - * .Fortran - * .External - * - * Currently shogun uses only the .External interface. */ - - R_CMethodDef cMethods[] = { {NULL, NULL, 0} }; - R_FortranMethodDef fortranMethods[] = { {NULL, NULL, 0} }; - -#ifdef HAVE_ELWMS - R_ExternalMethodDef externalMethods[] = { {"elwms", (void*(*)()) &Rsg, -1}, {NULL, NULL, 0} }; -#else - R_ExternalMethodDef externalMethods[] = { {"sg", (void*(*)()) &Rsg, -1}, {NULL, NULL, 0} }; -#endif - R_CallMethodDef callMethods[] = { {NULL, NULL, 0} }; - - /* Register the routines saved in the callMethods structure so that they are available under R. */ - R_registerRoutines(info, cMethods, callMethods, (R_FortranMethodDef*) fortranMethods, (R_ExternalMethodDef*) externalMethods); - -} - -SEXP Rsg(SEXP args) -{ - /* The SEXP (Simple Expression) args is a list of arguments of the .External call. - * it consists of "sg", "func" and additional arguments. - * */ - - try - { - if (!interface) - { - // init_shogun has to be called before anything else - // exit_shogun is called upon destruction of the interface (see - // destructor of CRInterface - init_shogun(&r_print_message, &r_print_warning, - &r_print_error, &r_cancel_computations); - interface=new CRInterface(args); -#ifdef HAVE_PYTHON - CPythonInterface::run_python_init(); -#endif -#ifdef HAVE_OCTAVE - COctaveInterface::run_octave_init(); -#endif - } - else - ((CRInterface*) interface)->reset(args); - - if (!interface->handle()) - SG_SERROR("Unknown command.\n"); - } - catch (std::bad_alloc) - { - error("Out of memory error.\n"); - return R_NilValue; - } - catch (ShogunException e) - { - error("%s", e.get_exception_string()); - return R_NilValue; - } - catch (...) - { - error("%s", "Returning from SHOGUN in error."); - return R_NilValue; - } - - return ((CRInterface*) interface)->get_return_values(); -} - -/* This method is called form within R when the current module is unregistered. - * Note that R does not allow unregistering of single symbols. */ - -#ifdef HAVE_ELWMS -void R_unload_elwms(DllInfo *info) -#else -void R_unload_sg(DllInfo *info) -#endif -{ -#ifdef HAVE_PYTHON - CPythonInterface::run_python_exit(); -#endif -#ifdef HAVE_OCTAVE - COctaveInterface::run_octave_exit(); -#endif - - exit_shogun(); -} - -} // extern "C" diff --git a/src/interfaces/r_static/RInterface.h b/src/interfaces/r_static/RInterface.h deleted file mode 100644 index 1585face81d..00000000000 --- a/src/interfaces/r_static/RInterface.h +++ /dev/null @@ -1,180 +0,0 @@ -#ifndef __RINTERFACE__H_ -#define __RINTERFACE__H_ - -#include - -#include -#include - -extern "C" { -#include -#include -} - -using namespace shogun; - -class CRInterface : public CSGInterface -{ - public: - CRInterface(SEXP prhs, bool skip=true); - ~CRInterface(); - - /// reset to clean state - virtual void reset(SEXP prhs); - - /** get functions - to pass data from the target interface to shogun */ - - /// get type of current argument (does not increment argument counter) - virtual IFType get_argument_type(); - - virtual int32_t get_int(); - virtual float64_t get_real(); - virtual bool get_bool(); - - virtual char* get_string(int32_t& len); - - virtual void get_vector(uint8_t*& vec, int32_t& len); - virtual void get_vector(char*& vec, int32_t& len); - virtual void get_vector(int32_t*& vec, int32_t& len); - virtual void get_vector(float32_t*& vec, int32_t& len); - virtual void get_vector(float64_t*& vec, int32_t& len); - virtual void get_vector(int16_t*& vec, int32_t& len); - virtual void get_vector(uint16_t*& vec, int32_t& len); - - virtual void get_matrix( - uint8_t*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - char*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - int32_t*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - float32_t*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - float64_t*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - int16_t*& matrix, int32_t& num_feat, int32_t& num_vec); - virtual void get_matrix( - uint16_t*& matrix, int32_t& num_feat, int32_t& num_vec); - - virtual void get_ndarray( - uint8_t*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - char*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - int32_t*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - float32_t*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - float64_t*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - int16_t*& array, int32_t*& dims, int32_t& num_dims); - virtual void get_ndarray( - uint16_t*& array, int32_t*& dims, int32_t& num_dims); - - virtual void get_sparse_matrix( - SGSparseVector*& matrix, int32_t& num_feat, int32_t& num_vec); - - virtual void get_string_list( - SGString*& strings, int32_t& num_str, - int32_t& max_string_len); - virtual void get_string_list( - SGString*& strings, int32_t& num_str, - int32_t& max_string_len); - virtual void get_string_list( - SGString*& strings, int32_t& num_str, - int32_t& max_string_len); - virtual void get_string_list( - SGString*& strings, int32_t& num_str, - int32_t& max_string_len); - virtual void get_string_list( - SGString*& strings, int32_t& num_str, - int32_t& max_string_len); - - virtual void get_attribute_struct( - const CDynamicArray* &attrs); - - /** set functions - to pass data from shogun to the target interface */ - virtual void set_int(int32_t scalar); - virtual void set_real(float64_t scalar); - virtual void set_bool(bool scalar); - - virtual bool create_return_values(int32_t num_val); - virtual void set_vector(const uint8_t* vec, int32_t len); - virtual void set_vector(const char* vec, int32_t len); - virtual void set_vector(const int32_t* vec, int32_t len); - virtual void set_vector(const float32_t* vec, int32_t len); - virtual void set_vector(const float64_t* vec, int32_t len); - virtual void set_vector(const int16_t* vec, int32_t len); - virtual void set_vector(const uint16_t* vec, int32_t len); - - virtual void set_matrix( - const uint8_t* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const char* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const int32_t* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const float32_t* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const float64_t* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const int16_t* matrix, int32_t num_feat, int32_t num_vec); - virtual void set_matrix( - const uint16_t* matrix, int32_t num_feat, int32_t num_vec); - - virtual void set_sparse_matrix( - const SGSparseVector* matrix, int32_t num_feat, - int32_t num_vec, int64_t nnz); - - virtual void set_string_list( - const SGString* strings, int32_t num_str); - virtual void set_string_list( - const SGString* strings, int32_t num_str); - virtual void set_string_list( - const SGString* strings, int32_t num_str); - virtual void set_string_list( - const SGString* strings, int32_t num_str); - virtual void set_string_list( - const SGString* strings, int32_t num_str); - - virtual void set_attribute_struct( - const CDynamicArray* attrs); - - SEXP get_return_values(); - - virtual bool cmd_run_python(); - virtual bool cmd_run_octave(); - static void run_r_init(); - static void run_r_exit(); - static bool run_r_helper(CSGInterface* from_if); - - private: - const SEXP get_arg_increment() - { - ASSERT(m_rhs_counter>=0 && m_rhs_counter=0 && m_lhs_counter - - -Usage ------ - -There are two major functions: - -- Generate testcase data. - This is run seldomly, usually when new functionality has been added or - algorithmic bugs have been found. The generator is implemented in python - and uses SHOGUN's interface python-modular or perl-modular. The resulting files are in - matlab/octave format. - - Just issue - - ./generate_testdata.py - - and all data files will be created in data/, within subdirectories - according to the category of which they are part of. This will not remove - previously created testcases, so you might want to run - - ./generate_testdata.py clear - - to remove these files. - - If you want to (re)create only the testcases for a specific category, issue - - ./generate_testdata.py - - where category is one of: classifier, clustering, distance, distribution, - kernel, preproc, regression. - - -- Test your current iteration of SHOGUN against the previously created - testcases. - For that you have to change directory into the interface you want to test - against, e.g. - - cd python-modular - - There you can test against all testcases or all of one category or - individually. - To test against all, just issue - - ./test_all.sh - - To test against all of one category, issue - - ./test_all.sh - - where category is one of: see above - - To test individually, issue - - ./test_one.py ../data//.m (python) - ./test_one.pl ../data//.m (perl) - ./test_one.sh ../data//.m (matlab/octave/R) - - If you run tests individually, you get to see more than ERROR in case - of failure. - - - - -Extension ---------- - -Now that was easy, the difficult part is how to extend the beast. The overview -given below is hopefully sufficient to help the uninitiated to succeed. - -- Generator - Change the working directory to generator/. - - There are 4 modules to help with recurring operations: - 1) category.py - defines the categories used, their names and - representation as number (as a sort of enum) - 2) dataop.py - helps generating various random data, like numbers, DNA - sequences, random numbers structured in clouds, etc. - 3) featop.py - helps creating feature objects out of the given data - 4) fileop.py - helps preparing and writing the generated data to file. - - The first task now is to define the category where you want your testcase - to be generated. If you should ever add another category, don't forget to - add it to the constant CATEGORIES in __init__.py and to update category.py - accordingly. - Each category module has a function run() which is called via __init__.py - and itself runs other run_* functions which group items together, - for instance kernels with Byte features, kernels with String features, - kernels that can handle subkernels, linear SVM classifiers, kernel-based - SVM classifiers and so on. - Basically each group function creates the necessary data and then - establishes a hash called params which contains all the parameters that are - necessary to run an item and must be written to file. This hash is - often modified while the group function is executed, depending on - which item is actually run. Sometimes it contains other hashes and tuples - like kernel parameters, sometimes it is just another integer. Each item's - parameters are defined either at the beginning of the group's run - function (if relevant to the whole group) or shortly before the item is - about to be run. - Usually, a compute_* function is then executed which actually feeds the - gathered data and parameters to SHOGUN and handles its output. Some - compute_* functions are relevant to only one group function, some are used - by several group functions, kernel.compute() is used by - kernel.run_feats_byte() and kernel.run_distance(), for example while - kernel.compute_pie() is only used by kernel.run_pie(). - Classifiers (and regressions) are a bit different, because some of them run - through a loop function which feeds the compute function with different - values for C, number of threads, epsilon, etc., modifying the hash params - while doing so. - The compute_* functions interface with the fileop module to gather output - data and write it all to file. Afterwards the next group or category is - run or the generator has finished its job. - - - -- Testers - Each interface has its own tester, although libshogun and cmdline have no - tester at all at the moment. Furthermore, there are 2 groups which run - somewhat similar code - the static and the dynamic interfaces. - - - Static interfaces (python/, perl/, matlab_and_octave/, r/): - - Of course, the details are dependent on the actual interface, but all of - them have some utility functions (util.py, util/*) to check accuracy, set - features, set distances/kernels/classifiers and fixing some name - inconsistencies between static and modular interface. the generator uses - the modular interface and hence some names in the testcase files don't - match what the static interface understands. These utility functions are - common to all categories. - The aforementioned test_all scripts call a test_one script for each - testcase which takes care of converting the matlab/octave format of the - testcase file into a local format. It then calls a module from the - appropriate category. As in the generator, each category is represented in - one file, e.g. kernel.py/kernel.m/kernel.R. - For python or for perl, the function test() is the entry point from which various - evaluate() functions are spawned, depending on the item to be tested. these - evaluations are similar to the generator's compute() functions, but instead - of writing the results and parameters too file, they are compared to the - results in the testcase file. - In octave/matlab, the main function of a category is named as the category. - Due to the nature of how global variables are handled in matlab/octave, - there is a file called util/globals.m which contains all possible variable - names that can be in the testcase files. So be aware that you might need to - update this file and also use 'global ;' in the function where you - want to use it. It is cludgy indeed, but did seem to be the only way to - handle the problem with the limited capabilities of matlab and me. - The structure of the R files resembles the structure octave/matlab, but - in its own syntax of course and without the global variable cludge. - - - Modular interfaces (python-modular/, perl-modular/, octave-modular/, r-modular/): - - Essentially, the structure is the same as in the static interfaces with - the main difference being a different object handling in python/R, perl/R as in - get_* functions to return the created objects instead of set_* where the - data is fed bit by bit into one big state machine called SHOGUN. - octave still uses set_* because it makes use of global variables which - might be able to converted to get_* as well. The inclined reader is - encouraged to try that. diff --git a/tests/integration/blacklist b/tests/integration/blacklist deleted file mode 100644 index 98898ed76a6..00000000000 --- a/tests/integration/blacklist +++ /dev/null @@ -1,12 +0,0 @@ -distribution/HMM_1en10_60_4_1_0_6_3_False_CUBE.m -classifier/SVMSGD_0017_1en05_1_True.m -classifier/SVMSGD_023_1en05_1_True.m -classifier/SVMSGD_15_1en05_1_True.m -classifier/SVMSGD_30_1en05_16_True.m -classifier/SVMSGD_30_1en05_1_True.m -classifier/SubGradientSVM_0017_1en05_1_05_False.m -classifier/SubGradientSVM_15_1en05_1_05_False.m -classifier/SubGradientSVM_023_1en05_1_05_False.m -classifier/SubGradientSVM_30_1en05_1_05_False.m -kernel/Poly_1_0_10_3_6_01_False_False_CUBE_1_FK.m -kernel/Poly_1_0_10_3_6_01_False_False_CUBE_1_TOP.m \ No newline at end of file diff --git a/tests/integration/cmdline_static/test.sg b/tests/integration/cmdline_static/test.sg deleted file mode 100644 index 04e9ead5772..00000000000 --- a/tests/integration/cmdline_static/test.sg +++ /dev/null @@ -1,35 +0,0 @@ -# this is a command file for shogun - # indented comment - # tabbed comment - -# show help -help - -# enable debugging -# loglevel ALL - -# execute system command -! ls -l - -# example with real valued features -set_features TRAIN ../data/fm_real.dat -fm_real2.dat = get_features TRAIN - -set_kernel GAUSSIAN REAL 10 100.2 -km.txt=get_kernel_matrix TRAIN - -# example with sparse real valued features -set_features TRAIN ../data/fm_real_sparse.dat -fm_real_sparse2.dat = get_features TRAIN - -set_kernel GAUSSIAN SPARSEREAL 100 1.2 -km_sparse.txt = get_kernel_matrix TRAIN - -# example with sparse real valued features -set_features TRAIN ../data/fm_stringchar_dna.dat DNA -fm_stringchar_dna2.dat = get_features TRAIN - -set_features TRAIN ../data/fm_stringchar_dnamatrix.dat DNA -fm_stringchar_dnamatrix2.dat = get_features TRAIN -set_kernel WEIGHTEDDEGREE CHAR 10 20 -km_dna.txt = get_kernel_matrix TRAIN diff --git a/tests/integration/cmdline_static/test_all.sh b/tests/integration/cmdline_static/test_all.sh deleted file mode 100755 index af8bb3c0eeb..00000000000 --- a/tests/integration/cmdline_static/test_all.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash - -# not implemented yet -exit 0 - -DATAPATH='../data' - -function test_all () { - datapath="$1" - echo "*** Testing in $datapath" - sleep 1 - for file in $datapath; do - echo -n "$file" - echo -n -e "\t\t" - - output=`${PYTHON} test_one.py "$file"` - ret=$? - - if [ $ret -eq 0 ] ; then - echo 'OK' - else - echo 'ERROR' - echo $output - fi - done - sleep 1 - echo -} - -if [ -n "$1" ]; then - test_all "$DATAPATH/$1/*.m" -else - for i in $DATAPATH/*; do - test_all "$i/*.m" - done -fi diff --git a/tests/integration/data b/tests/integration/data deleted file mode 120000 index a2dff648729..00000000000 --- a/tests/integration/data +++ /dev/null @@ -1 +0,0 @@ -../../data/testsuite \ No newline at end of file diff --git a/tests/integration/generate_testdata.py b/tests/integration/generate_testdata.py deleted file mode 100755 index ad59ad3b0ed..00000000000 --- a/tests/integration/generate_testdata.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python - -if __name__=='__main__': - import generator - import sys - generator.run(sys.argv) diff --git a/tests/integration/generator/__init__.py b/tests/integration/generator/__init__.py deleted file mode 100644 index 569549e96e4..00000000000 --- a/tests/integration/generator/__init__.py +++ /dev/null @@ -1,66 +0,0 @@ -"""A package to generate testcases for the Shogun toolbox - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation in version 3. - -Written (W) 2007-2008 Sebastian Henschel -Copyright (C) 2007-2008 Fraunhofer Institute FIRST and Max-Planck-Society -""" - -import sys -import os -import numpy.random as random - -from shogun.Library import Math_init_random -from dataop import INIT_RANDOM -from fileop import clean_dir_outdata - -CATEGORIES=['classifier', 'clustering', 'distance', 'distribution', 'kernel', \ - 'regression', 'preproc'] - -def run (argv): - """ - Run all individual generators or only one if present in - argument list. - """ - - # put some constantness into randomness - Math_init_random(INIT_RANDOM) - random.seed(INIT_RANDOM) - - arglen=len(argv) - if arglen==2: # run given category - if argv[1]=='clear': - clean_dir_outdata() - else: - try: - __import__(argv[1], globals(), locals()); - except ImportError: - raise ImportError, 'Unknown category ' + argv[1] - - category=eval(argv[1]) - category.run() - else: - # run given category by calling self again, one by one - # this is due to an issue somewhere with classifiers (atm) and - # 'static randomness' - - if arglen==1: - command=argv[0] - cats=CATEGORIES - else: - command=argv.pop(0) - cats=argv - - for cat in cats: - if not cat in CATEGORIES: - cats=', '.join(CATEGORIES) - msg="Unknown category: %s\nTry one of these: %s\n"%(cat, cats) - sys.stderr.write(msg) - sys.exit(1) - - ret=os.system('%s %s' % (command, cat)) - if ret!=0: - sys.exit(ret) - diff --git a/tests/integration/generator/category.py b/tests/integration/generator/category.py deleted file mode 100644 index af86bef4837..00000000000 --- a/tests/integration/generator/category.py +++ /dev/null @@ -1,54 +0,0 @@ -CLASSIFIER=0 -CLUSTERING=1 -DISTANCE=2 -DISTRIBUTION=3 -KERNEL=4 -PREPROC=5 -REGRESSION=6 - - -def get_name(catID): - """ - Get index to lookup the entity's name in the output data dictionary. - - @param catID ID of the category - @return string index for name lookup in output data - """ - - names={ - CLASSIFIER: 'classifier_name', - CLUSTERING: 'clustering_name', - DISTANCE: 'distance_name', - DISTRIBUTION: 'distribution_name', - KERNEL: 'kernel_name', - PREPROC: 'preproc_name', - REGRESSION: 'regression_name', - } - - try: - return names[catID] - except KeyError: - return '' - - -def get_as_string (catID): - """Returns the string representation of given category. - - @param catID ID of a category - @return string of the category or empty string if ID was invalid - """ - - strings={ - CLASSIFIER: 'classifier', - CLUSTERING: 'clustering', - DISTANCE: 'distance', - DISTRIBUTION: 'distribution', - KERNEL: 'kernel', - PREPROC: 'preproc', - REGRESSION: 'regression' - } - - try: - return strings[catID] - except KeyError: - return '' diff --git a/tests/integration/generator/classifier.py b/tests/integration/generator/classifier.py deleted file mode 100644 index 31ab466d510..00000000000 --- a/tests/integration/generator/classifier.py +++ /dev/null @@ -1,475 +0,0 @@ -"""Generator for Classifier""" - -import numpy -import shogun.Library as library -import shogun.Classifier as classifier -from shogun.Kernel import * -from shogun.Distance import EuclidianDistance -from shogun.Features import Labels, RAWDNA - -import fileop -import featop -import dataop -import category - - -########################################################################## -# svm -########################################################################## - -def _get_svm (params, labels, feats, kernel): - """Return an SVM object. - - This function instantiates an SVM depending on the parameters. - - @param name Name of the SVM to instantiate - @param labels Labels to be used for the SVM (if at all!) - @param params Misc parameters for the SVM's constructor - @return An SVM object - """ - - try: - svm=eval('classifier.'+params['name']) - except AttributeError, e: - return False - - if params['type']=='kernel': - kernel.parallel.set_num_threads(params['num_threads']) - kernel.init(feats['train'], feats['train']) - - if labels is None: - return svm(params['C'], kernel) - else: - return svm(params['C'], kernel, labels) - elif params['type']=='wdsvmocas': - return svm(params['C'], params['degree'], params['degree'], - feats['train'], labels) - else: - return svm(params['C'], feats['train'], labels) - - -def _get_svm_sum_alpha_and_sv (svm, ltype): - """Return sums of alphas and support vectors for given (MultiClass) SVM. - Since alphas and support vectors are only used for comparison, it is - alright to only regard their sums. Especially in case of - MultiClassSVMs it makes test files smaller and easier to comprehend. - - @param svm (MultiClass) SVM to retrieve alphas from - @param ltype label type of SVM - @return 2-element list with sums of alphas and support vectors - """ - - a=0 - sv=0 - if ltype=='series': - for i in xrange(svm.get_num_svms()): - subsvm=svm.get_svm(i) - for item in subsvm.get_alphas().tolist(): - a+=item - for item in subsvm.get_support_vectors().tolist(): - sv+=item - else: - for item in svm.get_alphas().tolist(): - a+=item - for item in svm.get_support_vectors().tolist(): - sv+=item - - return a, sv - - -def _compute_svm (params, labels, feats, kernel, pout): - """Perform computations on SVM. - - Perform all necessary computations on SVM and gather the output. - - @param params misc parameters for the SVM's constructor - @param labels labels to be used for the SVM (if at all) - @param feats features to the SVM - @param kernel kernel for kernel-SVMs - @param pout previously gathered output data ready to be written to file - """ - - svm=_get_svm(params, labels, feats, kernel) - if not svm: - return - - svm.parallel.set_num_threads(params['num_threads']) - try: - svm.set_epsilon(params['epsilon']) - except AttributeError: #SGD does not have an accuracy parameter - pass - - if params.has_key('bias_enabled'): - svm.set_bias_enabled(params['bias_enabled']) - if params.has_key('max_train_time'): - svm.set_max_train_time(params['max_train_time']) - params['max_train_time']=params['max_train_time'] - if params.has_key('linadd_enabled'): - svm.set_linadd_enabled(params['linadd_enabled']) - if params.has_key('batch_enabled'): - svm.set_batch_computation_enabled(params['batch_enabled']) - - svm.train() - - if ((params.has_key('bias_enabled') and params['bias_enabled']) or - params['type']=='kernel'): - params['bias']=svm.get_bias() - - if params['type']=='kernel': - params['alpha_sum'], params['sv_sum']= \ - _get_svm_sum_alpha_and_sv(svm, params['label_type']) - kernel.init(feats['train'], feats['test']) - elif params['type']=='linear' or params['type']=='wdsvmocas': - svm.set_features(feats['test']) - - params['classified']=svm.apply().get_labels() - - output=fileop.get_output(category.CLASSIFIER, params) - if pout: - output.update(pout) - fileop.write(category.CLASSIFIER, output) - - -def _loop_svm (svms, params, feats, kernel=None, pout=None): - """Loop through SVM computations, only slightly differing in parameters. - - Loop through SVM computations with little variations in the parameters for - the SVM. Not necessarily used by all SVMs in this generator. - - @param svms tuple of SVM names to loop through - @param params parameters to the SVMs - @param feats features to the SVMs - @param kernel kernel for kernel-based SVMs - @param pout previously gathered output data ready to be written to file - """ - - for name in svms: - parms={ 'name': name, 'num_threads': 1, 'C': .017, 'epsilon': 1e-5 } - parms['accuracy']=parms['epsilon']*10 - parms.update(params) - - if params['label_type'] is not None: - parms['labels'], labels=dataop.get_labels( - feats['train'].get_num_vectors(), params['label_type']) - else: - labels=None - - _compute_svm(parms, labels, feats, kernel, pout) - parms['C']=.23 - _compute_svm(parms, labels, feats, kernel, pout) - parms['C']=1.5 - _compute_svm(parms, labels, feats, kernel, pout) - parms['C']=30 - _compute_svm(parms, labels, feats, kernel, pout) - - if params['type']=='kernel': - _compute_svm(parms, labels, feats, kernel, pout) - - parms['num_threads']=16 - _compute_svm(parms, labels, feats, kernel, pout) - - -def _run_svm_kernel (): - """Run all kernel-based SVMs.""" - - kparams={ - 'name': 'Gaussian', - 'args': {'key': ('width',), 'val': (1.5,)}, - 'feature_class': 'simple', - 'feature_type': 'Real', - 'data': dataop.get_clouds(2) - } - feats=featop.get_features( - kparams['feature_class'], kparams['feature_type'], kparams['data']) - kernel=GaussianKernel(10, *kparams['args']['val']) - output=fileop.get_output(category.KERNEL, kparams) - - svms=('SVMLight', 'LibSVM', 'GPBTSVM', 'MPDSVM') - params={ - 'type': 'kernel', - 'label_type': 'twoclass' - } - _loop_svm(svms, params, feats, kernel, output) - - svms=('LibSVMOneClass',) - params['label_type']=None - _loop_svm(svms, params, feats, kernel, output) - - svms=('LibSVMMultiClass', 'GMNPSVM') - params['label_type']='series' - kparams['data']=dataop.get_clouds(3) - feats=featop.get_features( - kparams['feature_class'], kparams['feature_type'], kparams['data']) - output=fileop.get_output(category.KERNEL, kparams) - _loop_svm(svms, params, feats, kernel, output) - - svms=('SVMLight', 'GPBTSVM') - params['label_type']='twoclass' - kparams={ - 'name': 'Linear', - 'feature_class': 'simple', - 'feature_type': 'Real', - 'data': dataop.get_clouds(2), - 'normalizer': AvgDiagKernelNormalizer() - } - feats=featop.get_features( - kparams['feature_class'], kparams['feature_type'], kparams['data']) - kernel=LinearKernel() - kernel.set_normalizer(kparams['normalizer']) - output=fileop.get_output(category.KERNEL, kparams) - _loop_svm(svms, params, feats, kernel, output) - - kparams={ - 'name': 'CommWordString', - 'args': {'key': ('use_sign',), 'val': (False,)}, - 'data': dataop.get_dna(), - 'feature_class': 'string_complex', - 'feature_type': 'Word' - } - feats=featop.get_features( - kparams['feature_class'], kparams['feature_type'], kparams['data']) - kernel=CommWordStringKernel(10, *kparams['args']['val']) - output=fileop.get_output(category.KERNEL, kparams) - _loop_svm(svms, params, feats, kernel, output) - - kparams={ - 'name': 'CommUlongString', - 'args': {'key': ('use_sign',), 'val': (False,)}, - 'data': dataop.get_dna(), - 'feature_class': 'string_complex', - 'feature_type': 'Ulong' - } - feats=featop.get_features( - kparams['feature_class'], kparams['feature_type'], kparams['data']) - kernel=CommUlongStringKernel(10, *kparams['args']['val']) - output=fileop.get_output(category.KERNEL, kparams) - _loop_svm(svms, params, feats, kernel, output) - - kparams={ - 'name': 'WeightedDegreeString', - 'args': {'key': ('degree',), 'val': (3,)}, - 'data': dataop.get_dna(), - 'feature_class': 'string', - 'feature_type': 'Char' - } - feats=featop.get_features( - kparams['feature_class'], kparams['feature_type'], kparams['data']) - kernel=WeightedDegreeStringKernel(*kparams['args']['val']) - output=fileop.get_output(category.KERNEL, kparams) - _loop_svm(svms, params, feats, kernel, output) - params['linadd_enabled']=True - _loop_svm(svms, params, feats, kernel, output) - params['batch_enabled']=True - _loop_svm(svms, params, feats, kernel, output) - - kparams={ - 'name': 'WeightedDegreePositionString', - 'args': {'key': ('degree',), 'val': (20,)}, - 'data': dataop.get_dna(), - 'feature_class': 'string', - 'feature_type': 'Char' - } - feats=featop.get_features( - kparams['feature_class'], kparams['feature_type'], kparams['data']) - kernel=WeightedDegreePositionStringKernel(10, *kparams['args']['val']) - output=fileop.get_output(category.KERNEL, kparams) - del params['linadd_enabled'] - del params['batch_enabled'] - _loop_svm(svms, params, feats, kernel, output) - params['linadd_enabled']=True - _loop_svm(svms, params, feats, kernel, output) - params['batch_enabled']=True - _loop_svm(svms, params, feats, kernel, output) - - -def _run_svm_linear (): - """Run all SVMs based on (Sparse) Linear Classifiers.""" - - params={ - 'type': 'linear', - 'bias_enabled': False, - 'data': dataop.get_clouds(2), - 'feature_class': 'simple', - 'feature_type': 'Real', - 'label_type': 'twoclass' - } - feats=featop.get_features( - params['feature_class'], params['feature_type'], - params['data'], sparse=True) - - svms=('LibLinear', 'SVMLin', 'SVMSGD') - params['bias_enabled']=True - _loop_svm(svms, params, feats) - - # SubGradientSVM needs max_train_time to terminate - svms=('SubGradientSVM',) - params['bias_enabled']=False - params['max_train_time']=.5 # up to 2. does not improve test results :( - _loop_svm(svms, params, feats) - - svms=('SVMOcas',) - _loop_svm(svms, params, feats) - - params={ - 'type': 'linear', - 'bias_enabled': False, - 'label_type': 'twoclass', - 'feature_class': 'wd', - 'feature_type': 'Byte', - 'data': dataop.get_dna(), - 'alphabet': 'RAWDNA', - 'order': 1 - } - feats=featop.get_features( - params['feature_class'], params['feature_type'], - params['data'], params['order']) - _loop_svm(svms, params, feats) - - - -########################################################################## -# other classifiers -########################################################################## - -def _run_perceptron (): - """Run Perceptron classifier.""" - - params={ - 'name': 'Perceptron', - 'type': 'perceptron', - 'num_threads': 1, - 'learn_rate': .1, - 'max_iter': 1000, - 'data': dataop.get_clouds(2), - 'feature_class': 'simple', - 'feature_type': 'Real', - 'label_type': 'twoclass', - 'accuracy': 1e-7 - } - feats=featop.get_features( - params['feature_class'], params['feature_type'], params['data']) - num_vec=feats['train'].get_num_vectors() - params['labels'], labels=dataop.get_labels(num_vec, params['label_type']) - - perceptron=classifier.Perceptron(feats['train'], labels) - perceptron.parallel.set_num_threads(params['num_threads']) - perceptron.set_learn_rate(params['learn_rate']) - perceptron.set_max_iter(params['max_iter']) - perceptron.train() - - params['bias']=perceptron.get_bias() - perceptron.set_features(feats['test']) - params['classified']=perceptron.classify().get_labels() - - output=fileop.get_output(category.CLASSIFIER, params) - fileop.write(category.CLASSIFIER, output) - - -def _run_knn (): - """Run K-Nearest-Neighbour classifier. - """ - - params={ - 'name': 'EuclidianDistance', - 'data': dataop.get_clouds(2), - 'feature_class': 'simple', - 'feature_type': 'Real' - } - feats=featop.get_features( - params['feature_class'], params['feature_type'], params['data']) - dfun=eval(params['name']) - distance=dfun(feats['train'], feats['train']) - output=fileop.get_output(category.DISTANCE, params) - - params={ - 'name': 'KNN', - 'type': 'knn', - 'num_threads': 1, - 'k': 3, - 'label_type': 'twoclass', - 'accuracy': 1e-8 - } - params['labels'], labels=dataop.get_labels( - feats['train'].get_num_vectors(), params['label_type']) - - knn=classifier.KNN(params['k'], distance, labels) - knn.parallel.set_num_threads(params['num_threads']) - knn.train() - - distance.init(feats['train'], feats['test']) - params['classified']=knn.classify().get_labels() - - output.update(fileop.get_output(category.CLASSIFIER, params)) - fileop.write(category.CLASSIFIER, output) - - -def _run_lda (): - """Run Linear Discriminant Analysis classifier.""" - - params={ - 'name': 'LDA', - 'type': 'lda', - 'gamma': 0.1, - 'num_threads': 1, - 'data': dataop.get_clouds(2), - 'feature_class': 'simple', - 'feature_type': 'Real', - 'label_type': 'twoclass', - 'accuracy': 1e-7 - } - feats=featop.get_features( - params['feature_class'], params['feature_type'], params['data']) - params['labels'], labels=dataop.get_labels( - feats['train'].get_num_vectors(), params['label_type']) - - lda=classifier.LDA(params['gamma'], feats['train'], labels) - lda.parallel.set_num_threads(params['num_threads']) - lda.train() - - lda.set_features(feats['test']) - params['classified']=lda.classify().get_labels() - - output=fileop.get_output(category.CLASSIFIER, params) - fileop.write(category.CLASSIFIER, output) - - -def _run_wdsvmocas (): - """Run Weighted Degree SVM Ocas classifier.""" - - svms=('WDSVMOcas',) - params={ - 'type': 'wdsvmocas', - 'degree': 1, - 'bias_enabled': False, - #'data': dataop.get_rawdna(), - 'data': dataop.get_dna( - dataop.NUM_VEC_TRAIN, dataop.NUM_VEC_TRAIN, dataop.NUM_VEC_TRAIN), - 'feature_class': 'string_complex', - 'feature_type': 'Byte', - 'alphabet': 'RAWDNA', - 'label_type': 'twoclass', - 'order': 1, - 'gap': 0, - 'reverse': False - } - feats=featop.get_features( - params['feature_class'], params['feature_type'], - params['data'], eval(params['alphabet']), - params['order'], params['gap'], params['reverse']) - _loop_svm(svms, params, feats) - - -########################################################################## -# public -########################################################################## - -def run (): - """Run generator for all classifiers.""" - -# _run_svm_kernel() - _run_svm_linear() -# _run_knn() -# _run_lda() -# _run_perceptron() -# _run_wdsvmocas() - diff --git a/tests/integration/generator/clustering.py b/tests/integration/generator/clustering.py deleted file mode 100644 index 3e7a2150a43..00000000000 --- a/tests/integration/generator/clustering.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Generator for Clustering -""" - -from numpy import matrix -from shogun.Distance import EuclideanDistance -import shogun.Clustering as clustering - -from shogun.Library import Math_init_random - -import fileop -import featop -import dataop -import category - - -def _run (name, first_arg): - """ - Run generator for a specific clustering method. - - @param name Name of the clustering method to run. - @param first_arg First argument to the clustering's constructor; so far, only this distinguishes the instantion of the different methods. - """ - - # put some constantness into randomness - Math_init_random(dataop.INIT_RANDOM) - - num_clouds=3 - params={ - 'name': 'EuclideanDistance', - 'data': dataop.get_clouds(num_clouds, 5), - 'feature_class': 'simple', - 'feature_type': 'Real' - } - feats=featop.get_features( - params['feature_class'], params['feature_type'], params['data']) - dfun=eval(params['name']) - distance=dfun(feats['train'], feats['train']) - output=fileop.get_output(category.DISTANCE, params) - - params={ - 'name': name, - 'accuracy': 1e-8, - first_arg: num_clouds - } - fun=eval('clustering.'+name) - clustering=fun(params[first_arg], distance) - clustering.train() - - if name=='KMeans': - params['radi']=clustering.get_radiuses() - params['centers']=clustering.get_cluster_centers() - elif name=='Hierarchical': - params['merge_distance']=clustering.get_merge_distances() - params['pairs']=clustering.get_cluster_pairs() - - output.update(fileop.get_output(category.CLUSTERING, params)) - fileop.write(category.CLUSTERING, output) - - -def run (): - """Run all clustering methods.""" - - _run('KMeans', 'k') - _run('Hierarchical', 'merges') diff --git a/tests/integration/generator/dataop.py b/tests/integration/generator/dataop.py deleted file mode 100644 index 02760fce968..00000000000 --- a/tests/integration/generator/dataop.py +++ /dev/null @@ -1,195 +0,0 @@ -"""Common operations on train and test data""" - -import sys -import md5 -import string -import numpy -from numpy import random, round -from shogun.Features import BinaryLabels, RegressionLabels - -NUM_VEC_TRAIN=11 -NUM_VEC_TEST=17 -LEN_SEQ=60 -INIT_RANDOM=42 -DIGITS_ROUND=7 - -_NUM_FEATS=11 - - -def get_rand (dattype=numpy.double, num_feats=_NUM_FEATS, dim_square=False, - max_train=sys.maxint, max_test=sys.maxint): - """Return random numbers. - - Return random numbers, either float or integer, in a dict with elements - 'train' and 'test'. - - @param dattype (numpy) data type of the random numbers - @param num_feats number of features for in a train/test vector - @param dim_square Dimensions of the square of random numbers, implying that number of features and vectors in both train/test are all the same. - @param max_train Maximum value for data in train vectors - @param max_test Maximum value for data in test vectors - @return Dict which contains the random numbers - """ - - - if dim_square: - num_feats=num_vec_train=num_vec_test=dim_square - else: - num_vec_train=NUM_VEC_TRAIN - num_vec_test=NUM_VEC_TEST - - if dattype==numpy.double: - return { - 'train':round(random.rand(num_feats, num_vec_train), DIGITS_ROUND), - 'test':round(random.rand(num_feats, num_vec_test), DIGITS_ROUND) - } - elif dattype==numpy.chararray: - ord_a=ord('A') - ord_z=ord('Z') - rand_train=random.randint(ord_a, ord_z, num_feats*num_vec_train) - rand_test=random.randint(ord_a, ord_z, num_feats*num_vec_test) - return { - 'train': numpy.array(map(lambda x: chr(x), rand_train)).reshape( - num_feats, num_vec_train), - 'test': numpy.array(map(lambda x: chr(x), rand_test)).reshape( - num_feats, num_vec_test) - } - else: - if dattype==numpy.ushort: - maxval=2**16-1 - if max_train>maxval: - max_train=maxval - if max_test>maxval: - max_test=maxval - - # randint does not understand arg dtype - rand_train=random.randint(0, max_train, (num_feats, num_vec_train)) - rand_test=random.randint(0, max_test, (num_feats, num_vec_test)) - return { - 'train':rand_train.astype(dattype), - 'test':rand_test.astype(dattype) - } - - -def get_clouds (num_clouds, num_feats=_NUM_FEATS): - """Return random float numbers organised, but scrambled, in clouds. - - The float numbers generated here are first created in number clouds, shifted from each other by a constant value. Then they are permutated to hide these clusters a bit when used in classifiers or clustering methods. - It is a specialised case of random float number generation. - - @param num_clouds Number of clouds to generate - @param num_features Number of features in each cloud - @return Dict which contains the random numbers - """ - - clouds={} - - data=[random.rand(num_feats, NUM_VEC_TRAIN)+x/2 for x in xrange(num_clouds)] - clouds['train']=numpy.concatenate(data, axis=1) - clouds['train']=numpy.array([random.permutation(x) for x in clouds['train']]) - clouds['train']=round(clouds['train'], DIGITS_ROUND) - - data=[random.rand(num_feats, NUM_VEC_TEST)+x/2 for x in xrange(num_clouds)] - clouds['test']=numpy.concatenate(data, axis=1) - clouds['test']=numpy.array([random.permutation(x) for x in clouds['test']]) - clouds['test']=round(clouds['test'], DIGITS_ROUND) - - return clouds - - -def get_cubes (num_train=4, num_test=8): - """Return cubes of with random emissions. - - Used by the Hidden-Markov-Model, it creates two - seemingly random sequences of emissions of a 6-sided cube. - - @param num Number of hidden cubes - @return Dict of tuples of emissions, representing a hidden cube - """ - - leng=50 - rep=5 - weight=1 - - - sequence={'train':list(), 'test':list()} - num={'train': num_train, 'test': num_test} - - for tgt in ('train', 'test'): - for i in xrange(num[tgt]): - # generate a sequence with characters 1-6 drawn from 3 loaded cubes - loaded=[] - for j in xrange(3): - draw=[x*numpy.ones((1, numpy.ceil(leng*random.rand())), int)[0] \ - for x in xrange(1, 7)] - loaded.append(random.permutation(numpy.concatenate(draw))) - - draws=[] - for j in xrange(len(loaded)): - ones=numpy.ones((1, numpy.ceil(rep*random.rand())), int) - draws=numpy.concatenate((j*ones[0], draws)) - draws=random.permutation(draws) - - seq=[] - for j in xrange(len(draws)): - len_loaded=len(loaded[draws[j]]) - weighted=int(numpy.ceil( - ((1-weight)*random.rand()+weight)*len_loaded)) - perm=random.permutation(len_loaded) - shuffled=[str(loaded[draws[j]][x]) for x in perm[:weighted]] - seq=numpy.concatenate((seq, shuffled)) - - sequence[tgt].append(''.join(seq)) - - return sequence - - -def get_labels (num, ltype='twoclass'): - """Return labels used for classification. - - @param num Number of labels - @param ltype Type of labels, either twoclass or series. - @return Tuple to contain the labels as numbers in a tuple and labels as objects digestable for Shogun. - """ - - labels=[] - if ltype=='twoclass': - labels.append(random.rand(num).round()*2-1) - # essential to wrap in array(), will segfault sometimes otherwise - labels.append(BinaryLabels(numpy.array(labels[0]))) - elif ltype=='series': - labels.append([numpy.double(x) for x in xrange(num)]) - # essential to wrap in array(), will segfault sometimes otherwise - labels.append(RegressionLabels(numpy.array(labels[0]))) - else: - return [None, None] - - return labels - - -def get_dna ( - num_vec_train=NUM_VEC_TRAIN, num_vec_test=NUM_VEC_TEST, len_seq=LEN_SEQ): - """Return a random DNA sequence. - - @param num_vec_train number of training vectors - @param num_vec_test number of test vectors - @param len_seq sequence length - @return Dict of tuples of DNA sequences. - """ - - acgt=numpy.array(['A', 'C', 'G','T']) - len_acgt=len(acgt) - - train=[ - ''.join([ - acgt[numpy.floor(len_acgt*random.rand())] for j in xrange(len_seq) - ]) for i in xrange(num_vec_train) - ] - - test=[ - ''.join([ - acgt[numpy.floor(len_acgt*random.rand())] for j in xrange(len_seq) - ]) for i in xrange(num_vec_test) - ] - - return {'train': train, 'test': test} diff --git a/tests/integration/generator/distance.py b/tests/integration/generator/distance.py deleted file mode 100644 index a7de918b9b3..00000000000 --- a/tests/integration/generator/distance.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Generator for Distance""" - -import shogun.Distance as distance - -import fileop -import dataop -import featop -import category - - -def _compute (feats, params): - """Compute a distance and gather result data. - - @param feats Train and test features - @param params dict with parameters to distance - """ - - fun=eval('distance.'+params['name']) - if params.has_key('args'): - dist=fun(feats['train'], feats['train'], *params['args']['val']) - else: - dist=fun(feats['train'], feats['train']) - dm_train=dist.get_distance_matrix() - dist.init(feats['train'], feats['test']) - dm_test=dist.get_distance_matrix() - - output={ - 'distance_matrix_train':dm_train, - 'distance_matrix_test':dm_test, - } - output.update(fileop.get_output(category.DISTANCE, params)) - - fileop.write(category.DISTANCE, output) - -def _run_feats_real (): - """Run distances with RealFeatures.""" - - params={ - 'accuracy': 1e-8, - 'feature_class': 'simple', - 'feature_type': 'Real', - 'data': dataop.get_rand() - } - feats=featop.get_features( - params['feature_class'], params['feature_type'], params['data']) - - params['name']='EuclidianDistance' - _compute(feats, params) - params['name']='CanberraMetric' - _compute(feats, params) - params['name']='ChebyshewMetric' - _compute(feats, params) - params['name']='GeodesicMetric' - _compute(feats, params) - params['name']='JensenMetric' - _compute(feats, params) - params['name']='ManhattanMetric' - _compute(feats, params) - params['name']='BrayCurtisDistance' - _compute(feats, params) - params['name']='ChiSquareDistance' - _compute(feats, params) - params['name']='CosineDistance' - _compute(feats, params) - params['name']='TanimotoDistance' - _compute(feats, params) - params['name']='ManhattanMetric' - _compute(feats, params) - params['name']='MinkowskiMetric' - params['args']={'key': ('k',), 'val': (1.3,)} - _compute(feats, params) - - params['name']='SparseEuclidianDistance' - params['accuracy']=1e-7 - del params['args'] - feats=featop.get_features( - params['feature_class'], params['feature_type'], - params['data'], sparse=True) - _compute(feats, params) - - -def _run_feats_string_complex (): - """Run distances with complex StringFeatures, like WordString.""" - - params={ - 'accuracy': 1e-7, - 'feature_class': 'string_complex', - 'feature_type': 'Word', - 'data': dataop.get_dna(num_vec_test=dataop.NUM_VEC_TRAIN+42) - } - feats=featop.get_features( - params['feature_class'], params['feature_type'], params['data']) - - params['name']='CanberraWordDistance' - _compute(feats, params) - - params['accuracy']=1e-8 - params['name']='ManhattanWordDistance' - _compute(feats, params) - - params['name']='HammingWordDistance' - params['args']={'key': ('use_sign',), 'val': (False,)} - _compute(feats, params) - params['name']='HammingWordDistance' - params['args']={'key': ('use_sign',), 'val': (True,)} - _compute(feats, params) - - -def run (): - """Run generator for all distances.""" - - _run_feats_real() - _run_feats_string_complex() - diff --git a/tests/integration/generator/distribution.py b/tests/integration/generator/distribution.py deleted file mode 100644 index 89f29a86549..00000000000 --- a/tests/integration/generator/distribution.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Generator for Distribution""" - -import numpy -import shogun.Distribution as distribution -import shogun.Features as features - -from shogun.Library import Math_init_random -from dataop import INIT_RANDOM - -import fileop -import featop -import dataop -import category - -PREFIX='distribution_' - - -def _get_derivatives (dist, num_vec): - """Return the sum of all log_derivatives of a distribution. - - @param distribution Distribution to query - @param num_vec Number of feature vectors - @return Sum of all log_derivatives - """ - - num_param=dist.get_num_model_parameters() - derivatives=0 - - for i in xrange(num_param): - for j in xrange(num_vec): - val=dist.get_log_derivative(i, j) - if val!=-numpy.inf and val!=numpy.nan: # only sparse matrix! - derivatives+=val - - return derivatives - - -def _run (name): - """Run generator for a specific distribution method. - - @param name Name of the distribtuion method - """ - - # put some constantness into randomness - Math_init_random(INIT_RANDOM) - - params={ - 'name': name, - 'accuracy': 1e-7, - 'data':dataop.get_dna(), - 'alphabet': 'DNA', - 'feature_class': 'string_complex', - 'feature_type': 'Word' - } - output=fileop.get_output(category.DISTRIBUTION, params) - feats=featop.get_features( - params['feature_class'], params['feature_type'], params['data']) - - dfun=eval('distribution.'+name) - dist=dfun(feats['train']) - dist.train() - - output[PREFIX+'likelihood']=dist.get_log_likelihood_sample() - output[PREFIX+'derivatives']=_get_derivatives( - dist, feats['train'].get_num_vectors()) - - fileop.write(category.DISTRIBUTION, output) - - -def _run_hmm (): - """Run generator for Hidden-Markov-Model.""" - - # put some constantness into randomness - Math_init_random(INIT_RANDOM) - - num_examples=4 - params={ - 'name': 'HMM', - 'accuracy': 1e-6, - 'N': 3, - 'M': 6, - 'num_examples': num_examples, - 'pseudo': 1e-10, - 'order': 1, - 'alphabet': 'CUBE', - 'feature_class': 'string_complex', - 'feature_type': 'Word', - 'data': dataop.get_cubes(num_examples, 1) - } - output=fileop.get_output(category.DISTRIBUTION, params) - - feats=featop.get_features( - params['feature_class'], params['feature_type'], params['data'], - eval('features.'+params['alphabet']), params['order']) - - hmm=distribution.HMM( - feats['train'], params['N'], params['M'], params['pseudo']) - hmm.train() - hmm.baum_welch_viterbi_train(distribution.BW_NORMAL) - - output[PREFIX+'likelihood']=hmm.get_log_likelihood_sample() - output[PREFIX+'derivatives']=_get_derivatives( - hmm, feats['train'].get_num_vectors()) - - output[PREFIX+'best_path']=0 - output[PREFIX+'best_path_state']=0 - for i in xrange(num_examples): - output[PREFIX+'best_path']+=hmm.best_path(i) - for j in xrange(params['N']): - output[PREFIX+'best_path_state']+=hmm.get_best_path_state(i, j) - - fileop.write(category.DISTRIBUTION, output) - - -def run (): - """Run generator for all distribution methods.""" - - _run('Histogram') - _run('LinearHMM') - _run_hmm() diff --git a/tests/integration/generator/featop.py b/tests/integration/generator/featop.py deleted file mode 100644 index 406081d3376..00000000000 --- a/tests/integration/generator/featop.py +++ /dev/null @@ -1,145 +0,0 @@ -""" -Common operations on features -""" - -import shogun.Features as features -import shogun.Preprocessor as preproc - -WORDSTRING_ORDER=3 -WORDSTRING_GAP=0 -WORDSTRING_REVERSE=False - - -def get_features(fclass, ftype, data, *args, **kwargs): - if fclass=='simple': - return get_simple(ftype, data, *args, **kwargs) - elif fclass=='string': - return get_string(ftype, data, *args, **kwargs) - elif fclass=='string_complex': - return get_string_complex(ftype, data, *args, **kwargs) - elif fclass=='wd': - return get_wd(data, *args, **kwargs) - else: - raise ValueError, 'Unknown feature class %s.'%fclass - - -def get_simple (ftype, data, alphabet=features.DNA, sparse=False): - """Return SimpleFeatures. - - @param ftype Feature type, e.g. Real, Byte - @param data Train/test data for feature creation - @param alphabet Alphabet for feature creation - @param sparse Is feature sparse? - @return Dict with SimpleFeatures train/test - """ - - if ftype=='Byte' or ftype=='Char': - train=eval('features.'+ftype+'Features(alphabet)') - test=eval('features.'+ftype+'Features(alphabet)') - train.copy_feature_matrix(data['train']) - test.copy_feature_matrix(data['test']) - - else: - train=eval('features.'+ftype+"Features(data['train'])") - test=eval('features.'+ftype+"Features(data['test'])") - - if sparse: - sparse_train=eval('features.Sparse'+ftype+'Features()') - sparse_train.obtain_from_simple(train) - - sparse_test=eval('features.Sparse'+ftype+'Features()') - sparse_test.obtain_from_simple(test) - - return {'train':sparse_train, 'test':sparse_test} - else: - return {'train':train, 'test':test} - - -def get_string (ftype, data, alphabet=features.DNA): - """Return StringFeatures. - - @param ftype Feature type, e.g. Real, Byte - @param data Train/test data for feature creation - @param alphabet Alphabet for feature creation - @return Dict with StringFeatures train/test - """ - - train=eval('features.String'+ftype+"Features(data['train'], alphabet)") - test=eval('features.String'+ftype+"Features(data['test'], alphabet)") - return {'train':train, 'test':test} - - -def get_string_complex (ftype, data, alphabet=features.DNA, - order=WORDSTRING_ORDER, gap=WORDSTRING_GAP, reverse=WORDSTRING_REVERSE): - """Return complex StringFeatures. - - @param ftype Feature type, e.g. RealFeature, ByteFeature - @param data Train/test data for feature creation - @param alphabet Alphabet for feature creation - @param order Order of the feature - @param gap Gap of the feature - @param reverse Is feature reverse? - @return Dict with complex StringFeatures train/test - """ - - feats={} - - charfeat=features.StringCharFeatures(data['train'], alphabet) - feat=eval('features.String'+ftype+'Features(alphabet)') - feat.obtain_from_char(charfeat, order-1, order, gap, reverse) - feats['train']=feat - - charfeat=features.StringCharFeatures(data['test'], alphabet) - feat=eval('features.String'+ftype+'Features(alphabet)') - feat.obtain_from_char(charfeat, order-1, order, gap, reverse) - feats['test']=feat - - if ftype=='Word' or ftype=='Ulong': - name='Sort'+ftype+'String' - return add_preproc(name, feats) - else: - return feats - - -def get_wd (data, order=WORDSTRING_ORDER): - """Return WDFeatures. - - @param data Train/test data for feature creation - @param order Order of the feature - @return Dict with WDFeatures train/test - """ - - feats={} - - charfeat=features.StringCharFeatures(data['train'], features.DNA) - bytefeat=features.StringByteFeatures(features.RAWDNA) - bytefeat.obtain_from_char(charfeat, 0, 1, 0, False) - feats['train']=features.WDFeatures(bytefeat, order, order) - - charfeat=features.StringCharFeatures(data['test'], features.DNA) - bytefeat=features.StringByteFeatures(features.RAWDNA) - bytefeat.obtain_from_char(charfeat, 0, 1, 0, False) - feats['test']=features.WDFeatures(bytefeat, order, order) - - return feats - - -def add_preproc (name, feats, *args): - """Add a preprocessor to the given features. - - @param name Name of the preprocessor - @param feats Features train/test - @param *args Variable argument list of the preprocessor - @return Dict with features having a preprocessor applied - """ - - fun=eval('preproc.'+name) - preproc=fun(*args) - preproc.init(feats['train']) - feats['train'].add_preprocessor(preproc) - feats['train'].apply_preprocessor() - feats['test'].add_preprocessor(preproc) - feats['test'].apply_preprocessor() - - return feats - diff --git a/tests/integration/generator/fileop.py b/tests/integration/generator/fileop.py deleted file mode 100644 index a829ec63b63..00000000000 --- a/tests/integration/generator/fileop.py +++ /dev/null @@ -1,424 +0,0 @@ -"""Common operations related to file handling""" - -import os -from numpy import int, long, float, double, ushort, uint16, ubyte, short, matrix, int32, int64, uint32, uint64 - -import featop -import dataop -import category - -DIR_OUTPUT='data' -EXT_OUTPUT='.m' - - -def _get_matrix (name, kmatrix): - """Converts a numpy matrix into a matrix digestable by e.g. matlab. - - @param name Name of the matrix - @param kmatrix The matrix - @return String which contains a matrix digestable by e.g. matlab - """ - - line=list() - list_numbers=(int, long, float, double, ubyte, ushort, short, uint16, int32, int64, uint32, uint64) - matrix=[] - is_string=True - - try: - # assume, all elements have same data type - if isinstance(kmatrix[0, 0], list_numbers): - is_string=False - - for x in range(kmatrix.shape[0]): - for y in range(kmatrix.shape[1]): - if is_string: - line.append("'%s'" % kmatrix[x, y]) - else: - line.append('%.9g' % kmatrix[x, y]) - matrix.append(', '.join(line)) - line=list() - except IndexError: - if isinstance(kmatrix[0], list_numbers): - is_string=False - - for x in range(kmatrix.shape[0]): - if is_string: - line.append("'%s'" % kmatrix[x]) - else: - line.append('%.9g' % kmatrix[x]) - matrix.append(', '.join(line)) - line=list() - - matrix=';'.join(matrix) - - if is_string: - matrix=''.join([name, ' = {', matrix, '}']) - else: - matrix=''.join([name, ' = [', matrix, ']']) - - return matrix.replace('\n', '') - - -def _is_excluded_from_filename (key): - """ - Determine if given key's value shall not be part of the filename. - - @param key name of the value to check for. - @return true if to be excluded, false otherwise - """ - - if (key.find('feature_')!=-1 or - key.find('accuracy')!=-1 or - key.find('data_')!=-1 or - key.find('normalizer')!=-1 or - key=='name' or - key=='init_random' or - key=='regression_type' or - key=='regression_bias' or - key=='regression_alpha_sum' or - key=='regression_sv_sum' or - key=='distribution_likelihood' or - key=='distribution_derivatives' or - key=='distribution_best_path' or - key=='distribution_best_path_state' or - key=='classifier_bias' or - key=='classifier_label_type' or - key=='classifier_alpha_sum' or - key=='classifier_sv_sum' or - key=='classifier_type'): - return True - else: - return False - - -def _get_filename (catID, out): - """ - Return filename for testcase data's output. - - @param catID ID of the category - @param out data to be written into file - @return string with the filename - """ - - params=[] - - name=category.get_name(catID) - for key, val in out.iteritems(): - if _is_excluded_from_filename(key): - continue - if key==name: - continue - - cname=val.__class__.__name__ - if cname=='bool' or cname=='float' or cname=='int' or cname=='str': - val=str(val) - val=val.replace('-', 'n') - val=val.replace('+', 'p') - params.append(val) - - params='_'.join(params).replace('.', '') - if len(params)>0: - params='_'+params - else: # otherwise problems with one interface (FIXME: find out which) - params='_fnord' - - dir=DIR_OUTPUT+os.sep+category.get_as_string(catID)+os.sep - return dir+out[name]+params+EXT_OUTPUT - - -def _loop_args(args, prefix): - """ - Loop through all arguments in given dict and add to appropriately to - output data. - - @param params various clustering parameters - @param prefix prefix for parameter's name, e.g. 'preproc_' - @return dict containing testcase data ready to be written to file - """ - - out={} - for i in xrange(len(args['key'])): - try: - name=prefix+'arg'+str(i)+'_'+args['key'][i] - except IndexError: - break - - cname=args['val'][i].__class__.__name__ - if cname=='int' or cname=='float' or cname=='bool' or cname=='str': - out[name]=args['val'][i] - else: - out[name]=cname - - return out - - -def _get_output_classifier (params, prefix=''): - """ - Classifier-specific gathering of output data - - @param params various classifier parameters - @param prefix prefix for parameter's name, e.g. 'classifier_' - @return dict containing testcase data ready to be written to file - """ - - out={} - for key, val in params.iteritems(): - if key!='data' and val is not None: - out[prefix+key]=val - - return out - - -def _get_output_clustering (params, prefix=''): - """ - Clustering-specific gathering of output data - - @param params various clustering parameters - @param prefix prefix for parameter's name, e.g. 'clustering_' - @return dict containing testcase data ready to be written to file - """ - - out={} - for key, val in params.iteritems(): - out[prefix+key]=val - - return out - - -def _get_output_distance (params, prefix=''): - """ - Distance-specific gathering of outdata - - @param params various distance parameters - @param prefix prefix for parameter's name, e.g. 'subkernel' - @return dict containing testcase data ready to be written to file - """ - - if not params.has_key('args'): - return {} - - return _loop_args(params['args'], prefix) - - -def _get_output_distribution (params, prefix=''): - """ - Clustering-specific gathering of output data - - @param params various clustering parameters - @param prefix prefix for parameter's name, e.g. 'clustering_' - @return dict containing testcase data ready to be written to file - """ - - out={} - for key, val in params.iteritems(): - if key!='data' and val is not None: - out[prefix+key]=val - - return out - - -def _get_output_kernel (params, prefix=''): - """ - Kernel-specific gathering of outdata - - @param params various kernel parameters - @param prefix prefix for parameter's name, e.g. 'subkernel' - @return dict containing testcase data ready to be written to file - """ - - out={} - - if params.has_key('normalizer'): - out[prefix+'normalizer']=params['normalizer'].__class__.__name__ - - if not params.has_key('args'): - return out - - if params['name']=='AUC': - # remove element 'subkernel' - params['args']['key']=(params['args']['key'][0],) - - out.update(_loop_args(params['args'], prefix)) - return out - - -def _get_output_preproc (params, prefix=''): - """ - Preproc-specific gathering of output data - - @param params various preproc parameters - @param prefix prefix for parameter's name, e.g. 'preproc_' - @return dict containing testcase data ready to be written to file - """ - - if not params.has_key('args'): - return {} - - return _loop_args(params['args'], prefix) - - -def _get_output_regression (params, prefix=''): - """ - Regression-specific gathering of output data - - @param params various regression parameters - @param prefix prefix for parameter's name, e.g. 'regression_' - @return dict containing testcase data ready to be written to file - """ - - out={} - for key, val in params.iteritems(): - out[prefix+key]=val - - return out - - - -############################################################################ -# public -############################################################################ - -def write (catID, out): - """ - Write given testcase data to a file. - - @param cat ID of the category, like category.KERNEL - @param out data to be written into file - @return Success of operation - """ - - fnam=_get_filename(catID, out) - print 'Writing for '+category.get_as_string(catID).upper()+': '+ \ - os.path.basename(fnam) - - dirname=os.path.dirname(fnam) - if not os.path.isdir(dirname): - os.mkdir(dirname) - - mfile=open(fnam, mode='w') - for key, val in out.iteritems(): - cname=val.__class__.__name__ - if cname=='bool' or cname=='str': - mfile.write("%s = '%s';\n"%(key, val)) - elif cname=='ndarray' or cname=='matrix': - mfile.write("%s;\n"%_get_matrix(key, val)) - else: - mfile.write("%s = %s;\n"%(key, val)) - mfile.close() - - return True - - -def clean_dir_outdata (): - """ - Remove all old testdata files. - - @return Success of operation - """ - - success=True - - for dname in os.listdir(DIR_OUTPUT): - if dname=='.svn' or not os.path.isdir(DIR_OUTPUT+os.sep+dname): - continue - - for fname in os.listdir(DIR_OUTPUT+os.sep+dname): - if not fname.endswith(EXT_OUTPUT): - continue - - target=DIR_OUTPUT+os.sep+dname+os.sep+fname - if os.path.exists(target): - os.remove(target) - # os.remove returns False on removal??? - #print 'Could not remove file "%s"'%target - #success=False - - return success - - -def get_output (catID, params, prefix=''): - """ - Return output data to be written into the testcase's file. - - After computations, the gathered data is structured and - put into one data structure which can conveniently be written to a - file that will represent the testcase. - - @param catID ID of entity's category, e.g. category.DISTANCE - @param params hash with parameters to entity - @param prefix prefix for parameter's name, e.g. 'subkernel' - @return Dict containing testcase data ready to be written to file - """ - - out={} - prefix=category.get_as_string(catID)+'_'+prefix - if catID==category.CLASSIFIER: - out=_get_output_classifier(params, prefix) - elif catID==category.CLUSTERING: - out=_get_output_distribution(params, prefix) - elif catID==category.DISTANCE: - out=_get_output_distance(params, prefix) - elif catID==category.DISTRIBUTION: - out=_get_output_distribution(params, prefix) - elif catID==category.KERNEL: - out=_get_output_kernel(params, prefix) - elif catID==category.PREPROC: - out=_get_output_preproc(params, prefix) - elif catID==category.REGRESSION: - out=_get_output_regression(params, prefix) - else: - return out - - out[prefix+'name']=params['name'] - if params.has_key('accuracy'): - out[prefix+'accuracy']=params['accuracy'] - - # example data - if params.has_key('data'): - out[prefix+'data_train']=matrix(params['data']['train']) - out[prefix+'data_test']=matrix(params['data']['test']) - - - # params wrt feature class & type - if params.has_key('feature_class'): - fclass=params['feature_class'] - ftype=params['feature_type'] - out[prefix+'feature_class']=fclass - out[prefix+'feature_type']=ftype - - if fclass=='string' or (fclass=='simple' and ftype=='Char'): - if params.has_key('alphabet'): - out[prefix+'alphabet']=params['alphabet'] - else: - out[prefix+'alphabet']='DNA' - out[prefix+'seqlen']=dataop.LEN_SEQ - - elif fclass=='simple' and ftype=='Byte': - out[prefix+'alphabet']='RAWBYTE' - out[prefix+'seqlen']=dataop.LEN_SEQ - - elif fclass=='string_complex': - if params.has_key('alphabet'): - out[prefix+'alphabet']=params['alphabet'] - else: - out[prefix+'alphabet']='DNA' - if params.has_key('order'): - out[prefix+'order']=params['order'] - else: - out[prefix+'order']=featop.WORDSTRING_ORDER - if params.has_key('gap'): - out[prefix+'gap']=params['gap'] - else: - out[prefix+'gap']=featop.WORDSTRING_GAP - if params.has_key('reverse'): - out[prefix+'reverse']=params['reverse'] - else: - out[prefix+'reverse']=featop.WORDSTRING_REVERSE - if params.has_key('seqlen'): - out[prefix+'seqlen']=params['seqlen'] - else: - out[prefix+'seqlen']=dataop.LEN_SEQ - - out['init_random']=dataop.INIT_RANDOM - - return out diff --git a/tests/integration/generator/kernel.py b/tests/integration/generator/kernel.py deleted file mode 100644 index 68581e1bf21..00000000000 --- a/tests/integration/generator/kernel.py +++ /dev/null @@ -1,607 +0,0 @@ -""" -Generator for Kernel - -A word about args: it is organised as two correlated tuples, because the -order of the elements of dicts in Python is arbitrary, meaning that the item -added first might be the last when iterating over the dict. -""" - -import numpy -import shogun.Kernel as kernel -from shogun.Features import CombinedFeatures, TOPFeatures, FKFeatures, CUBE, RAWBYTE -from shogun.Classifier import PluginEstimate -from shogun.Distance import CanberraMetric -from shogun.Distribution import HMM, LinearHMM, BW_NORMAL -from shogun.Library import Math_init_random - -import fileop -import featop -import dataop -import category - - -################################################################## -## compute funcs -################################################################## - -def _compute_pie (feats, params): - """Compute a kernel with PluginEstimate. - - @param feats kernel features - @param params dict containing various kernel parameters - """ - - output=fileop.get_output(category.KERNEL, params) - - lab, labels=dataop.get_labels(feats['train'].get_num_vectors()) - output['classifier_labels']=lab - pie=PluginEstimate() - pie.set_labels(labels) - pie.set_features(feats['train']) - pie.train() - - kfun=eval('kernel.'+params['name']+'Kernel') - kern=kfun(feats['train'], feats['train'], pie) - output['kernel_matrix_train']=kern.get_kernel_matrix() - kern.init(feats['train'], feats['test']) - pie.set_features(feats['test']) - output['kernel_matrix_test']=kern.get_kernel_matrix() - - classified=pie.apply().get_labels() - output['classifier_classified']=classified - - fileop.write(category.KERNEL, output) - - -def _compute_top_fisher (feats, pout): - """Compute PolyKernel with TOP or FKFeatures - - @param feats features of the kernel - @param pout previously gathered data ready to be written to file - """ - - params={ - 'name': 'Poly', - 'accuracy': 1e-6, - 'args': { - 'key': ('size', 'degree', 'inhomogene'), - 'val': (10, 1, False) - } - } - output=fileop.get_output(category.KERNEL, params) - output.update(pout) - - kfun=eval('kernel.'+params['name']+'Kernel') - kern=kfun(feats['train'], feats['train'], *params['args']['val']) - output['kernel_matrix_train']=kern.get_kernel_matrix() - kern.init(feats['train'], feats['test']) - output['kernel_matrix_test']=kern.get_kernel_matrix() - - fileop.write(category.KERNEL, output) - - -def _compute (feats, params, pout=None): - """ - Compute a kernel and write gathered data to file. - - @param name name of the kernel - @param feats features of the kernel - @param params dict with parameters to kernel - @param pout previously gathered data ready to be written to file - """ - - output=fileop.get_output(category.KERNEL, params) - if pout: - output.update(pout) - - kfun=eval('kernel.'+params['name']+'Kernel') - if params.has_key('args'): - kern=kfun(*params['args']['val']) - else: - kern=kfun() - - if params.has_key('normalizer'): - kern.set_normalizer(params['normalizer']) - kern.init(feats['train'], feats['train']) - - output['kernel_matrix_train']=kern.get_kernel_matrix() - kern.init(feats['train'], feats['test']) - output['kernel_matrix_test']=kern.get_kernel_matrix() - - fileop.write(category.KERNEL, output) - - -################################################################## -## run funcs -################################################################## - -def _run_auc (): - """Run AUC kernel.""" - - # handle subkernel - params={ - 'name': 'Gaussian', - 'data': dataop.get_rand(), - 'feature_class': 'simple', - 'feature_type': 'Real', - 'args': {'key': ('size', 'width'), 'val': (10, 1.7)} - } - feats=featop.get_features( - params['feature_class'], params['feature_type'], params['data']) - subk=kernel.GaussianKernel(*params['args']['val']) - subk.init(feats['train'], feats['test']) - output=fileop.get_output(category.KERNEL, params, 'subkernel0_') - - # handle AUC - params={ - 'name': 'AUC', - 'data': dataop.get_rand(numpy.ushort, num_feats=2, - max_train=dataop.NUM_VEC_TRAIN, max_test=dataop.NUM_VEC_TEST), - 'feature_class': 'simple', - 'feature_type': 'Word', - 'accuracy': 1e-8, - 'args': {'key': ('size', 'subkernel'), 'val': (10, subk)} - } - feats=featop.get_features( - params['feature_class'], params['feature_type'], params['data']) - _compute(feats, params, output) - - -def _run_combined (): - """Run Combined kernel.""" - - kern=kernel.CombinedKernel() - feats={'train': CombinedFeatures(), 'test': CombinedFeatures()} - output={} - params={ - 'name': 'Combined', - 'accuracy': 1e-7 - } - subkdata=[ - { - 'name': 'FixedDegreeString', - 'feature_class': 'string', - 'feature_type': 'Char', - 'args': {'key': ('size', 'degree'), 'val': (10, 3)} - }, - { - 'name': 'PolyMatchString', - 'feature_class': 'string', - 'feature_type': 'Char', - 'args': { - 'key': ('size', 'degree', 'inhomogene'), - 'val': (10, 3, True) - } - }, - { - 'name': 'LocalAlignmentString', - 'feature_class': 'string', - 'feature_type': 'Char', - 'args': {'key': ('size',), 'val': (10,)} - } - ] - - i=0 - for sd in subkdata: - kfun=eval('kernel.'+sd['name']+'Kernel') - subk=kfun(*sd['args']['val']) - sd['data']=dataop.get_dna() - subkfeats=featop.get_features( - sd['feature_class'], sd['feature_type'], sd['data']) - output.update( - fileop.get_output(category.KERNEL, sd, 'subkernel'+str(i)+'_')) - - kern.append_kernel(subk) - feats['train'].append_feature_obj(subkfeats['train']) - feats['test'].append_feature_obj(subkfeats['test']) - - i+=1 - - output.update(fileop.get_output(category.KERNEL, params)) - kern.init(feats['train'], feats['train']) - output['kernel_matrix_train']=kern.get_kernel_matrix() - kern.init(feats['train'], feats['test']) - output['kernel_matrix_test']=kern.get_kernel_matrix() - - fileop.write(category.KERNEL, output) - - -def _run_subkernels (): - """Run all kernels handling subkernels.""" - - _run_auc() - _run_combined() - - -def _run_custom (): - """Run Custom kernel.""" - - params={ - 'name': 'Custom', - 'accuracy': 1e-7, - 'feature_class': 'simple', - 'feature_type': 'Real' - } - dim_square=7 - data=dataop.get_rand(dim_square=dim_square) - feats=featop.get_features( - params['feature_class'], params['feature_type'], data) - data=data['train'] - symdata=data+data.T - - lowertriangle=numpy.array([symdata[(x,y)] for x in xrange(symdata.shape[1]) - for y in xrange(symdata.shape[0]) if y<=x]) - kern=kernel.CustomKernel() - #kern.init(feats['train'], feats['train'] - kern.set_triangle_kernel_matrix_from_triangle(lowertriangle) - km_triangletriangle=kern.get_kernel_matrix() - kern.set_triangle_kernel_matrix_from_full(symdata) - km_fulltriangle=kern.get_kernel_matrix() - kern.set_full_kernel_matrix_from_full(data) - km_fullfull=kern.get_kernel_matrix() - - output={ - 'kernel_matrix_triangletriangle': km_triangletriangle, - 'kernel_matrix_fulltriangle': km_fulltriangle, - 'kernel_matrix_fullfull': km_fullfull, - 'kernel_symdata': numpy.matrix(symdata), - 'kernel_data': numpy.matrix(data), - 'kernel_dim_square': dim_square - } - output.update(fileop.get_output(category.KERNEL, params)) - - fileop.write(category.KERNEL, output) - - -def _run_distance (): - """Run distance kernel.""" - - params={ - 'name': 'Distance', - 'accuracy': 1e-9, - 'feature_class': 'simple', - 'feature_type': 'Real', - 'data': dataop.get_rand(), - 'args': { - 'key': ('size', 'width', 'distance'), - 'val': (10, 1.7, CanberraMetric()) - } - } - feats=featop.get_features( - params['feature_class'], params['feature_type'], params['data']) - - _compute(feats, params) - - -def _run_feats_byte (): - """Run kernel with ByteFeatures.""" - - params={ - 'name': 'Linear', - 'accuracy': 1e-8, - 'feature_class': 'simple', - 'feature_type': 'Byte', - 'data': dataop.get_rand(dattype=numpy.ubyte), - 'normalizer': kernel.AvgDiagKernelNormalizer() - } - feats=featop.get_features(params['feature_class'], params['feature_type'], - params['data'], RAWBYTE) - - _compute(feats, params) - - -def _run_mindygram (): - """Run Mindygram kernel.""" - return - - params={ - 'name': 'MindyGram', - 'accuracy': 1e-8, - 'data': dataop.get_dna(), - 'feature_class': 'mindy', - 'args': {'key': ('measure', 'width'), 'val': ('MEASURE', 1.5)} - } - feats={ - 'train': MindyGramFeatures('DNA', 'freq', '%20.,', 0), - 'test': MindyGramFeatures('DNA', 'freq', '%20.,', 0) - } - - _compute(feats, params) - - -def _run_feats_real (): - """Run kernel with RealFeatures.""" - - params={ - 'data': dataop.get_rand(), - 'accuracy': 1e-8, - 'feature_class': 'simple', - 'feature_type': 'Real' - } - feats=featop.get_features( - params['feature_class'], params['feature_type'], params['data']) - sparsefeats=featop.get_features( - params['feature_class'], params['feature_type'], - params['data'], sparse=True) - - params['name']='Gaussian' - params['args']={'key': ('size', 'width',), 'val': (10, 1.3)} - _compute(feats, params) - - params['name']='GaussianShift' - params['args']={ - 'key': ('size', 'width', 'max_shift', 'shift_step'), - 'val': (10, 1.3, 2, 1) - } - _compute(feats, params) - - params['name']='Gaussian' - params['args']={'key': ('size', 'width'), 'val': (10, 1.7)} - _compute(sparsefeats, params) - - params['accuracy']=0 - params['name']='Const' - params['args']={'key': ('c',), 'val': (23.,)} - _compute(feats, params) - - params['name']='Diag' - params['args']={'key': ('size', 'diag'), 'val': (10, 23.)} - _compute(feats, params) - - params['accuracy']=1e-9 - params['name']='Sigmoid' - params['args']={ - 'key': ('size', 'gamma', 'coef0'), - 'val': (10, 1.1, 1.3) - } - _compute(feats, params) - params['args']['val']=(10, 0.5, 0.7) - _compute(feats, params) - - params['name']='Chi2' - params['args']={'key': ('size', 'width'), 'val': (10, 1.2)} - _compute(feats, params) - - params['accuracy']=1e-8 - params['name']='Poly' - params['args']={ - 'key': ('size', 'degree', 'inhomogene'), - 'val': (10, 3, True) - } - _compute(sparsefeats, params) - params['args']['val']=(10, 3, False) - _compute(sparsefeats, params) - - params['name']='Poly' - params['normalizer']=kernel.SqrtDiagKernelNormalizer() - params['args']={ - 'key': ('size', 'degree', 'inhomogene'), - 'val': (10, 3, True) - } - _compute(feats, params) - params['args']['val']=(10, 3, False) - _compute(feats, params) - - params['normalizer']=kernel.AvgDiagKernelNormalizer() - del params['args'] - params['name']='Linear' - _compute(feats, params) - params['name']='Linear' - _compute(sparsefeats, params) - - -def _run_feats_string (): - """Run kernel with StringFeatures.""" - - params = { - 'accuracy': 1e-9, - 'data': dataop.get_dna(), - 'feature_class': 'string', - 'feature_type': 'Char', - } - feats=featop.get_features( - params['feature_class'], params['feature_type'], params['data']) - - params['name']='FixedDegreeString' - params['args']={'key': ('size', 'degree'), 'val': (10, 3)} - _compute(feats, params) - - params['accuracy']=1e-9 - params['name']='LocalAlignmentString' - params['args']={'key': ('size',), 'val': (10,)} - _compute(feats, params) - - params['accuracy']=1e-10 - params['name']='PolyMatchString' - params['args']={ - 'key': ('size', 'degree', 'inhomogene'), - 'val': (10, 3, True) - } - _compute(feats, params) - params['args']['val']=(10, 3, False) - _compute(feats, params) - - params['accuracy']=1e-15 - params['name']='SimpleLocalityImprovedString' - params['args']={ - 'key': ('size', 'length', 'inner_degree', 'outer_degree'), - 'val': (10, 5, 7, 5) - } - _compute(feats, params) - # buggy: - #params['name']='LocalityImprovedString' - #_compute(feats, params) - - params['name']='WeightedDegreeString' - params['accuracy']=1e-9 - params['args']={'key': ('degree',), 'val': (20,)} - _compute(feats, params) - params['args']={'key': ('degree',), 'val': (1,)} - _compute(feats, params) - - params['name']='WeightedDegreePositionString' - params['args']={'key': ('size', 'degree'), 'val': (10, 20)} - _compute(feats, params) - params['args']={'key': ('size', 'degree'), 'val': (10, 1)} - _compute(feats, params) - - params['name']='OligoString' - params['args']={'key': ('size', 'k', 'width'), 'val': (10, 3, 1.2)} - _compute(feats, params) - params['args']={'key': ('size', 'k', 'width'), 'val': (10, 4, 1.7)} - _compute(feats, params) - - params['name']='LinearString' - params['accuracy']=1e-8 - params['normalizer']=kernel.AvgDiagKernelNormalizer() - del params['args'] - _compute(feats, params) - - -def _run_feats_word (): - """Run kernel with WordFeatures.""" - - maxval=42 - params={ - 'name': 'Linear', - 'accuracy': 1e-8, - 'feature_class': 'simple', - 'feature_type': 'Word', - 'data': dataop.get_rand( - dattype=numpy.ushort, max_train=maxval, max_test=maxval), - 'normalizer': kernel.AvgDiagKernelNormalizer() - } - feats=featop.get_features( - params['feature_class'], params['feature_type'], params['data']) - - _compute(feats, params) - - -def _run_feats_string_complex (): - """Run kernel with complex StringFeatures.""" - - params={ - 'data': dataop.get_dna(), - 'feature_class': 'string_complex' - } - - params['feature_type']='Word' - wordfeats=featop.get_features( - params['feature_class'], params['feature_type'], params['data']) - - params['name']='CommWordString' - params['accuracy']=1e-9 - params['args']={'key': ('size', 'use_sign'), 'val': (10, False)} - _compute(wordfeats, params) - params['name']='WeightedCommWordString' - _compute(wordfeats, params) - - params['name']='PolyMatchWordString' - params['accuracy']=1e-10 - params['args']={ - 'key': ('size', 'degree', 'inhomogene'), - 'val': (10, 3, True) - } - _compute(wordfeats, params) - params['args']['val']=(10, 3, False) - _compute(wordfeats, params) - - params['name']='MatchWordString' - params['args']={'key': ('size', 'degree'), 'val': (10, 3)} - _compute(wordfeats, params) - - params['feature_type']='Ulong' - params['accuracy']=1e-9 - ulongfeats=featop.get_features( - params['feature_class'], params['feature_type'], params['data']) - params['name']='CommUlongString' - params['args']={'key': ('size', 'use_sign'), 'val': (10, False)} - _compute(ulongfeats, params) - - -def _run_pie (): - """Run kernel with PluginEstimate.""" - - params={ - 'data': dataop.get_dna(), - 'accuracy': 1e-6, - 'feature_class': 'string_complex', - 'feature_type': 'Word' - } - feats=featop.get_features( - params['feature_class'], params['feature_type'], params['data']) - - params['name']='HistogramWordString' - _compute_pie(feats, params) - params['name']='SalzbergWordString' - _compute_pie(feats, params) - - -def _run_top_fisher (): - """Run Linear Kernel with {Top,Fisher}Features.""" - - # put some constantness into randomness - Math_init_random(dataop.INIT_RANDOM) - - data=dataop.get_cubes(4, 8) - prefix='topfk_' - params={ - prefix+'N': 3, - prefix+'M': 6, - prefix+'pseudo': 1e-1, - prefix+'order': 1, - prefix+'gap': 0, - prefix+'reverse': False, - prefix+'alphabet': 'CUBE', - prefix+'feature_class': 'string_complex', - prefix+'feature_type': 'Word', - prefix+'data_train': numpy.matrix(data['train']), - prefix+'data_test': numpy.matrix(data['test']) - } - - wordfeats=featop.get_features( - params[prefix+'feature_class'], params[prefix+'feature_type'], - data, eval(params[prefix+'alphabet']), - params[prefix+'order'], params[prefix+'gap'], params[prefix+'reverse']) - pos_train=HMM(wordfeats['train'], - params[prefix+'N'], params[prefix+'M'], params[prefix+'pseudo']) - pos_train.train() - pos_train.baum_welch_viterbi_train(BW_NORMAL) - neg_train=HMM(wordfeats['train'], - params[prefix+'N'], params[prefix+'M'], params[prefix+'pseudo']) - neg_train.train() - neg_train.baum_welch_viterbi_train(BW_NORMAL) - pos_test=HMM(pos_train) - pos_test.set_observations(wordfeats['test']) - neg_test=HMM(neg_train) - neg_test.set_observations(wordfeats['test']) - feats={} - - feats['train']=TOPFeatures(10, pos_train, neg_train, False, False) - feats['test']=TOPFeatures(10, pos_test, neg_test, False, False) - params[prefix+'name']='TOP' - _compute_top_fisher(feats, params) - - feats['train']=FKFeatures(10, pos_train, neg_train) - feats['train'].set_opt_a(-1) #estimate prior - feats['test']=FKFeatures(10, pos_test, neg_test) - feats['test'].set_a(feats['train'].get_a()) #use prior from training data - params[prefix+'name']='FK' - _compute_top_fisher(feats, params) - - -def run (): - """Run generator for all kernels.""" - - #_run_mindygram() - _run_top_fisher() - _run_pie() - _run_custom() - _run_distance() - _run_subkernels() - - _run_feats_byte() - _run_feats_real() - _run_feats_string() - _run_feats_string_complex() - _run_feats_word() diff --git a/tests/integration/generator/preproc.py b/tests/integration/generator/preproc.py deleted file mode 100644 index 62bdd3de8cf..00000000000 --- a/tests/integration/generator/preproc.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Generator for Preprocessors""" - -import shogun.Library as library -from shogun.Kernel import GaussianKernel, CommWordStringKernel, \ - CommUlongStringKernel - -import fileop -import featop -import dataop -import category - - -def _compute (feats, params): - """Perform computations on kernel using preprocessors. - - @param name name of the kernel - @param feats features of the kernel - @return dict of testcase data ready to be written to file - """ - - output=fileop.get_output(category.KERNEL, params) - - fun=eval(params['name']+'Kernel') - if params.has_key('args'): - kernel=fun(feats['train'], feats['train'], *params['args']['val']) - else: - kernel=fun(feats['train'], feats['train']) - - output['kernel_matrix_train']=kernel.get_kernel_matrix() - kernel.init(feats['train'], feats['test']) - output['kernel_matrix_test']=kernel.get_kernel_matrix() - - return output - - -def _run_string_complex (ftype): - """Run preprocessor applied on complex StringFeatures. - - @param ftype Feature type, like Word - """ - - params={ - 'name': 'Comm'+ftype+'String', - 'accuracy': 1e-9, - 'feature_class': 'string_complex', - 'feature_type': ftype, - 'data': dataop.get_dna() - } - feats=featop.get_features( - params['feature_class'], params['feature_type'], params['data']) - # string_complex gets preproc added implicitely on Word/Ulong feats - output=_compute(feats, params) - - params={ - 'name': 'Sort'+ftype+'String' - } - output.update(fileop.get_output(category.PREPROC, params)) - - fileop.write(category.PREPROC, output) - - -def _run_real (name, args=None): - """Run preprocessor applied on RealFeatures. - - @param name name of the preprocessor - @param args argument list (in a dict) for the preprocessor - """ - - params={ - 'name': 'Gaussian', - 'accuracy': 1e-8, - 'data': dataop.get_rand(), - 'feature_class': 'simple', - 'feature_type': 'Real', - 'args': {'key': ('width',), 'val': (1.2,)} - } - feats=featop.get_features( - params['feature_class'], params['feature_type'], params['data']) - if args: - feats=featop.add_preproc(name, feats, *args['val']) - else: - feats=featop.add_preproc(name, feats) - - output=_compute(feats, params) - - params={ 'name': name } - if args: - params['args']=args - - output.update(fileop.get_output(category.PREPROC, params)) - - fileop.write(category.PREPROC, output) - - -def run(): - """Run generator for all preprocessors.""" - - _run_real('LogPlusOne') - _run_real('NormOne') - _run_real('PruneVarSubMean', {'key': ('divide',), 'val': (False,)}) - _run_real('PruneVarSubMean', {'key': ('divide',), 'val': (True,)}) - - _run_string_complex('Word') - _run_string_complex('Ulong') - -# _run_norm_derivative_lem3() -# _run_pcacut() diff --git a/tests/integration/generator/regression.py b/tests/integration/generator/regression.py deleted file mode 100644 index b9e320c3efd..00000000000 --- a/tests/integration/generator/regression.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Generator for Regression""" - -import shogun.Regression as regression -from shogun.Kernel import GaussianKernel - -import fileop -import featop -import dataop -import category - - -def _compute (params, feats, kernel, pout): - """ - Compute a regression and gather result data. - - @param params misc parameters for the regression method - @param feats features of the kernel/regression - @param kernel kernel - @param pout previously gathered data from kernel ready to be written to file - """ - - kernel.parallel.set_num_threads(params['num_threads']) - kernel.init(feats['train'], feats['train']) - params['labels'], labels=dataop.get_labels(feats['train'].get_num_vectors()) - - try: - fun=eval('regression.'+params['name']) - except AttributeError: - return - - if params['type']=='svm': - regression=fun(params['C'], params['epsilon'], kernel, labels) - regression.set_tube_epsilon(params['tube_epsilon']) - else: - regression=fun(params['tau'], kernel, labels) - regression.parallel.set_num_threads(params['num_threads']) - - regression.train() - - if params['type']=='svm': - params['bias']=regression.get_bias() - params['alpha_sum']=0 - for item in regression.get_alphas().tolist(): - params['alpha_sum']+=item - params['sv_sum']=0 - for item in regression.get_support_vectors(): - params['sv_sum']+=item - - kernel.init(feats['train'], feats['test']) - params['classified']=regression.apply().get_labels() - - output=pout.copy() - output.update(fileop.get_output(category.REGRESSION, params)) - fileop.write(category.REGRESSION, output) - - -def _loop (regressions, feats, kernel, pout): - """ - Loop through regression computations, only slightly differing in parameters. - - @param regressions names of the regression methods to loop through - @param feats features of the kernel/regression - @param kernel kernel - @param pout previously gathered data from kernel ready to be written to file - """ - - for r in regressions: - r['num_threads']=1 - if r['type']=='svm': - r['C']=.017 - r['epsilon']=1e-5 - r['accuracy']=r['epsilon']*10 - r['tube_epsilon']=1e-2 - _compute(r, feats, kernel, pout) - r['C']=.23 - _compute(r, feats, kernel, pout) - r['C']=1.5 - _compute(r, feats, kernel, pout) - r['C']=30 - _compute(r, feats, kernel, pout) - r['epsilon']=1e-4 - r['accuracy']=r['epsilon']*10 - _compute(r, feats, kernel, pout) - r['tube_epsilon']=1e-3 - _compute(r, feats, kernel, pout) - elif r['type']=='kernelmachine': - r['tau']=1e-6 - _compute(r, feats, kernel, pout) - r['tau']=1e-5 - _compute(r, feats, kernel, pout) - else: - continue - - # BUG in SVRLight: - # glibc detected *** /usr/bin/python: free(): invalid next size (fast) - if r['name']!='SVRLight': - r['num_threads']=16 - _compute(r, feats, kernel, pout) - -########################################################################## -# public -########################################################################## - -def run (): - """Run generator for all regression methods.""" - - regressions=( - {'name': 'SVRLight', 'type': 'svm', 'accuracy': 1e-6}, - {'name': 'LibSVR', 'type': 'svm', 'accuracy': 1e-6}, - {'name': 'KRR', 'type': 'kernelmachine', 'accuracy': 1e-8}, - ) - - params={ - 'name': 'Gaussian', - 'args': {'key': ('width',), 'val': (1.5,)}, - 'feature_class': 'simple', - 'feature_type': 'Real', - 'data': dataop.get_rand() - } - output=fileop.get_output(category.KERNEL, params) - feats=featop.get_simple('Real', params['data']) - kernel=GaussianKernel(10, *params['args']['val']) - - _loop(regressions, feats, kernel, output) - diff --git a/tests/integration/libshogun/test_all.sh b/tests/integration/libshogun/test_all.sh deleted file mode 100755 index af8bb3c0eeb..00000000000 --- a/tests/integration/libshogun/test_all.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash - -# not implemented yet -exit 0 - -DATAPATH='../data' - -function test_all () { - datapath="$1" - echo "*** Testing in $datapath" - sleep 1 - for file in $datapath; do - echo -n "$file" - echo -n -e "\t\t" - - output=`${PYTHON} test_one.py "$file"` - ret=$? - - if [ $ret -eq 0 ] ; then - echo 'OK' - else - echo 'ERROR' - echo $output - fi - done - sleep 1 - echo -} - -if [ -n "$1" ]; then - test_all "$DATAPATH/$1/*.m" -else - for i in $DATAPATH/*; do - test_all "$i/*.m" - done -fi diff --git a/tests/integration/lua_modular/generator.lua b/tests/integration/lua_modular/generator.lua deleted file mode 100644 index d2799e47b92..00000000000 --- a/tests/integration/lua_modular/generator.lua +++ /dev/null @@ -1,57 +0,0 @@ -package.path = package.path .. ";../../../src/interfaces/lua_modular/?.lua;" -package.cpath = package.cpath .. ";../../../src/interfaces/lua_modular/?.so;/usr/local/lib/?.so" - -require("lfs") - -example_dir = '../../examples/undocumented/lua_modular' -test_dir = '../../../testsuite/tests' -blacklist = {"load.lua", "MatrixTest.lua", "VectorTest.lua"} - -function get_test_mod(tests) - lfs.chdir(example_dir) - local r = {} - for _, v in pairs(tests) do - local flag = 0 - if string.sub(v, -3) ~= "lua" then - flag = 1 - end - for _, n in pairs(blacklist) do - if n == v then - flag = 1 - break - end - end - if flag == 0 then - mod = string.sub(v, 1, string.len(v)-4) - table.insert(r, mod) - end - end - return r -end -function setup_tests(tests) - if not tests then - local files={} - for i in io.popen("ls " .. example_dir):lines() do - table.insert(files, i) - end - table.sort(files) - return files - else - return tests - end -end - -function generator(tests) - r = get_test_mod(tests) - for _, t in pairs(r) do - require(t) - for i = 1, #parameter_list do - f = loadstring("a=" .. t .. "(unpack(parameter_list[" .. i .. "]))") - f() - print("OK") - end - end -end - -tests = setup_tests(...) -generator(tests) diff --git a/tests/integration/matlab_and_octave/classifier.m b/tests/integration/matlab_and_octave/classifier.m deleted file mode 100644 index 0677a89a415..00000000000 --- a/tests/integration/matlab_and_octave/classifier.m +++ /dev/null @@ -1,138 +0,0 @@ -function y = classifier(filename) - addpath('util'); - addpath('../data/classifier'); - y=true; - - eval('globals'); % ugly hack to have vars from filename as globals - % - %system(sprintf('ln -sf ../data/classifier/%s.m testscript.m', filename)); - %testscript; - %system('rm -f testscript.m'); %avoid ultra long filenames (>63 chars) - eval(filename); - - % b0rked, skip these - if strcmp(classifier_name, 'Perceptron')==1 || strcmp(classifier_name, 'SubGradientSVM')==1 - fprintf('%s currently does not have nice data.\n', classifier_name); - return; - end - - if strcmp(classifier_type, 'kernel')==1 - if ~set_features('kernel_') - return; - end - if ~set_kernel() - return; - end - elseif strcmp(classifier_type, 'knn')==1 - if ~set_features('distance_') - return; - end - if ~set_distance() - return; - end - else - if ~set_features('classifier_') - return; - end - end - - if ~isempty(classifier_labels) - sg('set_labels', 'TRAIN', classifier_labels); - end - - if strcmp(classifier_name, 'SVMOcas')==1 - sg('svm_use_bias', false); - end - - cname=fix_classifier_name_inconsistency(classifier_name); - try - sg('new_classifier', cname); - catch - fprintf('Cannot set classifier %s!\n', cname); - return; - end - - if ~isempty(classifier_bias_enabled) - sg('svm_use_bias', tobool(classifier_bias_enabled)); - end - if ~isempty(classifier_epsilon) - sg('svm_epsilon', classifier_epsilon); - end - if ~isempty(classifier_max_train_time) - sg('svm_max_train_time', classifier_max_train_time); - end - if ~isempty(classifier_linadd_enabled) - sg('use_linadd', true); - end - if ~isempty(classifier_batch_enabled) - sg('use_batch_computation', true); - end - if ~isempty(classifier_num_threads) - sg('threads', classifier_num_threads); - end - - if strcmp(classifier_type, 'knn')==1 - sg('train_classifier', classifier_k); - elseif strcmp(classifier_type, 'lda')==1 - sg('train_classifier', classifier_gamma); - else - if ~isempty(classifier_C) - sg('c', classifier_C); - end - sg('train_classifier'); - end - - alphas=0; - bias=0; - sv=0; - - if strcmp(classifier_type, 'lda')==1 - 0; % nop - else - if ~isempty(classifier_bias) && strcmp(classifier_label_type, 'series')~=1 - [bias, weights]=sg('get_svm'); - bias=abs(bias-classifier_bias); - end - - if ~isempty(classifier_alpha_sum) && ~isempty(classifier_sv_sum) - if strcmp(classifier_label_type, 'series')==1 - for i = 0:sg('get_num_svms')-1 - [dump, weights]=sg('get_svm', i); - weights=weights'; - for j = 1:length(weights(1:1, :)) - alphas=alphas+weights(1:1, j:j); - end - for j = 1:length(weights(2:2, :)) - sv=sv+weights(2:2, j:j); - end - end - alphas=abs(alphas-classifier_alpha_sum); - sv=abs(sv-classifier_sv_sum); - else - [dump, weights]=sg('get_svm'); - weights=weights'; - for i = 1:length(weights(1:1, :)) - alphas=alphas+weights(1:1, i:i); - end - alphas=abs(alphas-classifier_alpha_sum); - for i = 1:length(weights(2:2, :)) - sv=sv+weights(2:2, i:i); - end - sv=abs(sv-classifier_sv_sum); - end - end - end - - if strcmp(classifier_name, 'WDSVMOcas')==1 - converted=sg('get_features', 'TRAIN'); - for i = 1:length(classifier_data_train) - [classifier_data_train(i), converted(i)] - end - classified=sg('classify') - classifier_classified - end - - classified=max(abs(sg('classify')-classifier_classified)); - - data={'classifier', alphas, bias, sv, classified}; - y=check_accuracy(classifier_accuracy, data); diff --git a/tests/integration/matlab_and_octave/clustering.m b/tests/integration/matlab_and_octave/clustering.m deleted file mode 100644 index 674cc7ee814..00000000000 --- a/tests/integration/matlab_and_octave/clustering.m +++ /dev/null @@ -1,59 +0,0 @@ -function y = clustering(filename) - addpath('util'); - addpath('../data/clustering'); - y=true; - - eval('globals'); % ugly hack to have vars from filename as globals - eval(filename); - - % there is some randomness involved, alas this is - % not working correctly in matlab - rand('state', init_random); - - if ~set_features('distance_') - return; - end - - if ~set_distance() - return; - end - - cname=fix_clustering_name_inconsistency(clustering_name); - sg('new_clustering', cname); - - if ~isempty(clustering_max_iter) - max_iter=clustering_max_iter; - else - max_iter=1000; - end - - if ~isempty(clustering_k) - first_arg=clustering_k; - elseif ~isempty(clustering_merges) - first_arg=clustering_merges; - else - error('Incomplete clustering data!\n'); - end - - sg('init_random', init_random); - sg('train_clustering', first_arg, max_iter); - - if ~isempty(clustering_radi) - [radi, centers]=sg('get_clustering'); - radi=max(abs(radi'-clustering_radi)); - centers=max(max(abs(centers-clustering_centers))); - - data={'kmeans', centers, radi}; - y=check_accuracy(clustering_accuracy, data); - - elseif ~isempty(clustering_merge_distance) - [merge_distances, pairs]=sg('get_clustering'); - merge_distances=max(abs(merge_distances'-clustering_merge_distance)); - pairs=max(max(abs(pairs-clustering_pairs))); - - data={'hierarchical', merge_distances, pairs}; - y=check_accuracy(clustering_accuracy, data); - - else - error('Incomplete clustering data!\n'); - end diff --git a/tests/integration/matlab_and_octave/distance.m b/tests/integration/matlab_and_octave/distance.m deleted file mode 100644 index 04fffe05c0c..00000000000 --- a/tests/integration/matlab_and_octave/distance.m +++ /dev/null @@ -1,24 +0,0 @@ -function y = distance(filename) - addpath('util'); - addpath('../data/distance'); - y=true; - - eval('globals'); % ugly hack to have vars from filename as globals - eval(filename); - - if ~set_features('distance_') - return; - end - - if ~set_distance() - return; - end - - dmatrix=sg('get_distance_matrix', 'TRAIN'); - dm_train=max(max(abs(distance_matrix_train-dmatrix))); - - dmatrix=sg('get_distance_matrix', 'TEST'); - dm_test=max(max(abs(distance_matrix_test-dmatrix))); - - data={'distance', dm_train, dm_test}; - y=check_accuracy(distance_accuracy, data); diff --git a/tests/integration/matlab_and_octave/distribution.m b/tests/integration/matlab_and_octave/distribution.m deleted file mode 100644 index 1b65e01e4be..00000000000 --- a/tests/integration/matlab_and_octave/distribution.m +++ /dev/null @@ -1,28 +0,0 @@ -function y = distribution(filename) - addpath('util'); - addpath('../data/distribution'); - y=true; - - eval('globals'); % ugly hack to have vars from filename as globals - eval(filename); - - sg('init_random', init_random); - rand('state', init_random); - - if ~set_features('distribution_') - return; - end - - - if strcmp(distribution_name, 'HMM')==1 - sg('new_hmm', distribution_N, distribution_M); - sg('bw'); - else - fprintf('Cannot yet train other distributions than HMM!\n'); - return; - end - - likelihood=abs(sg('hmm_likelihood')-distribution_likelihood); - - data={'distribution', likelihood, 0}; - y=check_accuracy(distribution_accuracy, data); diff --git a/tests/integration/matlab_and_octave/kernel.m b/tests/integration/matlab_and_octave/kernel.m deleted file mode 100644 index 985a0207d1d..00000000000 --- a/tests/integration/matlab_and_octave/kernel.m +++ /dev/null @@ -1,27 +0,0 @@ -function y = kernel(filename) - addpath('util'); - addpath('../data/kernel'); - y=true; - - eval('globals'); % ugly hack to have vars from filename as globals - %system(sprintf('ln -sf ../data/kernel/%s.m testscript.m', filename)); - %testscript; - %system('rm -f testscript.m'); %avoid ultra long filenames (>63 chars) - eval(filename); - - if ~set_features('kernel_') - return; - end - - if ~set_kernel() - return; - end - - kmatrix=sg('get_kernel_matrix', 'TRAIN'); - km_train=max(max(abs(kernel_matrix_train-kmatrix))); - - kmatrix=sg('get_kernel_matrix', 'TEST'); - km_test=max(max(abs(kernel_matrix_test-kmatrix))); - - data={'kernel', km_train, km_test}; - y=check_accuracy(kernel_accuracy, data); diff --git a/tests/integration/matlab_and_octave/preprocessor.m b/tests/integration/matlab_and_octave/preprocessor.m deleted file mode 100644 index be70f3b4dd6..00000000000 --- a/tests/integration/matlab_and_octave/preprocessor.m +++ /dev/null @@ -1,34 +0,0 @@ -function y = preprocessor(filename) - addpath('util'); - addpath('../data/preprocessor'); - y=true; - - eval('globals'); % ugly hack to have vars from filename as globals - eval(filename); - - if ~set_features('kernel_') - return; - end - - pname=fix_preproc_name_inconsistency(preprocessor_name); - if strcmp(pname, 'PRUNEVARSUBMEAN')==1 - sg('add_preproc', pname, tobool(preprocessor_arg0_divide)); - else - sg('add_preproc', pname); - end - - sg('attach_preproc', 'TRAIN'); - sg('attach_preproc', 'TEST'); - - if ~set_kernel() - return; - end - - kmatrix=sg('get_kernel_matrix', 'TRAIN'); - km_train=max(max(abs(kernel_matrix_train-kmatrix))); - - kmatrix=sg('get_kernel_matrix', 'TEST'); - km_test=max(max(abs(kernel_matrix_test-kmatrix))); - - data={'kernel', km_train, km_test}; - y=check_accuracy(kernel_accuracy, data); diff --git a/tests/integration/matlab_and_octave/regression.m b/tests/integration/matlab_and_octave/regression.m deleted file mode 100644 index 2d82ccfd693..00000000000 --- a/tests/integration/matlab_and_octave/regression.m +++ /dev/null @@ -1,60 +0,0 @@ -function y = regression(filename) - addpath('util'); - addpath('../data/regression'); - y=true; - - eval('globals'); % ugly hack to have vars from filename as globals - eval(filename); - - if ~set_features('kernel_') - return; - end - - if ~set_kernel() - return; - end - - sg('threads', regression_num_threads); - sg('set_labels', 'TRAIN', regression_labels); - - rname=fix_regression_name_inconsistency(regression_name); - try - sg('new_regression', rname); - catch - fprintf('Cannot set regression %s!\n', rname); - return; - end - - if strcmp(regression_type, 'svm')==1 - sg('c', regression_C); - sg('svm_epsilon', regression_epsilon); - sg('svr_tube_epsilon', regression_tube_epsilon); - elseif strcmp(regression_type, 'kernelmachine')==1 - sg('krr_tau', regression_tau); - else - error('Incomplete regression data!\n'); - end - - sg('train_regression'); - - alphas=0; - bias=0; - sv=0; - if ~isempty(regression_bias) - [bias, weights]=sg('get_svm'); - bias=abs(bias-regression_bias); - weights=weights'; - for i = 1:length(weights(1:1, :)) - alphas=alphas+weights(1:1, i:i); - end - alphas=abs(alphas-regression_alpha_sum); - for i = 1:length(weights(2:2, :)) - sv=sv+weights(2:2, i:i); - end - sv=abs(sv-regression_sv_sum); - end - - classified=max(abs(sg('classify')-regression_classified)); - - data={'classifier', alphas, bias, sv, classified}; - y=check_accuracy(regression_accuracy, data); diff --git a/tests/integration/matlab_and_octave/test_all.sh b/tests/integration/matlab_and_octave/test_all.sh deleted file mode 100755 index a4fd744be0c..00000000000 --- a/tests/integration/matlab_and_octave/test_all.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash - -DATAPATH='../data' -exitcode=0 - -function test_all () { - datapath="$1" - - if echo "$datapath" | grep -q '\.\./data/tests' - then - continue - fi - - echo "*** Testing in $datapath" - sleep 1 - for file in $datapath; do - - echo -n "$file" - echo -n -e "\t\t" - - if grep -q $file ../blacklist - then - echo 'SKIPPING' - else - output=`./test_one.sh ${file} ${interface}` - - if [ "${interface}" == "octave" ]; then - ans=`echo $output | grep 'ans =' | awk '{print $NF}'` - else # matlab has '>>' as last element in output - ans=`echo $output | grep 'ans =' | awk '{print $(NF-1)}'` - fi - - if [ -z ${ans} ]; then - ans=0 - fi - - # thanks to matlab, 1 means ok and 0 means error - if [ "$?" -ne 0 -o "${ans}" -eq 0 ]; then - exitcode=1 - echo ERROR - echo ${output} - else - echo OK - fi - fi - done - sleep 1 - echo -} - -interface=${2-octave} - -if [ "${interface}" != "octave" -a "${interface}" != "matlab" ]; then - echo "Unknown interface ${interface}" - exit 1 -fi - -if [ -n "$1" -a "$1" != "-" ]; then - test_all "$DATAPATH/$1/*.m" -else - for i in $DATAPATH/*; do - test_all "$i/*.m" - done -fi -exit $exitcode diff --git a/tests/integration/matlab_and_octave/test_one.sh b/tests/integration/matlab_and_octave/test_one.sh deleted file mode 100755 index 3234837f401..00000000000 --- a/tests/integration/matlab_and_octave/test_one.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -path=${1} -interface=${2-octave} - -module=`echo ${1} | awk -F/ '{print $3}'` -filename=`echo ${1} | awk -F/ '{print $4}' | cut -d '.' -f 1` - -if [ "${interface}" == "octave" ]; then - echo "${module}('${filename}')" | octave -else - echo "${module}('${filename}')" | matlab -nojvm -nodisplay -fi diff --git a/tests/integration/matlab_and_octave/util/check_accuracy.m b/tests/integration/matlab_and_octave/util/check_accuracy.m deleted file mode 100644 index 634368e435d..00000000000 --- a/tests/integration/matlab_and_octave/util/check_accuracy.m +++ /dev/null @@ -1,27 +0,0 @@ -function y = check_accuracy(accuracy, data) - ending=sprintf('<--- accuracy: %e\n', accuracy); - if strcmp(data{1}, 'kernel')==1 || strcmp(data{1}, 'distance')==1 - fprintf('train: %e, test: %e %s', data{2}, data{3}, ending); - elseif strcmp(data{1}, 'classifier')==1 - fprintf('alphas: %e, bias: %e, sv: %e, classified: %e %s', ... - data{2}, data{3}, data{4}, data{5}, ending); - elseif strcmp(data{1}, 'kmeans')==1 - fprintf('centers: %e, radi: %e %s', data{2}, data{3}, ending); - elseif strcmp(data{1}, 'hierarchical')==1 - fprintf('merge_distances: %e, pairs: %e %s', ... - data{2}, data{3}, ending); - elseif strcmp(data{1}, 'distribution')==1 - fprintf('likelihood: %e, derivatives: %e %s', ... - data{2}, data{3}, ending); - elseif strcmp(data{1}, 'custom')==1 - fprintf('triangletriangle: %e, fulltriangle: %e, fullfull: %e %s', ... - data{2}, data{3}, data{4}, ending); - end - - y=true; - for i = 2:length(data) - if data{i}>accuracy - y=false; - return; - end - end diff --git a/tests/integration/matlab_and_octave/util/convert_features_and_add_preproc.m b/tests/integration/matlab_and_octave/util/convert_features_and_add_preproc.m deleted file mode 100644 index c33c0ee8569..00000000000 --- a/tests/integration/matlab_and_octave/util/convert_features_and_add_preproc.m +++ /dev/null @@ -1,34 +0,0 @@ -function y = convert_features_and_add_preproc(prefix) - eval(sprintf(['global ', prefix, 'order'])); - eval(sprintf(['global ', prefix, 'gap'])); - eval(sprintf(['global ', prefix, 'reverse'])); - eval(sprintf(['global ', prefix, 'feature_type'])); - order=eval(sprintf([prefix, 'order'])); - gap=eval(sprintf([prefix, 'gap'])); - reverse=eval(sprintf([prefix, 'reverse'])); - feature_type=eval(sprintf([prefix, 'feature_type'])); - - y=false; - - if isempty(order) - return; - end - - if strcmp(feature_type, 'Ulong')==1 - type='ULONG'; - elseif strcmp(feature_type, 'Word')==1 - type='WORD'; - else - return; - end - - sg('add_preproc', strcat('SORT', type, 'STRING')); - sg('convert', 'TRAIN', 'STRING', 'CHAR', 'STRING', type, ... - order, order-1, gap, reverse); - sg('attach_preproc', 'TRAIN'); - - sg('convert', 'TEST', 'STRING', 'CHAR', 'STRING', type, ... - order, order-1, gap, reverse); - sg('attach_preproc', 'TEST'); - - y=true; diff --git a/tests/integration/matlab_and_octave/util/fix_classifier_name_inconsistency.m b/tests/integration/matlab_and_octave/util/fix_classifier_name_inconsistency.m deleted file mode 100644 index 143e452e588..00000000000 --- a/tests/integration/matlab_and_octave/util/fix_classifier_name_inconsistency.m +++ /dev/null @@ -1,15 +0,0 @@ -function y = fix_classifier_name_inconsistency (cname) - cname=upper(cname); - - if findstr('LIBSVM', cname) - if length(cname) > length('LIBSVM') - pos=findstr('LIBSVM', cname); - y=strcat('LIBSVM_', cname(pos+6:end)); - return; - end - elseif findstr('LIBLINEAR', cname) - y='LIBLINEAR_L2R_LR'; - return - end - - y=cname; diff --git a/tests/integration/matlab_and_octave/util/fix_clustering_name_inconsistency.m b/tests/integration/matlab_and_octave/util/fix_clustering_name_inconsistency.m deleted file mode 100644 index 19535944847..00000000000 --- a/tests/integration/matlab_and_octave/util/fix_clustering_name_inconsistency.m +++ /dev/null @@ -1,2 +0,0 @@ -function y = fix_clustering_name_inconsistency (cname) - y=upper(cname); diff --git a/tests/integration/matlab_and_octave/util/fix_distance_name_inconsistency.m b/tests/integration/matlab_and_octave/util/fix_distance_name_inconsistency.m deleted file mode 100644 index 0c15818218b..00000000000 --- a/tests/integration/matlab_and_octave/util/fix_distance_name_inconsistency.m +++ /dev/null @@ -1,14 +0,0 @@ -function y = fix_distance_name_inconsistency (dname) - dname=upper(dname); - if findstr('WORDDISTANCE', dname) - pos=findstr('WORDDISTANCE', dname); - y=dname(1:pos-1); - elseif findstr('DISTANCE', dname) - pos=findstr('DISTANCE', dname); - y=dname(1:pos-1); - elseif findstr('METRIC', dname) - pos=findstr('METRIC', dname); - y=dname(1:pos-1); - else - y=dname; - end diff --git a/tests/integration/matlab_and_octave/util/fix_kernel_name_inconsistency.m b/tests/integration/matlab_and_octave/util/fix_kernel_name_inconsistency.m deleted file mode 100644 index 27dd733e586..00000000000 --- a/tests/integration/matlab_and_octave/util/fix_kernel_name_inconsistency.m +++ /dev/null @@ -1,32 +0,0 @@ -function y = fix_kernel_name_inconsistency (kname) - kname=upper(kname); - if findstr('SIMPLELOCALITYIMPROVEDSTRING', kname) - y='SLIK'; - elseif findstr('LOCALITYIMPROVEDSTRING', kname) - y='LIK'; - elseif findstr('SPARSEGAUSSIAN', kname) - y='GAUSSIAN'; - elseif findstr('SPARSEPOLY', kname) - y='POLY'; - elseif findstr('SPARSELINEAR', kname) - y='LINEAR'; - elseif findstr('WEIGHTEDDEGREEPOSITIONSTRING', kname) - y='WEIGHTEDDEGREEPOS'; - elseif strcmp(kname, 'WEIGHTEDCOMMWORDSTRING')==1 - y='WEIGHTEDCOMMSTRING'; - elseif findstr('COMMULONGSTRING', kname) - y='COMMSTRING'; - elseif findstr('COMMWORDSTRING', kname) - y='COMMSTRING'; - elseif findstr('WORDSTRING', kname) - pos=findstr('WORDSTRING', kname); - y=kname(1:pos-1); - elseif findstr('STRING', kname) - pos=findstr('STRING', kname); - y=kname(1:pos-1); - elseif findstr('WORD', kname) - pos=findstr('WORD', kname); - y=kname(1:pos-1); - else - y=kname; - end diff --git a/tests/integration/matlab_and_octave/util/fix_normalizer_name_inconsistency.m b/tests/integration/matlab_and_octave/util/fix_normalizer_name_inconsistency.m deleted file mode 100644 index 43bd037204c..00000000000 --- a/tests/integration/matlab_and_octave/util/fix_normalizer_name_inconsistency.m +++ /dev/null @@ -1,10 +0,0 @@ -function y=fix_normalizer_name_inconsistency(name) - if strcmp(name, 'IdentityKernelNormalizer')==1 - y='IDENTITY'; - elseif strcmp(name, 'AvgDiagKernelNormalizer')==1 - y='AVGDIAG'; - elseif strcmp(name, 'SqrtDiagKernelNormalizer')==1 - y='SQRTDIAG'; - elseif strcmp(name, 'FirstElementKernelNormalizer')==1 - y='FIRSTELEMENT'; - end diff --git a/tests/integration/matlab_and_octave/util/fix_preproc_name_inconsistency.m b/tests/integration/matlab_and_octave/util/fix_preproc_name_inconsistency.m deleted file mode 100644 index bd873ebf7f7..00000000000 --- a/tests/integration/matlab_and_octave/util/fix_preproc_name_inconsistency.m +++ /dev/null @@ -1,2 +0,0 @@ -function y = fix_preproc_name_inconsistency (pname) - y=upper(pname); diff --git a/tests/integration/matlab_and_octave/util/fix_regression_name_inconsistency.m b/tests/integration/matlab_and_octave/util/fix_regression_name_inconsistency.m deleted file mode 100644 index 61c6e385bac..00000000000 --- a/tests/integration/matlab_and_octave/util/fix_regression_name_inconsistency.m +++ /dev/null @@ -1,2 +0,0 @@ -function y = fix_regression_name_inconsistency (rname) - y=upper(rname); diff --git a/tests/integration/matlab_and_octave/util/globals.m b/tests/integration/matlab_and_octave/util/globals.m deleted file mode 100644 index 4f511ca99f9..00000000000 --- a/tests/integration/matlab_and_octave/util/globals.m +++ /dev/null @@ -1,178 +0,0 @@ -global feats_train; -global feats_test; - -global distance; -global kernel; -global pie; - -global kernel_name; -global kernel_feature_class; -global kernel_feature_type; -global kernel_data; -global kernel_data_train; -global kernel_data_test; -global kernal_matrix_train; -global kernel_matrix_test; -global kernel_alphabet; -global kernel_order; -global kernel_gap; -global kernel_reverse; -global kernel_normalizer; -global kernel_arg1_gamma; -global kernel_arg0_c; -global kernel_arg2_coef0; -global kernel_arg1_diag; -global kernel_arg0_degree; -global kernel_arg1_degree; -global kernel_arg2_distance; -global kernel_arg1_inhomogene; -global kernel_arg2_inhomogene; -global kernel_arg2_inner_degree; -global kernel_arg1_k; -global kernel_arg1_length; -global kernel_arg2_max_shift; -global kernel_arg1_normalization; -global kernel_arg3_outer_degree; -global kernel_arg0_scale; -global kernel_arg3_shift_step; -global kernel_arg0_size; -global kernel_arg1_size; -global kernel_arg1_use_sign; -global kernel_arg0_width; -global kernel_arg1_width; -global kernel_arg2_width; - -global kernel_subkernel0_name; -global kernel_subkernel0_accuracy; -global kernel_subkernel0_data_train; -global kernel_subkernel0_data_test; -global kernel_subkernel0_data_class; -global kernel_subkernel0_feature_class; -global kernel_subkernel0_feature_type; -global kernel_subkernel0_arg0_size; -global kernel_subkernel0_arg1_degree; -global kernel_subkernel0_arg1_width; -global kernel_subkernel0_alphabet; -global kernel_subkernel0_seqlen; -global kernel_subkernel1_name; -global kernel_subkernel1_accuracy; -global kernel_subkernel1_data_train; -global kernel_subkernel1_data_test; -global kernel_subkernel1_data_class; -global kernel_subkernel1_feature_type; -global kernel_subkernel1_feature_class; -global kernel_subkernel1_arg0_size; -global kernel_subkernel1_arg1_degree; -global kernel_subkernel1_arg2_inhomogene; -global kernel_subkernel1_alphabet; -global kernel_subkernel1_seqlen; -global kernel_subkernel2_name; -global kernel_subkernel2_accuracy; -global kernel_subkernel2_data_train; -global kernel_subkernel2_data_test; -global kernel_subkernel2_data_class; -global kernel_subkernel2_feature_type; -global kernel_subkernel2_feature_class; -global kernel_subkernel2_arg0_size; -global kernel_subkernel2_alphabet; -global kernel_subkernel2_seqlen; -global kernel_symdata; - -global distance_name; -global distance_feature_class; -global distance_feature_type; -global distance_data_train; -global distance_data_test; -global distance_matrix_train; -global distance_matrix_test; -global distance_alphabet; -global distance_order; -global distance_gap; -global distance_reverse; -global distance_arg0_use_sign; -global distance_arg0_k; - -global classifier_name; -global classifier_feature_class; -global classifier_feature_type; -global classifier_data_train; -global classifier_data_test; -global classifier_type; -global classifier_labels; -global classifier_label_type; -global classifier_bias; -global classifier_bias_enabled; -global classifier_epsilon; -global classifier_tube_epsilon; -global classifier_max_train_time; -global classifier_linadd_enabled; -global classifier_batch_enabled; -global classifier_num_threads; -global classifier_batch_enabled; -global classifier_k; -global classifier_gamma; -global classifier_C; -global classifier_classified; -global classifier_accuracy; -global classifier_alpha_sum; -global classifier_sv_sum; -global classifier_alphabet; - -global regression_name; -global regression_num_threads; -global regression_labels; -global regression_type; -global regression_C; -global regression_epsilon; -global regression_tube_epsilon; -global regression_tau; -global regression_bias; -global regression_alpha_sum; -global regression_sv_sum; -global regression_classified; -global regression_accuracy; - -global clustering_name; -global clustering_max_iter; -global clustering_k; -global clustering_merges; -global clustering_radi; -global clustering_merge_distance; -global clustering_accuracy; - -global distribution_name; -global distribution_feature_class; -global distribution_feature_type; -global distribution_data_train; -global distribution_data_test; -global distribution_alphabet; -global distribution_order; -global distribution_gap; -global distribution_reverse; -global distribution_N; -global distribution_M; -global distribution_pseudo; -global distribution_accuracy; -global distribution_likelihood; - -global topfk_name; -global topfk_N; -global topfk_M; -global topfk_accuracy; -global topfk_likelihood; -global topfk_feature_class; -global topfk_feature_type; -global topfk_data_train; -global topfk_data_test; -global topfk_alphabet; -global topfk_order; -global topfk_gap; -global topfk_reverse; -global topfk_pseudo; - -global pos_train; -global pos_test; -global neg_train; -global neg_test; - -global preproc_name; diff --git a/tests/integration/matlab_and_octave/util/set_distance.m b/tests/integration/matlab_and_octave/util/set_distance.m deleted file mode 100644 index b2b85dcc7ac..00000000000 --- a/tests/integration/matlab_and_octave/util/set_distance.m +++ /dev/null @@ -1,19 +0,0 @@ -function y = set_distance() - global distance_name; - global distance_feature_type; - ftype=upper(distance_feature_type); - dname=fix_distance_name_inconsistency(distance_name); - - if strcmp(dname, 'HAMMING')==1 - global distance_arg0_use_sign; - sg('set_distance', dname, ftype, tobool(distance_arg0_use_sign)); - elseif strcmp(dname, 'MINKOWSKI')==1 - global distance_arg0_k; - sg('set_distance', dname, ftype, distance_arg0_k); - elseif strcmp(dname, 'SPARSEEUCLIDEAN')==1 - sg('set_distance', 'EUCLIDEAN', 'SPARSEREAL'); - else - sg('set_distance', dname, ftype); - end - - y=true; diff --git a/tests/integration/matlab_and_octave/util/set_features.m b/tests/integration/matlab_and_octave/util/set_features.m deleted file mode 100644 index a56316daa1f..00000000000 --- a/tests/integration/matlab_and_octave/util/set_features.m +++ /dev/null @@ -1,115 +0,0 @@ -function y = set_features(prefix) - eval(sprintf(['global ', prefix, 'name'])); - eval(sprintf(['global ', prefix, 'feature_type'])); - eval(sprintf(['global ', prefix, 'feature_class'])); - eval(sprintf(['global ', prefix, 'alphabet'])); - eval(sprintf(['global ', prefix, 'data_train'])); - eval(sprintf(['global ', prefix, 'data_test'])); - global feats_train; - global feats_test; - global topfk_name; - global kernel_data; - global classifier_type; - - name=eval(sprintf([prefix, 'name'])); - feature_type=eval(sprintf([prefix, 'feature_type'])); - feature_class=eval(sprintf([prefix, 'feature_class'])); - alphabet=eval(sprintf([prefix, 'alphabet'])); - data_train=eval(sprintf([prefix, 'data_train'])); - data_test=eval(sprintf([prefix, 'data_test'])); - y=false; - - if findstr('Sparse', name) - if strcmp(feature_type, 'Real')~=1 - fprintf('Sparse features other than Real not supported yet!\n'); - return; - end - end - - if strcmp(alphabet, 'RAWBYTE')==1 - fprintf('Alphabet RAWBYTE not supported yet!\n'); - return; - end - - if ~isempty(topfk_name) - fprintf('Features %s not yet supported!\n', topfk_name); - return; - end - - if strcmp(name, 'Combined')==1 - global kernel_subkernel0_alphabet; - global kernel_subkernel0_data_train; - global kernel_subkernel0_data_test; - global kernel_subkernel1_alphabet; - global kernel_subkernel1_data_train; - global kernel_subkernel1_data_test; - global kernel_subkernel2_alphabet; - global kernel_subkernel2_data_train; - global kernel_subkernel2_data_test; - - if isempty(kernel_subkernel0_alphabet) - sg('add_features', 'TRAIN', kernel_subkernel0_data_train); - sg('add_features', 'TEST', kernel_subkernel0_data_test); - else - sg('add_features', 'TRAIN', ... - kernel_subkernel0_data_train, kernel_subkernel0_alphabet); - sg('add_features', 'TEST', ... - kernel_subkernel0_data_test, kernel_subkernel0_alphabet); - end - - if isempty(kernel_subkernel1_alphabet) - sg('add_features', 'TRAIN', kernel_subkernel1_data_train); - sg('add_features', 'TEST', kernel_subkernel1_data_test); - else - sg('add_features', 'TRAIN', ... - kernel_subkernel1_data_train, kernel_subkernel1_alphabet); - sg('add_features', 'TEST', ... - kernel_subkernel1_data_test, kernel_subkernel1_alphabet); - end - - if isempty(kernel_subkernel2_alphabet) - sg('add_features', 'TRAIN', kernel_subkernel2_data_train); - sg('add_features', 'TEST', kernel_subkernel2_data_test); - else - sg('add_features', 'TRAIN', ... - kernel_subkernel2_data_train, kernel_subkernel2_alphabet); - sg('add_features', 'TEST', ... - kernel_subkernel2_data_test, kernel_subkernel2_alphabet); - end - - elseif ~isempty(alphabet) - if strcmp(alphabet, 'RAWDNA')==1 - sg('set_features', 'TRAIN', data_train, 'DNA'); - sg('convert', 'TRAIN', 'STRING', 'CHAR', 'STRING', 'BYTE'); - sg('set_features', 'TEST', data_test, 'DNA'); - sg('convert', 'TEST', 'STRING', 'CHAR', 'STRING', 'BYTE'); - else - sg('set_features', 'TRAIN', data_train, alphabet); - sg('set_features', 'TEST', data_test, alphabet); - end - - elseif ~isempty(kernel_data) - sg('set_features', 'TRAIN', kernel_data); - sg('set_features', 'TEST', kernel_data); - - else - fname='double'; - if strcmp(feature_type, 'Word')==1 - fname='uint16'; - elseif strcmp(classifier_type, 'linear')==1 - fname='sparse'; - elseif findstr('Sparse', name) - fname='sparse'; - end - - if iscell(data_train) - data_train=cellfun(@str2num, data_train); - data_test=cellfun(@str2num, data_test); - end - - sg('set_features', 'TRAIN', feval(fname, data_train)); - sg('set_features', 'TEST', feval(fname, data_test)); - end - - convert_features_and_add_preproc(prefix); - y=true; diff --git a/tests/integration/matlab_and_octave/util/set_kernel.m b/tests/integration/matlab_and_octave/util/set_kernel.m deleted file mode 100644 index 7ba20223052..00000000000 --- a/tests/integration/matlab_and_octave/util/set_kernel.m +++ /dev/null @@ -1,208 +0,0 @@ -function y = set_kernel() - global kernel_name; - global kernel_feature_type; - global kernel_arg0_size; - global kernel_arg1_size; - global kernel_normalizer; - y=false; - - if findstr('Sparse', kernel_name) - is_sparse=true; - else - is_sparse=false; - end - kname=fix_kernel_name_inconsistency(kernel_name); - - if strcmp(kname, 'AUC')==1 || strcmp(kname, 'CUSTOM')==1 - fprintf('Kernel %s not supported yet!\n', kname); - return; - end - - if ~isempty(kernel_arg0_size) - size_cache=kernel_arg0_size; - elseif ~isempty(kernel_arg1_size) - size_cache=kernel_arg1_size; - else - size_cache=10; - end - - - if ~isempty(kernel_feature_type) - ftype=upper(kernel_feature_type); - else - ftype='UNKNOWN'; - end - - % this endless list sux, but dunno how to do it differently here :( - if strcmp(kname, 'SIGMOID')==1 - global kernel_arg1_gamma; - global kernel_arg2_coef0; - sg('set_kernel', kname, ftype, size_cache, kernel_arg1_gamma, kernel_arg2_coef0); - - elseif strcmp(kname, 'CHI2')==1 - global kernel_arg1_width; - sg('set_kernel', kname, ftype, size_cache, kernel_arg1_width); - - elseif strcmp(kname, 'CONST')==1 - global kernel_arg0_c; - sg('set_kernel', kname, ftype, size_cache, kernel_arg0_c); - - elseif strcmp(kname, 'DIAG')==1 - global kernel_arg1_diag; - sg('set_kernel', kname, ftype, size_cache, kernel_arg1_diag); - - elseif strcmp(kname, 'GAUSSIANSHIFT')==1 - global kernel_arg1_width; - global kernel_arg2_max_shift; - global kernel_arg3_shift_step; - sg('set_kernel', kname, ftype, size_cache, kernel_arg1_width, kernel_arg2_max_shift, kernel_arg3_shift_step); - - elseif strcmp(kname, 'GAUSSIAN')==1 - global kernel_arg0_width; - global kernel_arg1_width; - if ~isempty(kernel_arg0_width) - width=kernel_arg0_width; - else - width=kernel_arg1_width; - end - if is_sparse - ftype=strcat('SPARSE', ftype); - end - sg('set_kernel', kname, ftype, size_cache, width); - - elseif strcmp(kname, 'LINEAR')==1 - global kernel_arg0_scale; - if isempty(kernel_arg0_scale) - kernel_arg0_scale=-1; - end - if is_sparse - ftype=strcat('SPARSE', ftype); - end - sg('set_kernel', kname, ftype, size_cache, kernel_arg0_scale); - - elseif strcmp(kname, 'POLYMATCH')==1 || strcmp(kname, 'POLYMATCHWORD')==1 - global kernel_arg1_degree; - global kernel_arg2_inhomogene; - sg('set_kernel', kname, ftype, size_cache, ... - kernel_arg1_degree, tobool(kernel_arg2_inhomogene)); - - elseif strcmp(kname, 'POLY')==1 - if is_sparse - global kernel_arg1_degree; - global kernel_arg2_inhomogene; - degree=kernel_arg1_degree; - inhomogene=kernel_arg2_inhomogene; - ftype=strcat('SPARSE', ftype); - sg('set_kernel', kname, ftype, size_cache, ... - degree, tobool(inhomogene)); - else - global kernel_arg1_degree; - global kernel_arg2_inhomogene; - degree=kernel_arg1_degree; - inhomogene=tobool(kernel_arg2_inhomogene); - sg('set_kernel', kname, ftype, size_cache, degree, inhomogene); - end - - elseif strcmp(kname, 'MATCH')==1 - global kernel_arg1_degree; - sg('set_kernel', kname, ftype, size_cache, kernel_arg1_degree); - - elseif findstr(kname, 'COMMSTRING') % normal + WEIGHTED - global kernel_arg1_use_sign; - if ~isempty(kernel_arg1_use_sign) - sg('set_kernel', kname, ftype, size_cache, ... - tobool(kernel_arg1_use_sign)); - else - sg('set_kernel', kname, ftype, size_cache); - end - - elseif findstr(kname, 'DEGREE') % FIXED + WEIGHTED - global kernel_arg0_degree; - global kernel_arg1_degree; - if ~isempty(kernel_arg0_degree) - degree=kernel_arg0_degree; - else - degree=kernel_arg1_degree; - end - sg('set_kernel', kname, ftype, size_cache, degree); - - elseif strcmp(kname, 'HISTOGRAM')==1 || strcmp(kname, 'SALZBERG')==1 - global classifier_labels; - pseudo_pos=1e-10; - pseudo_neg=1e-10; - sg('new_plugin_estimator', pseudo_pos, pseudo_neg); - sg('set_labels', 'TRAIN', double(classifier_labels)); - sg('train_estimator'); - - sg('set_kernel', kname, ftype, size_cache); - - elseif strcmp(kname, 'DISTANCE')==1 - global kernel_arg2_distance; - global kernel_arg1_width; - dname=fix_distance_name_inconsistency(kernel_arg2_distance); - % FIXME: REAL is cheating and will break in the future - sg('set_distance', dname, 'REAL'); - sg('set_kernel', kname, size_cache, kernel_arg1_width); - - elseif strcmp(kname, 'LOCALALIGNMENT')==1 - sg('set_kernel', kname, ftype, size_cache); - - elseif findstr(kname, 'LIK') - global kernel_arg1_length; - global kernel_arg2_inner_degree; - global kernel_arg3_outer_degree; - sg('set_kernel', kname, ftype, size_cache, ... - kernel_arg1_length, ... - kernel_arg2_inner_degree, ... - kernel_arg3_outer_degree); - - elseif strcmp(kname, 'OLIGO')==1 - global kernel_arg1_k; - global kernel_arg2_width; - sg('set_kernel', kname, ftype, size_cache, ... - kernel_arg1_k, kernel_arg2_width); - - elseif strcmp(kname, 'COMBINED') - % this will break when test file is changed! - global kernel_subkernel0_name; - global kernel_subkernel0_feature_type; - global kernel_subkernel0_arg0_size; - global kernel_subkernel0_arg1_degree; - global kernel_subkernel1_name; - global kernel_subkernel1_feature_type; - global kernel_subkernel1_arg0_size; - global kernel_subkernel1_arg1_degree; - global kernel_subkernel1_arg2_inhomogene; - global kernel_subkernel2_name; - global kernel_subkernel2_feature_type; - global kernel_subkernel2_arg0_size; - - sg('set_kernel', 'COMBINED', size_cache); - - subkernel_name=fix_kernel_name_inconsistency(kernel_subkernel0_name); - sg('add_kernel', 1., subkernel_name, ... - upper(kernel_subkernel0_feature_type), ... - kernel_subkernel0_arg0_size, ... - kernel_subkernel0_arg1_degree); - - subkernel_name=fix_kernel_name_inconsistency(kernel_subkernel1_name); - sg('add_kernel', 1., subkernel_name, ... - upper(kernel_subkernel1_feature_type), ... - kernel_subkernel1_arg0_size, ... - kernel_subkernel1_arg1_degree, ... - tobool(kernel_subkernel1_arg2_inhomogene)); - - subkernel_name=fix_kernel_name_inconsistency(kernel_subkernel2_name); - sg('add_kernel', 1., subkernel_name, ... - upper(kernel_subkernel2_feature_type), ... - kernel_subkernel2_arg0_size); - - else - error('Unknown kernel %s!\n', kname); - end - - if ~isempty(kernel_normalizer) - nname=fix_normalizer_name_inconsistency(kernel_normalizer); - sg('set_kernel_normalization', nname); - end - y=true; diff --git a/tests/integration/matlab_and_octave/util/tobool.m b/tests/integration/matlab_and_octave/util/tobool.m deleted file mode 100644 index be2e2c9f9dd..00000000000 --- a/tests/integration/matlab_and_octave/util/tobool.m +++ /dev/null @@ -1,2 +0,0 @@ -function y = tobool(arg) - y=eval(lower(arg)); diff --git a/tests/integration/matlab_static b/tests/integration/matlab_static deleted file mode 120000 index 0dd5d7e1c64..00000000000 --- a/tests/integration/matlab_static +++ /dev/null @@ -1 +0,0 @@ -matlab_and_octave \ No newline at end of file diff --git a/tests/integration/octave_modular/classifier.m b/tests/integration/octave_modular/classifier.m deleted file mode 100644 index d449932d99d..00000000000 --- a/tests/integration/octave_modular/classifier.m +++ /dev/null @@ -1,164 +0,0 @@ -function y = classifier(filename) - init_shogun; - y=true; - addpath('util'); - addpath('../data/classifier'); - - eval('globals'); % ugly hack to have vars from filename as globals - eval(filename); - - if strcmp(classifier_type, 'kernel')==1 - if ~set_features('kernel_') - return; - end - if ~set_kernel() - return; - end - elseif strcmp(classifier_type, 'knn')==1 - if ~set_features('distance_') - return; - end - if ~set_distance() - return; - end - else - if ~set_features('classifier_') - return; - end - end - - if ~isempty(classifier_labels) - lab=BinaryLabels(classifier_labels); - end - - if strcmp(classifier_name, 'GMNPSVM')==1 - classifier=GMNPSVM(classifier_C, kernel, lab); - - elseif strcmp(classifier_name, 'GPBTSVM')==1 - classifier=GPBTSVM(classifier_C, kernel, lab); - - elseif strcmp(classifier_name, 'KNN')==1 - classifier=KNN(classifier_k, distance, lab); - - elseif strcmp(classifier_name, 'LDA')==1 - classifier=LDA(classifier_gamma, feats_train, lab); - - elseif strcmp(classifier_name, 'LibLinear')==1 - classifier=LibLinear(classifier_C, feats_train, lab); - classifier.set_liblinear_solver_type(L2R_LR) - - elseif strcmp(classifier_name, 'LibSVMMultiClass')==1 - classifier=LibSVMMultiClass(classifier_C, kernel, lab); - classifier.set_solver_type(L2R_LR); - - elseif strcmp(classifier_name, 'LibSVMOneClass')==1 - classifier=LibSVMOneClass(classifier_C, kernel); - - elseif strcmp(classifier_name, 'LibSVM')==1 - classifier=LibSVM(classifier_C, kernel, lab); - - elseif strcmp(classifier_name, 'MPDSVM')==1 - classifier=MPDSVM(classifier_C, kernel, lab); - - elseif strcmp(classifier_name, 'Perceptron')==1 - classifier=Perceptron(feats_train, lab); - classifier.set_learn_rate(classifier_learn_rate); - classifier.set_max_iter(classifier_max_iter); - - elseif strcmp(classifier_name, 'SVMLight')==1 - try - classifier=SVMLight(classifier_C, kernel, lab); - catch - disp('No support for SVMLight available.'); - return; - end - - elseif strcmp(classifier_name, 'SVMLin')==1 - classifier=SVMLin(classifier_C, feats_train, lab); - - elseif strcmp(classifier_name, 'SVMOcas')==1 - classifier=SVMOcas(classifier_C, feats_train, lab); - classifier.set_bias_enabled(false); - - elseif strcmp(classifier_name, 'SVMSGD')==1 - classifier=SVMSGD(classifier_C, feats_train, lab); - - elseif strcmp(classifier_name, 'SubGradientSVM')==1 - classifier=SubGradientSVM(classifier_C, feats_train, lab); - - else - error('Unsupported classifier %s', classifier_name); - end - - classifier.parallel.set_num_threads(classifier_num_threads); - if strcmp(classifier_type, 'linear')==1 && ~isempty(classifier_bias) - classifier.set_bias_enabled(true); - end - if ~isempty(classifier_epsilon) && strcmp(classifier_name, 'SVMSGD')!=1 - classifier.set_epsilon(classifier_epsilon); - end - if ~isempty(classifier_max_train_time) - classifier.set_max_train_time(classifier_max_train_time); - end - if ~isempty(classifier_linadd_enabled) - classifier.set_linadd_enabled(tobool(classifier_linadd_enabled)); - end - if ~isempty(classifier_batch_enabled) - classifier.set_batch_computation_enabled( - tobool(classifier_batch_enabled)); - end - - classifier.train(); - - bias=0; - if ~isempty(classifier_bias) - bias=classifier.get_bias(); - bias=abs(bias-classifier_bias); - end - - alphas=0; - sv=0; - if ~isempty(classifier_alpha_sum) && ~isempty(classifier_sv_sum) - if strcmp(classifier_label_type, 'series')==1 - for i = 0:classifier.get_num_svms()-1 - subsvm=classifier.get_svm(i); - tmp=subsvm.get_alphas(); - for j = 1:length(tmp) - alphas=alphas+tmp(j:j); - end - tmp=subsvm.get_support_vectors(); - for j = 1:length(tmp) - sv=sv+tmp(j:j); - end - end - alphas=abs(alphas-classifier_alpha_sum); - sv=abs(sv-classifier_sv_sum); - else - tmp=classifier.get_alphas(); - for i = 1:length(tmp) - alphas=alphas+tmp(i:i); - end - alphas=abs(alphas-classifier_alpha_sum); - tmp=classifier.get_support_vectors(); - for i = 1:length(tmp) - sv=sv+tmp(i:i); - end - sv=abs(sv-classifier_sv_sum); - end - end - - if strcmp(classifier_type, 'knn')==1 - distance.init(feats_train, feats_test); - elseif strcmp(classifier_type, 'kernel')==1 - kernel.init(feats_train, feats_test); - elseif (strcmp(classifier_type, 'lda')==1 || - strcmp(classifier_type, 'linear')==1 || - strcmp(classifier_type, 'perceptron')==1) - classifier.set_features(feats_test); - end - - classified=max(abs( - classifier.apply().get_values()-classifier_classified)); - - data={'classifier', alphas, bias, sv, classified}; - y=check_accuracy(classifier_accuracy, data); diff --git a/tests/integration/octave_modular/clustering.m b/tests/integration/octave_modular/clustering.m deleted file mode 100644 index 756118019b6..00000000000 --- a/tests/integration/octave_modular/clustering.m +++ /dev/null @@ -1,46 +0,0 @@ -function y = clustering(filename) - init_shogun; - y=true; - addpath('util'); - addpath('../data/clustering'); - - eval('globals'); % ugly hack to have vars from filename as globals - eval(filename); - - Math_init_random(init_random); - rand('state', init_random); - - if ~set_features('distance_') - return; - end - if ~set_distance() - return; - end - - if strcmp(clustering_name, 'KMeans')==1 - clustering=KMeans(clustering_k, distance); - clustering.train(); - - radi=clustering.get_radiuses(); - radi=max(max(abs(radi-clustering_radi))); - centers=clustering.get_cluster_centers(); - centers=max(max(abs(centers-clustering_centers))); - - data={'kmeans', centers, radi}; - y=check_accuracy(clustering_accuracy, data); - - elseif strcmp(clustering_name, 'Hierarchical')==1 - clustering=Hierarchical(clustering_merges, distance); - clustering.train(); - - merge_distances=clustering.get_merge_distances(); - merge_distances=max(max(abs( - merge_distances-clustering_merge_distance))); - pairs=clustering.get_cluster_pairs(); - pairs=max(max(abs(pairs-clustering_pairs))); - - data={'hierarchical', merge_distances, pairs}; - y=check_accuracy(clustering_accuracy, data); - else - error('Unsupported clustering %s', clustering_name); - end diff --git a/tests/integration/octave_modular/distance.m b/tests/integration/octave_modular/distance.m deleted file mode 100644 index 4abaac72e6d..00000000000 --- a/tests/integration/octave_modular/distance.m +++ /dev/null @@ -1,23 +0,0 @@ -function y = distance(filename) - init_shogun; - y=true; - - addpath('util'); - addpath('../data/distance'); - - eval('globals'); % ugly hack to have vars from filename as globals - eval(filename); - - if ~set_features('distance_') - return; - end - if ~set_distance() - return; - end - - dm_train=max(max(abs(distance_matrix_train-distance.get_distance_matrix()))); - distance.init(feats_train, feats_test); - dm_test=max(max(abs(distance_matrix_test-distance.get_distance_matrix()))); - - data={'distance', dm_train, dm_test}; - y=check_accuracy(distance_accuracy, data); diff --git a/tests/integration/octave_modular/distribution.m b/tests/integration/octave_modular/distribution.m deleted file mode 100644 index 1f4066776bc..00000000000 --- a/tests/integration/octave_modular/distribution.m +++ /dev/null @@ -1,52 +0,0 @@ -function y = distribution(filename) - init_shogun; - y=true; - addpath('util'); - addpath('../data/distribution'); - - eval('globals'); % ugly hack to have vars from filename as globals - eval(filename); - - Math_init_random(init_random); - - if ~set_features('distribution_') - return; - end - - if strcmp(distribution_name, 'Histogram')==1 - distribution=Histogram(feats_train); - distribution.train(); - - elseif strcmp(distribution_name, 'HMM')==1 - global BW_NORMAL; - distribution=HMM(feats_train, - distribution_N, distribution_M, distribution_pseudo); - distribution.train(); - distribution.baum_welch_viterbi_train(BW_NORMAL); - - elseif strcmp(distribution_name, 'LinearHMM')==1 - distribution=LinearHMM(feats_train); - distribution.train(); - - else - error('Unsupported distribution %s!', distribution_name); - end - - - likelihood=max(abs(distribution.get_log_likelihood_sample()-distribution_likelihood)); - - num_examples=feats_train.get_num_vectors(); - num_param=distribution.get_num_model_parameters(); - derivatives=0; - for i = 0:num_param-1 - for j = 0:num_examples-1 - val=distribution.get_log_derivative(i, j); - if val!=-Inf && val!=NaN % only consider sparse matrix! - derivatives+=val; - end - end - end - derivatives=max(abs(derivatives-distribution_derivatives)); - - data={'distribution', likelihood, derivatives}; - y=check_accuracy(distribution_accuracy, data); diff --git a/tests/integration/octave_modular/kernel.m b/tests/integration/octave_modular/kernel.m deleted file mode 100644 index cf2473b4693..00000000000 --- a/tests/integration/octave_modular/kernel.m +++ /dev/null @@ -1,42 +0,0 @@ -function y = kernel(filename) - init_shogun; - y=true; - addpath('util'); - addpath('../data/kernel'); - - eval('globals'); % ugly hack to have vars from filename as globals - eval(filename); - - if ~set_features('kernel_') - return; - end - if ~set_kernel() - return; - end - - if strcmp(kernel_name, 'Custom')==1 - kmt=[]; - sz=size(kernel_matrix_triangletriangle, 2); - for i=1:size(kernel_matrix_triangletriangle, 2) - kmt=[kmt kernel_matrix_triangletriangle(i, i:sz)]; - end - - kernel.set_triangle_kernel_matrix_from_triangle(kmt); - triangletriangle=max(max(abs(kernel_matrix_triangletriangle-kernel.get_kernel_matrix()))); - - kernel.set_triangle_kernel_matrix_from_full(kernel_matrix_fulltriangle); - fulltriangle=max(max(abs(kernel_matrix_fulltriangle-kernel.get_kernel_matrix()))); - - kernel.set_full_kernel_matrix_from_full(kernel_matrix_fullfull); - fullfull=max(max(abs(kernel_matrix_fullfull-kernel.get_kernel_matrix()))); - - data={'custom', triangletriangle, fulltriangle, fullfull}; - y=check_accuracy(kernel_accuracy, data); - else - km_train=max(max(abs(kernel_matrix_train-kernel.get_kernel_matrix()))); - kernel.init(feats_train, feats_test); - km_test=max(max(abs(kernel_matrix_test-kernel.get_kernel_matrix()))); - - data={'kernel', km_train, km_test}; - y=check_accuracy(kernel_accuracy, data); - end diff --git a/tests/integration/octave_modular/preprocessor.m b/tests/integration/octave_modular/preprocessor.m deleted file mode 100644 index 6ec3f4d1428..00000000000 --- a/tests/integration/octave_modular/preprocessor.m +++ /dev/null @@ -1,44 +0,0 @@ -function y = preprocessor(filename) - init_shogun; - y=true; - - addpath('util'); - addpath('../data/preprocessor'); - - eval('globals'); % ugly hack to have vars from filename as globals - eval(filename); - - if strcmp(preprocessor_name, 'LogPlusOne')==1 - preproc=LogPlusOne(); - elseif strcmp(preprocessor_name, 'NormOne')==1 - preproc=NormOne(); - elseif strcmp(preprocessor_name, 'PruneVarSubMean')==1 - preproc=PruneVarSubMean(tobool(preprocessor_arg0_divide)); - elseif strcmp(preprocessor_name, 'SortUlongString')==1 - preproc=SortUlongString(); - elseif strcmp(preprocessor_name, 'SortWordString')==1 - preproc=SortWordString(); - else - error('Unsupported preproc %s', preprocessor_name); - end - - if ~set_features('kernel_') - return; - end - - preproc.init(feats_train); - feats_train.add_preprocessor(preproc); - feats_train.apply_preprocessor(); - feats_test.add_preprocessor(preproc); - feats_test.apply_preprocessor(); - - if ~set_kernel() - return; - end - - km_train=max(max(abs(kernel_matrix_train-kernel.get_kernel_matrix()))); - kernel.init(feats_train, feats_test); - km_test=max(max(abs(kernel_matrix_test-kernel.get_kernel_matrix()))); - - data={'kernel', km_train, km_test}; - y=check_accuracy(kernel_accuracy, data); diff --git a/tests/integration/octave_modular/regression.m b/tests/integration/octave_modular/regression.m deleted file mode 100644 index 8081b13d3cf..00000000000 --- a/tests/integration/octave_modular/regression.m +++ /dev/null @@ -1,70 +0,0 @@ -function y = regression(filename) - init_shogun; - y=true; - addpath('util'); - addpath('../data/regression'); - - eval('globals'); % ugly hack to have vars from filename as globals - eval(filename); - prefix='regression_'; - - if ~set_features('kernel_') - return; - end - if ~set_kernel() - return; - end - kernel.parallel.set_num_threads(regression_num_threads); - - lab=RegressionLabels(regression_labels); - - if strcmp(regression_name, 'KERNELRIDGEREGRESSION')==1 - regression=KernelRidgeRegression(regression_tau, kernel, lab); - - elseif strcmp(regression_name, 'LibSVR')==1 - regression=LibSVR(regression_C, regression_epsilon, kernel, lab); - regression.set_tube_epsilon(regression_tube_epsilon); - - elseif strcmp(regression_name, 'SVRLight')==1 - try - regression=SVRLight(regression_C, regression_epsilon, kernel, lab); - regression.set_tube_epsilon(regression_tube_epsilon); - catch - disp('No support for SVRLight available.'); - return; - end - - else - error('Unsupported regression %s!', regression_name); - end - - regression.parallel.set_num_threads(regression_num_threads); - regression.train(); - - bias=0; - if ~isempty(regression_bias) - bias=regression.get_bias(); - bias=abs(bias-regression_bias); - end - - alphas=0; - sv=0; - if ~isempty(regression_alpha_sum) && ~isempty(regression_sv_sum) - tmp=regression.get_alphas(); - for i = 1:length(tmp) - alphas=alphas+tmp(i:i); - end - alphas=abs(alphas-regression_alpha_sum); - tmp=regression.get_support_vectors(); - for i = 1:length(tmp) - sv=sv+tmp(i:i); - end - sv=abs(sv-regression_sv_sum); - end - - kernel.init(feats_train, feats_test); - classified=max(abs( - regression.apply().get_labels()-regression_classified)); - - data={'classifier', alphas, bias, sv, classified}; - y=check_accuracy(regression_accuracy, data); diff --git a/tests/integration/octave_modular/test_all.sh b/tests/integration/octave_modular/test_all.sh deleted file mode 100755 index db772e23b92..00000000000 --- a/tests/integration/octave_modular/test_all.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash - -DATAPATH='../data' -exitcode=0 - -function test_all () { - datapath=${1} - - if echo "$datapath" | grep -q '\.\./data/tests' - then - continue - fi - echo "*** Testing in ${datapath}" - sleep 1 - for file in ${datapath}; do - echo -n "${file}" - echo -n -e "\t\t" - - if grep -q $file ../blacklist - then - echo 'SKIPPING' - else - output=`./test_one.sh ${file}` - ans=`echo ${output} | grep 'ans =' | awk '{print $NF}'` - if [ -z ${ans} ]; then - ans=0 - fi - - # thanks to matlab, 1 means ok and 0 means error - if [ ${ans} -eq 0 ]; then - exitcode=1 - echo ERROR - # remove octave banner - echo ${output} | grep -v 'GNU Octave' - else - echo OK - fi - fi - done - sleep 1 - echo -} - -if [ -n "${1}" ]; then - test_all "${DATAPATH}/${1}/*.m" -else - for i in ${DATAPATH}/*; do - test_all "${i}/*.m" - done -fi -exit $exitcode diff --git a/tests/integration/octave_modular/test_one.sh b/tests/integration/octave_modular/test_one.sh deleted file mode 100755 index c7190d0a508..00000000000 --- a/tests/integration/octave_modular/test_one.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh - -path=${1} - -module=`echo ${1} | awk -F/ '{print $3}'` -filename=`echo ${1} | awk -F/ '{print $4}' | cut -d '.' -f 1` - -dump=`echo ${filename} | grep WDSVMOcas` -if [ $? -eq 0 ]; then - echo "WDSVMOcas totally unsupported" - echo "ans = 0" # matlab compat hack - exit 1 -fi - -echo "${module}('${filename}') " | octave diff --git a/tests/integration/octave_modular/util/check_accuracy.m b/tests/integration/octave_modular/util/check_accuracy.m deleted file mode 100644 index 634368e435d..00000000000 --- a/tests/integration/octave_modular/util/check_accuracy.m +++ /dev/null @@ -1,27 +0,0 @@ -function y = check_accuracy(accuracy, data) - ending=sprintf('<--- accuracy: %e\n', accuracy); - if strcmp(data{1}, 'kernel')==1 || strcmp(data{1}, 'distance')==1 - fprintf('train: %e, test: %e %s', data{2}, data{3}, ending); - elseif strcmp(data{1}, 'classifier')==1 - fprintf('alphas: %e, bias: %e, sv: %e, classified: %e %s', ... - data{2}, data{3}, data{4}, data{5}, ending); - elseif strcmp(data{1}, 'kmeans')==1 - fprintf('centers: %e, radi: %e %s', data{2}, data{3}, ending); - elseif strcmp(data{1}, 'hierarchical')==1 - fprintf('merge_distances: %e, pairs: %e %s', ... - data{2}, data{3}, ending); - elseif strcmp(data{1}, 'distribution')==1 - fprintf('likelihood: %e, derivatives: %e %s', ... - data{2}, data{3}, ending); - elseif strcmp(data{1}, 'custom')==1 - fprintf('triangletriangle: %e, fulltriangle: %e, fullfull: %e %s', ... - data{2}, data{3}, data{4}, ending); - end - - y=true; - for i = 2:length(data) - if data{i}>accuracy - y=false; - return; - end - end diff --git a/tests/integration/octave_modular/util/convert_features_and_add_preproc.m b/tests/integration/octave_modular/util/convert_features_and_add_preproc.m deleted file mode 100644 index 51255f418eb..00000000000 --- a/tests/integration/octave_modular/util/convert_features_and_add_preproc.m +++ /dev/null @@ -1,48 +0,0 @@ -function y = convert_features_and_add_preproc(prefix) - eval(sprintf(['global ', prefix, 'order'])); - eval(sprintf(['global ', prefix, 'gap'])); - eval(sprintf(['global ', prefix, 'reverse'])); - eval(sprintf(['global ', prefix, 'feature_type'])); - order=eval(sprintf([prefix, 'order'])); - gap=eval(sprintf([prefix, 'gap'])); - reverse=eval(sprintf([prefix, 'reverse'])); - feature_type=eval(sprintf([prefix, 'feature_type'])); - global feats_train; - global feats_test; - y=false; - - if isempty(order) - return; - end - - charfeat_train=feats_train; - charfeat_test=feats_test; - - if strcmp(feature_type, 'Ulong')==1 - global StringUlongFeatures; - global SortUlongString; - feats_train=StringUlongFeatures(charfeat_train.get_alphabet()); - feats_test=StringUlongFeatures(charfeat_test.get_alphabet()); - preproc=SortUlongString(); - elseif strcmp(feature_type, 'Word')==1 - global StringWordFeatures; - global SortWordString; - feats_train=StringWordFeatures(charfeat_train.get_alphabet()); - feats_test=StringWordFeatures(charfeat_test.get_alphabet()); - preproc=SortWordString(); - else - return; - end - - feats_train.obtain_from_char(charfeat_train, - order-1, order, gap, tobool(reverse)); - feats_test.obtain_from_char(charfeat_test, - order-1, order, gap, tobool(reverse)); - - preproc.init(feats_train); - feats_train.add_preprocessor(preproc); - feats_train.apply_preprocessor(); - feats_test.add_preprocessor(preproc); - feats_test.apply_preprocessor(); - - y=true; diff --git a/tests/integration/octave_modular/util/globals.m b/tests/integration/octave_modular/util/globals.m deleted file mode 100644 index cd96773f4d9..00000000000 --- a/tests/integration/octave_modular/util/globals.m +++ /dev/null @@ -1,179 +0,0 @@ -global init_random; - -global feats_train; -global feats_test; - -global distance; -global kernel; -global pie; - -global kernel_name; -global kernel_feature_class; -global kernel_feature_type; -global kernel_data; -global kernel_data_train; -global kernel_data_test; -global kernal_matrix_train; -global kernel_matrix_test; -global kernel_alphabet; -global kernel_order; -global kernel_gap; -global kernel_reverse; -global kernel_normalizer; -global kernel_arg1_gamma; -global kernel_arg0_c; -global kernel_arg2_coef0; -global kernel_arg1_diag; -global kernel_arg0_degree; -global kernel_arg1_degree; -global kernel_arg2_distance; -global kernel_arg1_inhomogene; -global kernel_arg2_inhomogene; -global kernel_arg2_inner_degree; -global kernel_arg1_k; -global kernel_arg1_length; -global kernel_arg2_max_shift; -global kernel_arg1_normalization; -global kernel_arg3_outer_degree; -global kernel_arg0_scale; -global kernel_arg3_shift_step; -global kernel_arg0_size; -global kernel_arg1_size; -global kernel_arg1_use_sign; -global kernel_arg0_width; -global kernel_arg1_width; -global kernel_arg2_width; - -global kernel_subkernel0_name; -global kernel_subkernel0_accuracy; -global kernel_subkernel0_data_train; -global kernel_subkernel0_data_test; -global kernel_subkernel0_data_class; -global kernel_subkernel0_feature_class; -global kernel_subkernel0_feature_type; -global kernel_subkernel0_arg0_size; -global kernel_subkernel0_arg1_degree; -global kernel_subkernel0_arg1_width; -global kernel_subkernel0_alphabet; -global kernel_subkernel0_seqlen; -global kernel_subkernel1_name; -global kernel_subkernel1_accuracy; -global kernel_subkernel1_data_train; -global kernel_subkernel1_data_test; -global kernel_subkernel1_data_class; -global kernel_subkernel1_feature_type; -global kernel_subkernel1_feature_class; -global kernel_subkernel1_arg0_size; -global kernel_subkernel1_arg1_degree; -global kernel_subkernel1_arg2_inhomogene; -global kernel_subkernel1_alphabet; -global kernel_subkernel1_seqlen; -global kernel_subkernel2_name; -global kernel_subkernel2_accuracy; -global kernel_subkernel2_data_train; -global kernel_subkernel2_data_test; -global kernel_subkernel2_data_class; -global kernel_subkernel2_feature_type; -global kernel_subkernel2_feature_class; -global kernel_subkernel2_arg0_size; -global kernel_subkernel2_alphabet; -global kernel_subkernel2_seqlen; -global kernel_symdata; - -global distance_name; -global distance_feature_class; -global distance_feature_type; -global distance_data_train; -global distance_data_test; -global distance_matrix_train; -global distance_matrix_test; -global distance_alphabet; -global distance_order; -global distance_gap; -global distance_reverse; -global distance_arg0_use_sign; -global distance_arg0_k; - -global classifier_name; -global classifier_feature_class; -global classifier_feature_type; -global classifier_data_train; -global classifier_data_test; -global classifier_type; -global classifier_labels; -global classifier_label_type; -global classifier_bias; -global classifier_epsilon; -global classifier_tube_epsilon; -global classifier_max_train_time; -global classifier_linadd_enabled; -global classifier_batch_enabled; -global classifier_num_threads; -global classifier_batch_enabled; -global classifier_k; -global classifier_gamma; -global classifier_C; -global classifier_classified; -global classifier_accuracy; -global classifier_alpha_sum; -global classifier_sv_sum; -global classifier_order; - -global regression_name; -global regression_num_threads; -global regression_labels; -global regression_type; -global regression_C; -global regression_epsilon; -global regression_tube_epsilon; -global regression_tau; -global regression_bias; -global regression_alpha_sum; -global regression_sv_sum; -global regression_classified; -global regression_accuracy; - -global clustering_name; -global clustering_max_iter; -global clustering_k; -global clustering_merges; -global clustering_radi; -global clustering_merge_distance; -global clustering_accuracy; - -global distribution_name; -global distribution_feature_class; -global distribution_feature_type; -global distribution_data_train; -global distribution_data_test; -global distribution_alphabet; -global distribution_order; -global distribution_gap; -global distribution_reverse; -global distribution_N; -global distribution_M; -global distribution_pseudo; -global distribution_accuracy; -global distribution_likelihood; - -global topfk_name; -global topfk_N; -global topfk_M; -global topfk_accuracy; -global topfk_likelihood; -global topfk_feature_class; -global topfk_feature_type; -global topfk_data_train; -global topfk_data_test; -global topfk_alphabet; -global topfk_order; -global topfk_gap; -global topfk_reverse; -global topfk_pseudo; - -global pos_train; -global pos_test; -global neg_train; -global neg_test; - -global preproc_name; diff --git a/tests/integration/octave_modular/util/set_distance.m b/tests/integration/octave_modular/util/set_distance.m deleted file mode 100644 index 38e49e3a29b..00000000000 --- a/tests/integration/octave_modular/util/set_distance.m +++ /dev/null @@ -1,81 +0,0 @@ -function y = set_distance() - global distance_name; - global feats_train; - global feats_test; - global distance; - global kernel_arg2_distance; - y=false; - - if ~isempty(kernel_arg2_distance) - dname=kernel_arg2_distance; - else - dname=distance_name; - end - - if strcmp(dname, 'BrayCurtisDistance')==1 - global BrayCurtisDistance; - distance=BrayCurtisDistance(feats_train, feats_train); - elseif strcmp(dname, 'CanberraMetric')==1 - global CanberraMetric; - distance=CanberraMetric(feats_train, feats_train); - - elseif strcmp(dname, 'CanberraWordDistance')==1 - global CanberraWordDistance; - distance=CanberraWordDistance(feats_train, feats_train); - - elseif strcmp(dname, 'ChebyshewMetric')==1 - global ChebyshewMetric; - distance=ChebyshewMetric(feats_train, feats_train); - - elseif strcmp(dname, 'ChiSquareDistance')==1 - global ChiSquareDistance; - distance=ChiSquareDistance(feats_train, feats_train); - - elseif strcmp(dname, 'CosineDistance')==1 - global CosineDistance; - distance=CosineDistance(feats_train, feats_train); - - elseif strcmp(dname, 'EuclideanDistance')==1 - global EuclideanDistance; - distance=EuclideanDistance(feats_train, feats_train); - - elseif strcmp(dname, 'GeodesicMetric')==1 - global GeodesicMetric; - distance=GeodesicMetric(feats_train, feats_train); - - elseif strcmp(dname, 'HammingWordDistance')==1 - global HammingWordDistance; - global distance_arg0_use_sign; - distance=HammingWordDistance(feats_train, feats_train, ... - tobool(distance_arg0_use_sign)); - - elseif strcmp(dname, 'JensenMetric')==1 - global JensenMetric; - distance=JensenMetric(feats_train, feats_train); - - elseif strcmp(dname, 'ManhattanMetric')==1 - global ManhattanMetric; - distance=ManhattanMetric(feats_train, feats_train); - - elseif strcmp(dname, 'ManhattanWordDistance')==1 - global ManhattanWordDistance; - distance=ManhattanWordDistance(feats_train, feats_train); - - elseif strcmp(dname, 'MinkowskiMetric')==1 - global MinkowskiMetric; - global distance_arg0_k; - distance=MinkowskiMetric(feats_train, feats_train, distance_arg0_k); - - elseif strcmp(dname, 'SparseEuclideanDistance')==1 - global SparseEuclideanDistance; - distance=SparseEuclideanDistance(feats_train, feats_train); - - elseif strcmp(dname, 'TanimotoDistance')==1 - global TanimotoDistance; - distance=TanimotoDistance(feats_train, feats_train); - - else - error('Unknown distance %s!', dname); - end - - y=true; diff --git a/tests/integration/octave_modular/util/set_features.m b/tests/integration/octave_modular/util/set_features.m deleted file mode 100644 index 1364369cc1f..00000000000 --- a/tests/integration/octave_modular/util/set_features.m +++ /dev/null @@ -1,189 +0,0 @@ -function y = set_features(prefix) - eval(sprintf(['global ', prefix, 'name'])); - eval(sprintf(['global ', prefix, 'feature_type'])); - eval(sprintf(['global ', prefix, 'feature_class'])); - eval(sprintf(['global ', prefix, 'alphabet'])); - eval(sprintf(['global ', prefix, 'data_train'])); - eval(sprintf(['global ', prefix, 'data_test'])); - global feats_train; - global feats_test; - global topfk_name; - - name=eval(sprintf([prefix, 'name'])); - feature_type=eval(sprintf([prefix, 'feature_type'])); - feature_class=eval(sprintf([prefix, 'feature_class'])); - alphabet=eval(sprintf([prefix, 'alphabet'])); - data_train=eval(sprintf([prefix, 'data_train'])); - data_test=eval(sprintf([prefix, 'data_test'])); - y=false; - size_cache=10; - - if strcmp(name, 'Combined')==1 - % this will break when subkernels in data file are changed - % it blindly assumes that all features are StringChar - global kernel_subkernel0_data_train; - global kernel_subkernel0_data_test; - global kernel_subkernel1_data_train; - global kernel_subkernel1_data_test; - global kernel_subkernel2_data_train; - global kernel_subkernel2_data_test; - global CombinedFeatures; - global StringCharFeatures; - global DNA; - - feats_train=CombinedFeatures(); - feats_test=CombinedFeatures(); - - feat=StringCharFeatures(DNA); - feat.set_features(kernel_subkernel0_data_train); - feats_train.append_feature_obj(feat); - feat=StringCharFeatures(DNA); - feat.set_features(kernel_subkernel0_data_test); - feats_test.append_feature_obj(feat); - - feat=StringCharFeatures(DNA); - feat.set_features(kernel_subkernel1_data_train); - feats_train.append_feature_obj(feat); - feat=StringCharFeatures(DNA); - feat.set_features(kernel_subkernel1_data_test); - feats_test.append_feature_obj(feat); - - feat=StringCharFeatures(DNA); - feat.set_features(kernel_subkernel2_data_train); - feats_train.append_feature_obj(feat); - feat=StringCharFeatures(DNA); - feat.set_features(kernel_subkernel2_data_test); - feats_test.append_feature_obj(feat); - - elseif strcmp(name, 'Custom')==1 - global kernel_data; - global RealFeatures; - feats_train=RealFeatures(kernel_data); - feats_test=RealFeatures(kernel_data); - - elseif strcmp(topfk_name, 'FK')==1 - global pos_train; - global pos_test; - global neg_train; - global neg_test; - global FKFeatures; - - if !set_pos_and_neg('topfk_') - return; - end - - feats_train=FKFeatures(size_cache, pos_train, neg_train); - feats_train.set_opt_a(-1); %estimate prior - feats_test=FKFeatures(size_cache, pos_test, neg_test); - feats_test.set_a(feats_train.get_a()); %use prior from training data - - elseif strcmp(topfk_name, 'TOP')==1 - global pos_train; - global pos_test; - global neg_train; - global neg_test; - global TOPFeatures; - - if !set_pos_and_neg('topfk_') - return; - end - - feats_train=TOPFeatures(size_cache, pos_train, neg_train, false, false); - feats_test=TOPFeatures(size_cache, pos_test, neg_test, false, false); - - else - global classifier_type; - if findstr(name, 'Sparse') - is_sparse=true; - elseif (!isempty(classifier_type) && - strcmp(classifier_type, 'linear')==1) - is_sparse=true; - else - is_sparse=false; - end - - if strcmp(feature_class, 'simple')==1 - if strcmp(feature_type, 'Real')==1 - global RealFeatures; - feats_train=RealFeatures(double(data_train)); - feats_test=RealFeatures(double(data_test)); - - if is_sparse - global SparseRealFeatures; - sparse=SparseRealFeatures(); - sparse.obtain_from_simple(feats_train); - feats_train=sparse; - sparse=SparseRealFeatures(); - sparse.obtain_from_simple(feats_test); - feats_test=sparse; - end - - elseif strcmp(feature_type, 'Word')==1 - global WordFeatures; - feats_train=WordFeatures(uint16(data_train)); - feats_test=WordFeatures(uint16(data_test)); - - elseif strcmp(feature_type, 'Byte')==1 - global RAWBYTE; - global ByteFeatures; - feats_train=ByteFeatures(RAWBYTE); - feats_train.copy_feature_matrix(uint8(data_train)); - feats_test=ByteFeatures(RAWBYTE); - feats_test.copy_feature_matrix(uint8(data_test)); - - else - fprintf('Simple feature type %s not supported yet!\n', feature_type); - return; - end - elseif strcmp(feature_class, 'string')==1 || strcmp(feature_class, 'string_complex')==1 - global DNA; - global CUBE; - - if strcmp(alphabet, 'DNA')==1 - alphabet=DNA; - elseif strcmp(alphabet, 'CUBE')==1 - alphabet=CUBE; - elseif strcmp(alphabet, 'RAWBYTE')==1 - disp('Alphabet RAWBYTE not supported yet.'); - return; - else - error('Alphabet %s not supported yet!', alphabet); - end - - global StringCharFeatures; - feats_train=StringCharFeatures(alphabet); - feats_train.set_features(data_train); - feats_test=StringCharFeatures(alphabet); - feats_test.set_features(data_test); - - if strcmp(feature_class, 'string_complex')==1 - convert_features_and_add_preproc(prefix); - end - - elseif strcmp(feature_class, 'wd')==1 - global DNA; - global RAWDNA; - global StringCharFeatures; - global StringByteFeatures; - global WDFeatures; - eval(sprintf(['global ', prefix, 'order'])); - order=eval(sprintf([prefix, 'order'])); - - charfeat=StringCharFeatures(DNA); - charfeat.set_features(data_train); - bytefeat=StringByteFeatures(RAWDNA); - bytefeat.obtain_from_char(charfeat, 0, 1, 0, false); - feats_train=WDFeatures(bytefeat, order, order); - - charfeat=StringCharFeatures(DNA); - charfeat.set_features(data_test); - bytefeat=StringByteFeatures(RAWDNA); - bytefeat.obtain_from_char(charfeat, 0, 1, 0, false); - feats_test=WDFeatures(bytefeat, order, order); - - else - error('Unknown feature class %s', feature_class); - end - end - - y=true; diff --git a/tests/integration/octave_modular/util/set_kernel.m b/tests/integration/octave_modular/util/set_kernel.m deleted file mode 100644 index 3c0d452349f..00000000000 --- a/tests/integration/octave_modular/util/set_kernel.m +++ /dev/null @@ -1,325 +0,0 @@ -function y = set_kernel() - global kernel_name; - global feats_train; - global feats_test; - global kernel; - global kernel_arg0_size; - global kernel_arg1_size; - global kernel_normalizer; - global IdentityKernelNormalizer; - global AvgDiagKernelNormalizer; - global SqrtDiagKernelNormalizer; - y=false; - - if !isempty(kernel_arg0_size) - size_cache=kernel_arg0_size; - elseif !isempty(kernel_arg1_size) - size_cache=kernel_arg1_size; - else - size_cache=10; - end - - if !isempty(kernel_name) - kname=kernel_name; - else - kname=name; - end - - % this sux awfully, but dunno how to do it differently here :( - if strcmp(kname, 'AUC')==1 - global kernel_subkernel0_name; - global kernel_subkernel0_data_train; - global kernel_subkernel0_data_test; - global kernel_subkernel0_arg1_width; - global subkernel; % subkernel will be destroyed otherwise - global RealFeatures; - global GaussianKernel; - global AUCKernel; - - % this will break when testcase is changed! - if strcmp(kernel_subkernel0_name, 'Gaussian')!=1 - error('Dunno how to handle AUC subkernel %s', ... - kernel_subkernel0_name); - end - - subfeats_train=RealFeatures(kernel_subkernel0_data_train); - subfeats_test=RealFeatures(kernel_subkernel0_data_test); - subkernel=GaussianKernel(subfeats_train, subfeats_test, ... - kernel_subkernel0_arg1_width); - kernel=AUCKernel(0, subkernel); - kernel.init(feats_train, feats_train); - - %feats_train=WordFeatures(kernel_subkernel0_data_train); - %feats_test=WordFeatures(kernel_subkernel0_data_test); - - elseif strcmp(kname, 'Chi2')==1 - global Chi2Kernel; - global kernel_arg1_width; - kernel=Chi2Kernel(feats_train, feats_train, ... - kernel_arg1_width, size_cache); - - elseif strcmp(kname, 'Combined') - % this will break when test file is changed! - global CombinedKernel; - global FixedDegreeStringKernel; - global PolyMatchStringKernel; - global LocalAlignmentStringKernel; - global kernel_subkernel0_name; - global kernel_subkernel0_feature_type; - global kernel_subkernel0_arg0_size; - global kernel_subkernel0_arg1_degree; - global kernel_subkernel1_name; - global kernel_subkernel1_feature_type; - global kernel_subkernel1_arg0_size; - global kernel_subkernel1_arg1_degree; - global kernel_subkernel1_arg2_inhomogene; - global kernel_subkernel2_name; - global kernel_subkernel2_feature_type; - global kernel_subkernel2_arg0_size; - global StringCharFeatures; - global DNA; - - kernel=CombinedKernel(); - subkernel=FixedDegreeStringKernel(... - kernel_subkernel0_arg0_size, ... - kernel_subkernel0_arg1_degree); - kernel.append_kernel(subkernel); - subkernel=PolyMatchStringKernel(kernel_subkernel1_arg0_size, ... - kernel_subkernel1_arg1_degree, ... - tobool(kernel_subkernel1_arg2_inhomogene)); - kernel.append_kernel(subkernel); - subkernel=LocalAlignmentStringKernel(... - kernel_subkernel2_arg0_size); - kernel.append_kernel(subkernel); - kernel.init(feats_train, feats_train); - - elseif strcmp(kname, 'CommUlongString')==1 - global CommUlongStringKernel; - global kernel_arg1_use_sign; - if ~isempty(kernel_arg1_use_sign) - kernel=CommUlongStringKernel(feats_train, feats_train, ... - tobool(kernel_arg1_use_sign)); - else - kernel=CommUlongStringKernel(feats_train, feats_train); - end - - elseif strcmp(kname, 'CommWordString')==1 - global CommWordStringKernel; - global kernel_arg1_use_sign; - if ~isempty(kernel_arg1_use_sign) - kernel=CommWordStringKernel(feats_train, feats_train, ... - tobool(kernel_arg1_use_sign)); - else - kernel=CommWordStringKernel(feats_train, feats_train); - end - - elseif strcmp(kname, 'Const')==1 - global ConstKernel; - global kernel_arg0_c; - kernel=ConstKernel(feats_train, feats_train, kernel_arg0_c); - - elseif strcmp(kname, 'Custom')==1 - global CustomKernel; - kernel=CustomKernel(); - - elseif strcmp(kname, 'Diag')==1 - global DiagKernel; - global kernel_arg1_diag; - kernel=DiagKernel(feats_train, feats_train, kernel_arg1_diag); - - elseif strcmp(kname, 'Distance')==1 - global DistanceKernel; - global kernel_arg1_width; - global distance; - if !set_distance() - y=false; - return; - end - kernel=DistanceKernel(feats_train, feats_train, ... - kernel_arg1_width, distance); - - elseif strcmp(kname, 'FixedDegreeString')==1 - global FixedDegreeStringKernel; - global kernel_arg1_degree; - kernel=FixedDegreeStringKernel(feats_train, feats_train, ... - kernel_arg1_degree); - - elseif strcmp(kname, 'GaussianShift')==1 - global GaussianShiftKernel; - global kernel_arg1_width; - global kernel_arg2_max_shift; - global kernel_arg3_shift_step; - kernel=GaussianShiftKernel(feats_train, feats_train, ... - kernel_arg1_width, kernel_arg2_max_shift, kernel_arg3_shift_step); - - elseif strcmp(kname, 'Gaussian')==1 - global GaussianKernel; - global kernel_arg0_width; - global kernel_arg1_width; - if isempty(kernel_arg0_width) - width=kernel_arg1_width; - else - width=kernel_arg0_width; - end - kernel=GaussianKernel(feats_train, feats_train, width); - - elseif strcmp(kname, 'HistogramWordString')==1 - global HistogramWordStringKernel; - global pie; - if !set_pie() - return; - end - kernel=HistogramWordStringKernel(feats_train, feats_train, pie); - - elseif strcmp(kname, 'LinearByte')==1 - global LinearByteKernel; - kernel=LinearByteKernel(); - kernel.set_normalizer(AvgDiagKernelNormalizer(-1)); - kernel.init(feats_train, feats_train); - - elseif strcmp(kname, 'LinearString')==1 - global LinearStringKernel; - kernel=LinearStringKernel(); - kernel.set_normalizer(AvgDiagKernelNormalizer(-1)); - kernel.init(feats_train, feats_train); - - elseif strcmp(kname, 'LinearWord')==1 - global LinearWordKernel; - kernel=LinearWordKernel(); - kernel.set_normalizer(AvgDiagKernelNormalizer(-1)); - kernel.init(feats_train, feats_train); - - elseif strcmp(kname, 'Linear')==1 - global LinearKernel; - global kernel_arg0_scale; - kernel=LinearKernel(); - if kernel_arg0_scale - kernel.set_normalizer(AvgDiagKernelNormalizer(kernel_arg0_scale)); - else - kernel.set_normalizer(AvgDiagKernelNormalizer(-1)); - end - kernel.init(feats_train, feats_train); - - elseif strcmp(kname, 'LocalAlignmentString')==1 - global LocalAlignmentStringKernel; - kernel=LocalAlignmentStringKernel(feats_train, feats_train); - - elseif strcmp(kname, 'OligoString')==1 - global OligoStringKernel; - global kernel_arg1_k; - global kernel_arg2_width; - kernel=OligoStringKernel(size_cache, kernel_arg1_k, kernel_arg2_width); - kernel.init(feats_train, feats_train); - - elseif strcmp(kname, 'PolyMatchString')==1 - global PolyMatchStringKernel; - global kernel_arg1_degree; - global kernel_arg2_inhomogene; - kernel=PolyMatchStringKernel(feats_train, feats_train, ... - kernel_arg1_degree, tobool(kernel_arg2_inhomogene)); - - elseif strcmp(kname, 'PolyMatchWordString')==1 - global PolyMatchWordStringKernel; - global kernel_arg1_degree; - global kernel_arg2_inhomogene; - kernel=PolyMatchWordStringKernel(feats_train, feats_train, ... - kernel_arg1_degree, tobool(kernel_arg2_inhomogene)); - - elseif strcmp(kname, 'Poly')==1 - global PolyKernel; - global kernel_arg1_degree; - global kernel_arg2_inhomogene; - if ~isempty(kernel_normalizer) - kernel=PolyKernel(size_cache, kernel_arg1_degree, ... - tobool(kernel_arg2_inhomogene)); - kernel.set_normalizer(eval(sprintf([kernel_normalizer, '()']))); - kernel.init(feats_train, feats_train); - else - kernel=PolyKernel(feats_train, feats_train, kernel_arg1_degree, ... - tobool(kernel_arg2_inhomogene)); - end - - elseif strcmp(kname, 'SalzbergWordString')==1 - global SalzbergWordStringKernel; - global pie; - if !set_pie() - return; - end - kernel=SalzbergWordStringKernel(feats_train, feats_train, pie); - - elseif strcmp(kname, 'Sigmoid')==1 - global SigmoidKernel; - global kernel_arg1_gamma; - global kernel_arg2_coef0; - kernel=SigmoidKernel(feats_train, feats_train, size_cache, ... - kernel_arg1_gamma, kernel_arg2_coef0); - - elseif strcmp(kname, 'SimpleLocalityImprovedString')==1 - global SimpleLocalityImprovedStringKernel; - global kernel_arg1_length; - global kernel_arg2_inner_degree; - global kernel_arg3_outer_degree; - kernel=SimpleLocalityImprovedStringKernel(feats_train, feats_train, ... - kernel_arg1_length, kernel_arg2_inner_degree, - kernel_arg3_outer_degree); - - elseif strcmp(kname, 'SparseGaussian')==1 - global SparseGaussianKernel; - global kernel_arg1_width; - kernel=SparseGaussianKernel(feats_train, feats_train, ... - kernel_arg1_width); - - elseif strcmp(kname, 'SparseLinear')==1 - global SparseLinearKernel; - global kernel_arg0_scale; - kernel=SparseLinearKernel(); - if kernel_arg0_scale - kernel.set_normalizer(AvgDiagKernelNormalizer(kernel_arg0_scale)); - else - kernel.set_normalizer(AvgDiagKernelNormalizer(-1)); - end - kernel.init(feats_train, feats_train); - - elseif strcmp(kname, 'SparsePoly')==1 - global SparsePolyKernel; - global kernel_arg1_degree; - global kernel_arg2_inhomogene; - kernel=SparsePolyKernel(size_cache, kernel_arg1_degree, ... - tobool(kernel_arg2_inhomogene)); - kernel.init(feats_train, feats_train); - - elseif strcmp(kname, 'WeightedCommWordString')==1 - global WeightedCommWordStringKernel; - global kernel_arg1_use_sign; - kernel=WeightedCommWordStringKernel(feats_train, feats_train, ... - tobool(kernel_arg1_use_sign)); - - elseif strcmp(kname, 'WeightedDegreePositionString')==1 - global WeightedDegreePositionStringKernel; - global kernel_arg0_degree; - global kernel_arg1_degree; - if isempty(kernel_arg0_degree) - degree=kernel_arg1_degree; - else - degree=kernel_arg0_degree; - end - kernel=WeightedDegreePositionStringKernel(... - feats_train, feats_train, degree); - - elseif strcmp(kname, 'WeightedDegreeString')==1 - global WeightedDegreeStringKernel; - global kernel_arg0_degree; - kernel=WeightedDegreeStringKernel(feats_train, feats_train, ... - kernel_arg0_degree); - - elseif strcmp(kname, 'MatchWordString')==1 - global MatchWordStringKernel; - global kernel_arg1_degree; - kernel=MatchWordStringKernel(feats_train, feats_train, ... - kernel_arg1_degree); - - else - error('Unsupported kernel %s!', kname); - end - - y=true; diff --git a/tests/integration/octave_modular/util/set_pie.m b/tests/integration/octave_modular/util/set_pie.m deleted file mode 100644 index 4bb17573245..00000000000 --- a/tests/integration/octave_modular/util/set_pie.m +++ /dev/null @@ -1,13 +0,0 @@ -function y = set_pie() - global classifier_labels; - global pie; - global PluginEstimate; - global BinaryLabels; - global feats_train; - y=true; - - pie=PluginEstimate(); - labels=BinaryLabels(classifier_labels); - pie.set_labels(labels); - pie.set_features(feats_train); - pie.train(); diff --git a/tests/integration/octave_modular/util/set_pos_and_neg.m b/tests/integration/octave_modular/util/set_pos_and_neg.m deleted file mode 100644 index 46223fef00a..00000000000 --- a/tests/integration/octave_modular/util/set_pos_and_neg.m +++ /dev/null @@ -1,59 +0,0 @@ -function y = set_pos_and_neg(prefix) - eval(sprintf(['global ', prefix, 'N'])); - eval(sprintf(['global ', prefix, 'M'])); - eval(sprintf(['global ', prefix, 'pseudo'])); - eval(sprintf(['global ', prefix, 'order'])); - eval(sprintf(['global ', prefix, 'gap'])); - eval(sprintf(['global ', prefix, 'reverse'])); - eval(sprintf(['global ', prefix, 'data_train'])); - eval(sprintf(['global ', prefix, 'data_test'])); - global StringCharFeatures; - global StringWordFeatures; - global SortWordString; - global CUBE; - global HMM; - global BW_NORMAL; - global init_random; - - Math_init_random(init_random); - N=eval(sprintf([prefix, 'N'])); - M=eval(sprintf([prefix, 'M'])); - pseudo=eval(sprintf([prefix, 'pseudo'])); - order=eval(sprintf([prefix, 'order'])); - gap=eval(sprintf([prefix, 'gap'])); - reverse=tobool(eval(sprintf([prefix, 'reverse']))); - data_train=eval(sprintf([prefix, 'data_train'])); - data_test=eval(sprintf([prefix, 'data_test'])); - - global pos_train; - global pos_test; - global neg_train; - global neg_test; - y=true; - - charfeat=StringCharFeatures(CUBE); - charfeat.set_features(data_train); - wordfeats_train=StringWordFeatures(charfeat.get_alphabet()); - wordfeats_train.obtain_from_char(charfeat, order-1, order, gap, reverse); - preproc=SortWordString(); - preproc.init(wordfeats_train); - wordfeats_train.add_preprocessor(preproc); - wordfeats_train.apply_preprocessor(); - - charfeat=StringCharFeatures(CUBE); - charfeat.set_features(data_test); - wordfeats_test=StringWordFeatures(charfeat.get_alphabet()); - wordfeats_test.obtain_from_char(charfeat, order-1, order, gap, reverse); - wordfeats_test.add_preprocessor(preproc); - wordfeats_test.apply_preprocessor(); - - pos_train=HMM(wordfeats_train, N, M, pseudo); - pos_train.train(); - pos_train.baum_welch_viterbi_train(BW_NORMAL); - neg_train=HMM(wordfeats_train, N, M, pseudo); - neg_train.train(); - neg_train.baum_welch_viterbi_train(BW_NORMAL); - pos_test=HMM(pos_train); - pos_test.set_observations(wordfeats_test); - neg_test=HMM(neg_train); - neg_test.set_observations(wordfeats_test); diff --git a/tests/integration/octave_modular/util/tobool.m b/tests/integration/octave_modular/util/tobool.m deleted file mode 100644 index be2e2c9f9dd..00000000000 --- a/tests/integration/octave_modular/util/tobool.m +++ /dev/null @@ -1,2 +0,0 @@ -function y = tobool(arg) - y=eval(lower(arg)); diff --git a/tests/integration/octave_static b/tests/integration/octave_static deleted file mode 120000 index 0dd5d7e1c64..00000000000 --- a/tests/integration/octave_static +++ /dev/null @@ -1 +0,0 @@ -matlab_and_octave \ No newline at end of file diff --git a/tests/integration/perl_modular/README b/tests/integration/perl_modular/README deleted file mode 100644 index 0b6d6ab5c52..00000000000 --- a/tests/integration/perl_modular/README +++ /dev/null @@ -1,13 +0,0 @@ -This directory contains mostly legacy regression test scripts that will soon no -longer be used (bummer, all this work for nothing!). - -The new way to check for regressions is via the following two -scripts: - - generator.pl - to generate regression tests and - tester.pl - to check for regressions - -So tester.pl is sufficient. - - -26/10/2012 diff --git a/tests/integration/perl_modular/classifier.pm b/tests/integration/perl_modular/classifier.pm deleted file mode 100644 index 0405e5732d0..00000000000 --- a/tests/integration/perl_modular/classifier.pm +++ /dev/null @@ -1,215 +0,0 @@ -package classifier; -use modshogun; -use util; -use PDL; -use Data::Dumper; - -sub _get_machine -{ - my ($indata, $prefix, $feats) = @_; - my $machine; - if ($indata->{$prefix.'type'} eq 'kernel') { - my $pre='kernel_'; - my $kargs=&util::get_args($indata, $pre); - my $kfun=*{$indata->{$pre.'name'}.'Kernel'}; - my $machine=$kfun->($feats->{'train'}, $feats->{'train'}, $kargs); - if ($indata->{$pre.'name'} eq 'Linear') { - my $normalizer=*{$indata->{$pre.'normalizer'}->()}; - $machine->set_normalizer($normalizer); - $machine->init($feats->{'train'}, $feats->{'train'}); - } - $machine->parallel->set_num_threads($indata->{$prefix.'num_threads'}); - } elsif( $indata->{$prefix.'type'} eq 'knn') { - my $pre='distance_'; - my $dargs=&util::get_args($indata, $pre); - my $dfun= *{$indata->{$pre.'name'}}; - $machine=$dfun->($feats->{'train'}, $feats->{'train'}, $dargs); - $machine->parallel->set_num_threads($indata->{$prefix.'num_threads'}); - } else { - $machine=undef; - } - return $machine; -} -sub _get_results_alpha_and_sv -{ - my ($indata, $prefix, $classifier) = @_; - if( not defined($indata->{$prefix.'alpha_sum'}) - and - not defined($indata->{$prefix.'sv_sum'})){ - return (undef, undef); - } - my $a=0; - my $sv=0; - if( defined($indata->{$prefix.'label_type'}) - and - $indata->{$prefix.'label_type'} eq 'series'){ - foreach my $i (0..$classifier->get_num_svms()){ - my $subsvm=$classifier->get_svm($i); - foreach my $item (@{$subsvm->get_alphas()}){ - $a+=$item; - } - foreach my $item (@{$subsvm->get_support_vectors()}){ - $sv+=$item; - } - } - $a=abs($a-$indata->{$prefix.'alpha_sum'}); - $sv=abs($sv-$indata->{$prefix.'sv_sum'}); - }else{ - foreach my $item (@{$classifier->get_alphas()}){ - $a+=$item; - $a=abs($a-$indata->{$prefix.'alpha_sum'}); - foreach my $item (@{$classifier->get_support_vectors()->tolist()}){ - $sv+=$item; - } - $sv=abs($sv-$indata->{$prefix.'sv_sum'}); - } - } - return ($a, $sv); -} - -sub _get_results -{ - my ($indata, $prefix, $classifier, $machine, $feats) = @_; - my $e; - my %res=( - 'alphas'=>0, - 'bias'=>0, - 'sv'=>0, - 'accuracy'=>$indata->{$prefix.'accuracy'} - ); - - if(defined($indata->{$prefix.'bias'})) { - $res{'bias'}=abs($classifier->get_bias()-$indata->{$prefix.'bias'}); - } - ($res{'alphas'}, $res{'sv'}) - = &_get_results_alpha_and_sv($indata, $prefix, $classifier); - - my $ctype=$indata->{$prefix.'type'}; - if( $ctype eq 'kernel' or $ctype eq 'knn' ) { - $machine->init($feats->{'train'}, $feats->{'test'}); - }else{ - @ctypes=('linear', 'perceptron', 'lda', 'wdsvmocas'); - if(grep($ctype, @ctypes)) { - $classifier->set_features($feats->{'test'}); - } - } - $res{'classified'} - = max(abs( -#PTZ121009 this is called differently on this branch $classifier->apply()->get_confidences() - $classifier->apply()->get_values() - - $indata->{$prefix.'classified'}) - ); - return \%res; -} - -sub _evaluate { - my ($indata) = @_; - my $prefix='classifier_'; - my $ctype=$indata->{$prefix.'type'}; - my $feats; - if($indata->{$prefix.'name'} eq 'KNN'){ - $feats=&util::get_features($indata, 'distance_'); - }elsif( $ctype eq 'kernel'){ - $feats=&util::get_features($indata, 'kernel_'); - }else{ - $feats=&util::get_features($indata, $prefix); - } - my $machine=&_get_machine($indata, $prefix, $feats); - #my $fun=*{'modshogun::' . $indata->{$prefix.'name'}}; - my $fun= eval('modshogun::' . $indata->{$prefix.'name'}); - if($@) {#except NameError, e: - warn( "%s is disabled/unavailable!", $indata->{$prefix.'name'}); - return false; - } - # cannot refactor into function, because labels is unrefed otherwise - my $classifier; - if(defined($indata->{$prefix.'labels'})){ -#vector_from_pdl is_pdl_vector(ST(0), PDL_D); - my $labels= modshogun::BinaryLabels->new(&double($indata->{$prefix.'labels'})); - if( $ctype eq 'kernel'){ - $classifier=$fun->new($indata->{$prefix.'C'}, $machine, $labels); - } elsif( $ctype eq 'linear'){ -#Can't locate object method "swig_train_get" via package "modshogun::SparseRealFeatures" at /usr/src/shogun/src/interfaces/perldl_modular/modshogun.pm line 33. - $classifier=$fun->new($indata->{$prefix.'C'}, $feats->{'train'}, $labels); - } elsif( $ctype eq 'knn') { - $classifier=$fun->new($indata->{$prefix.'k'}, $machine, $labels); - } elsif( $ctype eq 'lda') { - $classifier=$fun->new($indata->{$prefix.'gamma'}, $feats->{'train'}, $labels); - } elsif( $ctype eq 'perceptron') { - $classifier=$fun->new($feats->{'train'}, labels); - } elsif( $ctype eq 'wdsvmocas'){ - $classifier=$fun->new($indata->{$prefix.'C'}, $indata->{$prefix.'degree'}, - $indata->{$prefix.'degree'}, $feats->{'train'}, $labels); - } else { - return false; - } - } else { - $classifier=$fun->new($indata->{$prefix.'C'}, $machine); - } - if($classifier->get_name() eq 'LibLinear'){ - print ($classifier->get_name(), " - yes\n"); - $classifier->set_liblinear_solver_type($modshogun::L2R_LR); - } - my $sgio = $classifier->get_global_io(); - $sgio->enable_progress(); - $sgio->set_loglevel($modshogun::MSG_GCDEBUG); - $sgio->set_loglevel($modshogun::MSG_INFO); - - $classifier->{parallel} = modshogun::Parallel->new(); - $classifier->{parallel}->set_num_threads($indata->{$prefix.'num_threads'}); - #PTZ121009 threads are not working..it crashes in Perl_IO_stdin... - # but also is not translating into the ::Parrallel instance?! - #$classifier->{parallel}->set_num_threads(1); - if($@) { - warn($@); - return 0; - } - - if($ctype eq 'linear'){ - if(defined($indata->{$prefix.'bias'})){ - $classifier->set_bias_enabled(1); - } - }else{ - $classifier->set_bias_enabled(0); - } - if( $ctype eq 'perceptron'){ - $classifier->set_learn_rate=$indata->{$prefix.'learn_rate'}; - $classifier->set_max_iter=$indata->{$prefix.'max_iter'}; - } - if(defined($indata->{$prefix.'epsilon'})){ - eval { $classifier->set_epsilon($indata->{$prefix.'epsilon'}); }; - if($@) { -#Can't locate object method "set_epsilon" via package "modshogun::SVMSGD" at classifier.pm line 167. - warn($@, Dumper($classifier)); - #return false; - } - } - if(defined($indata->{$prefix.'max_train_time'})){ - $classifier->set_max_train_time($indata->{$prefix.'max_train_time'} * 1000); - } - if(defined($indata->{$prefix.'linadd_enabled'})){ - $classifier->set_linadd_enabled($indata->{$prefix.'linadd_enabled'}); - } - if(defined($indata->{$prefix.'batch_enabled'})){ - $classifier->set_batch_computation_enabled($indata->{$prefix.'batch_enabled'}); - } - $classifier->train(); - - my $res = &_get_results($indata, $prefix, $classifier, $machine, $feats); - return &util::check_accuracy - ($res->{'accuracy'}, - {alphas=>$res->{'alphas'}, bias=>$res->{'bias'}, sv=>$res->{'sv'}, - classified=>$res->{'classified'}}); -} - -######################################################################## -# public -######################################################################## - -sub test { - my ($indata) = @_; - return &_evaluate($indata); -} - -1; -__END__ diff --git a/tests/integration/perl_modular/clustering.pm b/tests/integration/perl_modular/clustering.pm deleted file mode 100644 index c26fefe7bec..00000000000 --- a/tests/integration/perl_modular/clustering.pm +++ /dev/null @@ -1,56 +0,0 @@ -package clustering; -use util; -use PDL; -use modshogun; - -sub _evaluate -{ - my ($indata) = @_; - my $first_arg; - if(defined( $indata->{'clustering_k'})) { - $first_arg = $indata->{'clustering_k'}; - }elsif(defined($indata->{'clustering_merges'})) { - $first_arg = $indata->{'clustering_merges'}; - }else{ - return false; - } - my $feats = &util::get_features($indata, 'distance_'); - my $dfun = eval('modshogun::' . $indata->{'distance_name'}); - my $distance = $dfun->new($feats->{'train'}, $feats->{'train'}); - - my $cfun = eval('modshogun::' . $indata->{'clustering_name'}); - my $clustering = $cfun->new($first_arg, $distance); - $clustering->train(); - - if(defined($indata->{'clustering_radi'})) { - my $radi=max(abs($clustering->get_radiuses() - $indata->{'clustering_radi'})); - my $centers=max(abs($clustering->get_cluster_centers() - - $indata->{'clustering_centers'})->flat()); - return(&util::check_accuracy( - $indata->{'clustering_accuracy'} - , {radi=>$radi, centers=>$centers})); - } elsif(defined($indata->{'clustering_merge_distance'})) { - my $merge_distance = max(abs($clustering->get_merge_distances() - - $indata->{'clustering_merge_distance'})); - $pairs = max(abs($clustering->get_cluster_pairs() - - $indata->{'clustering_pairs'})->flat()); - return &util::check_accuracy($indata->{'clustering_accuracy'} - , {merge_distance=>$merge_distance, pairs=>$pairs} - ); - } else { - return &util::check_accuracy($indata->{'clustering_accuracy'}); - } -} -######################################################################## -# public -######################################################################## - -sub test -{ - my ($indata) = @_; - return &_evaluate($indata); -} - -1; -__END__ - diff --git a/tests/integration/perl_modular/distance.pm b/tests/integration/perl_modular/distance.pm deleted file mode 100644 index 19c901a19aa..00000000000 --- a/tests/integration/perl_modular/distance.pm +++ /dev/null @@ -1,43 +0,0 @@ -package distance; -use modshogun; -#use shogun::Distance; - -use util; -use PDL; - -sub _evaluate { - my ($indata) = @_; - my $prefix ='distance_'; - my $feats = &util::get_features($indata, $prefix); - - my $dfun = eval('modshogun::' . $indata->{$prefix.'name'}); - - my $dargs = &util::get_args($indata, $prefix); - my $distance = $dfun->new($feats->{'train'}, $feats->{'train'}, @$dargs); - - my $dm_train=max(abs( - $indata->{$prefix.'matrix_train'} - -$distance->get_distance_matrix())->flat); - $distance->init($feats->{'train'}, $feats->{'test'}); - my $dm_test=max(abs( - $indata->{$prefix.'matrix_test'} - -$distance->get_distance_matrix())->flat); - - return &util::check_accuracy( - $indata->{$prefix.'accuracy'} - , {dm_train=>$dm_train, dm_test=>$dm_test}); -} - - -######################################################################## -# public -######################################################################## - -sub test -{ - my ($indata)=@_; - return &_evaluate($indata); -} - -1; -__END__ diff --git a/tests/integration/perl_modular/distribution.pm b/tests/integration/perl_modular/distribution.pm deleted file mode 100644 index 800b20d4a1d..00000000000 --- a/tests/integration/perl_modular/distribution.pm +++ /dev/null @@ -1,81 +0,0 @@ -package distribution; -use util; -use PDL; -use modshogun; - -sub _evaluate -{ - my ($indata) = @_; - my $prefix = 'distribution_'; - my $feats = &util::get_features($indata, $prefix); - my $distribution; - if($indata->{$prefix.'name'} eq 'HMM'){ - $distribution = modshogun::HMM->new($feats->{'train'}, $indata->{$prefix.'N'}, - $indata->{$prefix.'M'}, $indata->{$prefix.'pseudo'}); - $distribution->train(); - $distribution->baum_welch_viterbi_train($modshogun::BW_NORMAL); - }else{ - my $dfun = eval('modshogun::' . $indata->{$prefix.'name'}); - $distribution = $dfun->new($feats->{'train'}); - $distribution->train(); - } - my $likelihood = $distribution->get_log_likelihood_sample(); - my $num_examples = $feats->{'train'}->get_num_vectors(); - my $num_param = $distribution->get_num_model_parameters(); - my $derivatives = 0; - foreach my $i (0..$num_param-1) { - foreach my $j (0..$num_examples-1) { - my $val = $distribution->get_log_derivative($i, $j); -#PTZ121013 SG or PDL representation of NAN and INF PDL::Type::badvalue, PDL::Type::usenan - if( $val ne 'inf' and $val ne 'nan') { # only consider sparse matrix! - $derivatives += $val; - } - } - } - $derivatives = abs($derivatives-$indata->{$prefix.'derivatives'}); - $likelihood = abs($likelihood-$indata->{$prefix.'likelihood'}); - - if( $indata->{$prefix.'name'} eq 'HMM'){ - my $best_path = 0; - my $best_path_state = 0; - foreach my $i (0..$indata->{$prefix.'num_examples'}-1) { - $best_path+=$distribution->best_path($i); - foreach my $j (0..$indata->{$prefix.'N'}-1) { - $best_path_state += $distribution->get_best_path_state($i, $j); - } - } - $best_path=abs($best_path - $indata->{$prefix.'best_path'}); - $best_path_state=abs($best_path_state - - $indata->{$prefix.'best_path_state'}); - - return &util::check_accuracy($indata->{$prefix.'accuracy'} - , { - derivatives=>$derivatives - , likelihood=>$likelihood - , best_path=>$best_path - , best_path_state=>$best_path_state - } - ); - } else { - return &util::check_accuracy($indata->{$prefix.'accuracy'}, { - derivatives=>$derivatives, likelihood=>$likelihood}); - } -} - -######################################################################## -# public -######################################################################## - -sub test -{ - my ($indata) = @_; - return &_evaluate($indata); -} -1; -__END__ - -=head2 - - Test Distribution - -=cut diff --git a/tests/integration/perl_modular/generator.pm b/tests/integration/perl_modular/generator.pm deleted file mode 100644 index bd2d05ca043..00000000000 --- a/tests/integration/perl_modular/generator.pm +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/perl -package generator; - -use IO::File; -use File::Basename; -use File::Find::Rule; -use File::Slurp; -use File::Spec; -use File::pushd; -use base qw(Exporter); - -our %EXPORT_TAGS = -( - #FIELDS => [ @EXPORT_OK, @EXPORT ] - all => [qw(setup_tests get_fname blacklist get_test_mod run_test)] -); -#Exporter::export_tags(qw/all/); -Exporter::export_ok_tags(qw/all/); - -our $example_dir = '../../../examples/undocumented/perl_modular'; -#TODO::PTZ121109 relpath is quite wrong -our $test_dir = '../../regression/tests'; -our @blacklist = ("classifier_libsvm_minimal_modular.t", - "kernel_combined_modular.t", - "kernel_distance_modular.t", - "distribution_hmm_modular.t"); - -sub get_fname -{ - my ($mod_name, $i) = @_; - return File::Spec->catfile($test_dir, $mod_name . $i . '.txt'); -} - -sub setup_tests($) { - my ($tests) = @_; - my $edir = pushd( $example_dir ); - if($#$tests < 0) { - $tests = File::Find::Rule->file() - ->name( '*.t' ) - ->in( '.' ); - } else { - $tests = [ map(basename($_), @$tests) ]; - } - #sys.path.insert(0, '.') - return $tests; -} - -sub check_for_function($) -{ -#TODO::PTZ121109 - my ($fname) = @_; - my $fh = IO::File->new($fname, "r"); - if (defined $fh) { - while(my $l = <$fh>) { - if($l =~ /^\s*sub /) { - return 1; - } - } - } - return 0; -} -sub get_test_mod -{ - my ($t) = @_; - if (($t =~ /.t$/) - and not ($t =~ /^\./) - and !(grep(/$t/, @blacklist))) - { - (my $mod_name = $t) =~ s/\..*$//; - my $mod = $mod_name; - $mod =~ s#.*/##g; - $mod =~ s#_.*##g; - #TODO::PTZ121109 $t is path less... - #if(not &check_for_function($t)) { - #TODO::PTZ121109 warn("ERROR (no function)"); - #} - $mod = eval( 'modshogun::' . $mod)->new(); - #TODO::PTZ121109 something for $mod->{parameter_list} - return($mod, $mod_name); - } -} - -sub run_test -{ - my ($mod, $mod_name, $i) = @_; - my $fname = &get_fname($mod_name, $i); - my $par = $mod->[$i]; - my $a = &getattr($mod, $mod_name)->($par); - return $a; -} - - -sub generator -{ - my ($tests) = @_; - foreach my $t (@$tests) { - my ($mod, $mod_name) = &get_test_mod($t); - unless( @$mod ) { next; } - if($@) { - warn("%-60s", $mod_name); - next; - } - my $fname = ""; - - printf("%-60s", $mod_name); - #print "%+60s" % "...", - foreach my $i (0..$#{$mod}) { - $fname = &get_fname($mod_name, $i); - my $a = &run_test($mod, $mod_name, $i); - write_file($fname, $a); - if(@_) { - warn( "ERROR generating '%s' using '%s'" , $fname, $t); - next; - } - print "OK"; - } - } -} - -1; -__END__ - -=head2 SYNOPSYS - - my $tests = &setup_tests(\@ARGV); - &generator($tests); - -=cut diff --git a/tests/integration/perl_modular/kernel.pm b/tests/integration/perl_modular/kernel.pm deleted file mode 100644 index 8169d58175e..00000000000 --- a/tests/integration/perl_modular/kernel.pm +++ /dev/null @@ -1,327 +0,0 @@ -package kernel; - -use modshogun; - -=pod - -comming from modshogun::? -use Shogun::Features; -use Shogun::Kernel; -use Shogun::Preprocessor; -use Shogun::Distance; -use Shogun::Classifier qw(PluginEstimate); -use Shogun::Distribution qw(HMM BW_NORMAL); - -=cut - - -use PDL;# qw( array ushort ubyte double); - -use util; - -######################################################################## -# kernel computation -######################################################################## - -sub _evaluate { - my ($indata, $prefix) = @_; - my $feats = &util::get_features($indata, $prefix); - my $kfun = eval('modshogun::' . $indata->{$prefix.'name'}.'Kernel'); - my $kargs = &util::get_args($indata, $prefix); - my $kernel = $kfun->new(@$kargs); - if( defined($indata->{$prefix.'normalizer'})) { - my $fnorm = eval('modshogun::' . $indata->{$prefix.'normalizer'})->new(); - $kernel->set_normalizer($fnorm); - } - $kernel->init($feats->{'train'}, $feats->{'train'}); - my $km_train=max(abs( - $indata->{$prefix.'matrix_train'} - - $kernel->get_kernel_matrix())->flat()); - $kernel->init($feats->{'train'}, $feats->{'test'}); - my $km_test=max(abs( - $indata->{$prefix.'matrix_test'} - -$kernel->get_kernel_matrix())->flat); - - return &util::check_accuracy( - $indata->{$prefix.'accuracy'} - , {km_train => $km_train, km_test => $km_test}); -} - -sub _get_subkernels -{ - my ($indata, $prefix) = @_; - my %subkernels; - $prefix.='subkernel'; - my $len_prefix = strlen($prefix); - - # loop through indata (unordered dict) to gather subkernel data - foreach my $key (keys %$indata) { - unless($prefix =~ /${key}/) { next;} - - # get subkernel's number - my $num= substr($key, $len_prefix, 1); - unless ($num) { - warn('Cannot find number for subkernel: "%s"!', $data); - } - # get item's name - my $name=substr($key, $len_prefix + 2); - - # append new item - $subkernels{$num}{$name} = $indata->{$key}; - - # got all necessary information in new structure, now create a kernel - # object for each subkernel - while(my ($num, $data) = each(%subkernels)) { - my $fun = eval('modshogun::' . $data{'name'}.'Kernel'); - my $args= &util::get_args($data, ''); - $subkernels{$num}{'kernel'}=$fun->($args); - } - } - return \%subkernels; -} - -sub _evaluate_combined -{ - my ($indata, $prefix) = @_; - my $kernel = modshogun::CombinedKernel->new(); - my %feats=('train'=> modshogun::CombinedFeatures->new() - , 'test' => modshogun::CombinedFeatures->new() - ); - - my %subkernels = (&_get_subkernels($indata, $prefix)); - foreach my $subk (keys %subkernels) { - $feats_subk=&util::get_features($subk, ''); - push(@{$feats{'train'}}, @{$feats_subk{'train'}}); - - push(@{$feats{'test'}}, @{$feats_subk{'test'}}); - $kernel->append_kernel($subk{'kernel'}); - } - $kernel->init($feats{'train'}, $feats{'train'}); - my $km_train=max(abs($indata->{'kernel_matrix_train'} - - $kernel->get_kernel_matrix())->flat); - my $kernel->init($feats{'train'}, $feats{'test'}); - my $km_test=max(abs( - $indata->{'kernel_matrix_test'} - - $kernel->get_kernel_matrix())->flat); - - return &util::check_accuracy - ( - $indata->{$prefix.'accuracy'} - , { - km_train=>$km_train - , km_test=>$km_test - } - ); - -} -sub _evaluate_auc -{ - my ($indata, $prefix) = @_; - my $subk=&_get_subkernels(indata, prefix)->{'0'}; - my $feats_subk = &util::get_features($subk, ''); - $subk{'kernel'}->init($feats_subk{'train'}, $feats_subk{'test'}); - my %feats= ( - 'train'=> modshogun::WordFeatures->new($indata->{$prefix.'data_train'}->astype(&ushort)) - , 'test'=> modshogun::WordFeatures->new($indata->{$prefix.'data_test'}->astype(&ushort)) - ); - my $kernel= modshogun::AUCKernel->new(10, $subk{'kernel'}); - - $kernel->init($feats{'train'}, $feats{'train'}); - my $km_train=max(abs( - $indata->{$prefix.'matrix_train'}-$kernel->get_kernel_matrix())->flat); - $kernel->init(feats{'train'}, $feats{'test'}); - my $km_test=max(abs( - $indata->{$prefix.'matrix_test'}-$kernel->get_kernel_matrix())->flat); - - return &util::check_accuracy - ($indata->{$prefix.'accuracy'} - , { km_train=>$km_train - , km_test=>$km_test - } - ); - -} -sub _evaluate_custom -{ - my ($indata, $prefix) = @_; - my %feats=( - 'train'=> modshogun::RealFeatures->new($indata->{$prefix.'data'}), - 'test'=> modshogun::RealFeatures->new($indata->{$prefix.'data'}) - ); - - my $symdata=$indata->{$prefix.'symdata'}; - if(0) { -#PTZ121005not realy sure about this... - my @c; - foreach my $x (0..$symdata->shape()->at(1)-1) { - my @r; - foreach my $y (0..$symdata->shape()->at(0)-1) { - if($y <= $x) { - push(@r, $symdata->at($x, $y)); - } - } - push(@c, \@r); - } - - my $lowertriangle = pdl(\@c); - } - my $ns = $symdata->shape()->at(0); - my $lowertriangle = zeroes($ns * ($ns + 1) / 2); - $symdata->squaretotri($lowertriangle); - -# -# if($SIZE (m) != (ns * (ns+1))/2) { -# barf("Wrong sized args for squaretotri"); -# } -# threadloop %{ -# loop(m) %{ -# $b () = $a (n0 => mna, n1 => nb); -# mna++; if(mna > nb) {mna = 0; nb ++;} -# %} -# -# lowertriangle=array([symdata[(x,y)] for x in xrange(symdata.shape[1]) -# for y in xrange(symdata.shape[0]) if y<=x]) - - my $kernel= modshogun::CustomKernel->new(); - $kernel->set_triangle_kernel_matrix_from_triangle($lowertriangle); - my $triangletriangle - =max(abs( - $indata->{$prefix.'matrix_triangletriangle'} - -$kernel->get_kernel_matrix())->flat); - $kernel->set_triangle_kernel_matrix_from_full($indata->{$prefix.'symdata'}); - my $fulltriangle - =max(abs( - $indata->{$prefix.'matrix_fulltriangle'} - -$kernel->get_kernel_matrix())->flat); - $kernel->set_full_kernel_matrix_from_full($indata->{$prefix.'data'}); - my $fullfull - =max(abs( - $indata->{$prefix.'matrix_fullfull'} - -$kernel->get_kernel_matrix())->flat); - - return &util::check_accuracy - ($indata->{$prefix.'accuracy'} - ,{ - triangletriangle=>$triangletriangle - , fulltriangle=>$fulltriangle - , fullfull=>$fullfull - } - ); -} - -sub _evaluate_pie -{ - my ($indata, $prefix) = @_; - my $pie=modshogun::PluginEstimate->new(); - my $feats=&util::get_features($indata, $prefix); - my $labels=modshogun::BinaryLabels->new($indata->{'classifier_labels'}); - $pie->set_labels($labels); - $pie->set_features($feats->{'train'}); - $pie->train(); - - my $fun = eval('modshogun::' . $indata->{$prefix.'name'}.'Kernel'); - my $kernel = $fun->new($feats->{'train'}, $feats->{'train'}, $pie); - my $km_train = max(abs( - $indata->{$prefix.'matrix_train'} - -$kernel->get_kernel_matrix())->flat); - - $kernel->init($feats->{'train'}, $feats->{'test'}); - $pie->set_features($feats->{'test'}); - my $km_test = max(abs( - $indata->{$prefix.'matrix_test'} - -$kernel->get_kernel_matrix())->flat); - -#PTZ121013 did not find get_confidences() anywhere in Labels it disappeared! - my $classified = max(abs( - $pie->apply()->get_confidences() - - $indata->{'classifier_classified'})); - - return &util::check_accuracy( - $indata->{$prefix.'accuracy'} - , { - km_train=>$km_train, km_test=>$km_test, classified=>$classified}); - -} - -sub _evaluate_top_fisher -{ - my ($indata, $prefix) = @_; - - my %feats; - my $wordfeats = &util::get_features($indata, $prefix); - - my $pos_train = modshogun::HMM->new($wordfeats->{'train'}, $indata->{$prefix.'N'}, $indata->{$prefix.'M'}, - $indata->{$prefix.'pseudo'}); - $pos_train->train(); - $pos_train->baum_welch_viterbi_train($modshogun::BW_NORMAL); - my $neg_train= modshogun::HMM->new($wordfeats->{'train'}, $indata->{$prefix.'N'}, $indata->{$prefix.'M'}, - $indata->{$prefix.'pseudo'}); - $neg_train->train(); - $neg_train->baum_welch_viterbi_train($modshogun::BW_NORMAL); - my $pos_test= modshogun::HMM->new($pos_train); - $pos_test->set_observations($wordfeats->{'test'}); - my $neg_test= modshogun::HMM->new($neg_train); - $neg_test->set_observations($wordfeats->{'test'}); - - if($indata->{$prefix.'name'} eq 'TOP'){ - $feats{'train'} = modshogun::TOPFeatures->new(10, $pos_train, $neg_train, 0, 0); - $feats{'test'} = modshogun::TOPFeatures->new(10, $pos_test, $neg_test, 0, 0); - }else{ - $feats{'train'} = modshogun::FKFeatures->new(10, $pos_train, $neg_train); - $feats{'train'}->set_opt_a(-1); #estimate prior - $feats{'test'} = modshogun::FKFeatures->new(10, $pos_test, $neg_test); - $feats{'test'}->set_a($feats{'train'}->get_a()); #use prior from training data - } - $prefix = 'kernel_'; - my $args = &util::get_args($indata, $prefix); - my $kernel = modshogun::PolyKernel->new($feats{'train'}, $feats{'train'}, @$args); - #$kernel->init($feats{'train'}, $feats{'train'}); - my $km_train = max(abs( - $indata->{$prefix.'matrix_train'} - -$kernel->get_kernel_matrix())->flat); - $kernel->init($feats{'train'}, $feats{'test'}); - my $km_test = max(abs( - $indata->{$prefix.'matrix_test'} - -$kernel->get_kernel_matrix())->flat); - - return &util::check_accuracy($indata->{$prefix.'accuracy'} - , {km_train=>$km_train, km_test=>$km_test}); - -} -######################################################################## -# public -######################################################################## - -sub test -{ - my ($indata) =@_; - my $prefix='topfk_'; - if(defined($indata->{$prefix.'name'})) { - return &_evaluate_top_fisher($indata, $prefix); - } - $prefix='kernel_'; - my @names=('Combined', 'AUC', 'Custom'); - foreach my $name (@names) { - if( $indata->{$prefix.'name'}eq $name) { - return eval('&_evaluate_'. lc $name . '($indata, $prefix)'); - } - } - @names=('HistogramWordString', 'SalzbergWordString'); - foreach my $name (@names) { - if($indata->{$prefix . 'name'} eq $name) { - return &_evaluate_pie($indata, $prefix); - } - } - return &_evaluate($indata, $prefix); -} - -1; - -__END__ - -=head1 - -Test Kernel - -=cut - diff --git a/tests/integration/perl_modular/preprocessor.pm b/tests/integration/perl_modular/preprocessor.pm deleted file mode 100644 index 6ddbc8bc401..00000000000 --- a/tests/integration/perl_modular/preprocessor.pm +++ /dev/null @@ -1,44 +0,0 @@ -package preprocessor; -use util; -use PDL; - -######################################################################## -# kernel computation -######################################################################## - -sub _evaluate (indata): -{ - my ($indata) = @_; - my $prefix = 'kernel_'; - my $feats = &util::get_features($indata, $prefix); - my $kfun = eval('modshogun::' . $indata->{$prefix.'name'}.'Kernel'); - my $kargs = &util::get_args($indata, $prefix); - - $prefix = 'preprocessor_'; - my $pargs = &util::get_args($indata, $prefix); - $feats = &util::add_preprocessor($indata->{$prefix.'name'}, $feats, $pargs); - - $prefix = 'kernel_'; - $kernel = $kfun->new($feats->{'train'}, $feats->{'train'}, @$kargs); - $km_train=max(abs( - $indata->{$prefix.'matrix_train'} - - $kernel->get_kernel_matrix())->flat); - $kernel->init($feats->{'train'}, $feats->{'test'}); - $km_test = max(abs( - $indata->{$prefix.'matrix_test'} - - $kernel->get_kernel_matrix())->flat); - - return &util::check_accuracy( - $indata->{$prefix.'accuracy'} - , {km_train=>$km_train, km_test=>$km_test}); -} -######################################################################## -# public -######################################################################## -sub test -{ - my ($indata) = @_; - return &_evaluate($indata); -} -1; -__END__ diff --git a/tests/integration/perl_modular/regression.pm b/tests/integration/perl_modular/regression.pm deleted file mode 100644 index 847c856f776..00000000000 --- a/tests/integration/perl_modular/regression.pm +++ /dev/null @@ -1,74 +0,0 @@ -package regression; -use util; -use PDL; -use modshogun; - -sub _evaluate (indata): -{ - my ($indata) = @_; - my $prefix = 'kernel_'; - my $feats = &util::get_features($indata, $prefix); - my $kargs = &util::get_args($indata, $prefix); - my $fun = eval('modshogun::' . $indata->{$prefix.'name'}.'Kernel'); - my $kernel = $fun->new($feats->{'train'}, $feats->{'train'}, @$kargs); - - $prefix='regression_'; - $kernel->{parallel}->set_num_threads($indata->{$prefix.'num_threads'}); - my $rfun = eval('modshogun::' . $indata->{$prefix.'name'}); - if($@) {#except NameError, e: - warn( "%s is disabled/unavailable!",$indata->{$prefix.'name'}); - return false; - } - my $labels = modshogun::RegressionLabels->new($indata->{$prefix.'labels'}); - if($indata->{$prefix.'type'} eq 'svm') { - $regression = $rfun->new( - $indata->{$prefix.'C'}, $indata->{$prefix.'epsilon'}, $kernel, $labels); - }elsif($indata->{$prefix.'type'} eq 'kernelmachine') { - $regression = $rfun->new($indata->{$prefix.'tau'}, $kernel, $labels); - }else{ - return false; - } - $regression->{parallel}->set_num_threads($indata->{$prefix.'num_threads'}); - if(defined($indata->{$prefix.'tube_epsilon'})) { - $regression->set_tube_epsilon($indata->{$prefix.'tube_epsilon'}); - } - $regression->train(); - - my $alphas=0; - my $bias=0; - my $sv=0; - if(defined($indata->{$prefix.'bias'})) { - $bias = abs($regression->get_bias()-$indata->{$prefix.'bias'}); - } - if(defined($indata->{$prefix.'alphas'})) { - foreach my $item (@{ $regression->get_alphas()->tolist()}) { - $alphas+=$item; - } - $alphas=abs($alphas-$indata->{$prefix.'alphas'}); - } - if(defined($indata->{$prefix.'support_vectors'})){ - foreach my $item (@{$inregression->get_support_vectors()->tolist()}) { - $sv+=$item; - } - $sv=abs($sv - $indata->{$prefix.'support_vectors'}); - } - $kernel->init($feats->{'train'}, $feats->{'test'}); - my $classified=max(abs( - $regression->apply()->get_labels()-$indata->{$prefix.'classified'})); - - return &util::check_accuracy($indata->{$prefix.'accuracy'} - , {alphas=>$alphas, - bias=>$bias, support_vectors=>$sv, classified=>$classified}); -} -######################################################################## -# public -######################################################################## -sub test -{ - my ($indata) = @_; - return &_evaluate($indata); -} -1; -__END__ - - diff --git a/tests/integration/perl_modular/test_all.sh b/tests/integration/perl_modular/test_all.sh deleted file mode 100755 index 515e766ba7e..00000000000 --- a/tests/integration/perl_modular/test_all.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -if test -z "$PERL" -then - PERL=perl -fi - -DATAPATH='../data' -exitcode=0 - -function test_all () { - datapath="$1" - - if echo "$datapath" | grep -q '\.\./data/tests' - then - continue - fi - - echo "*** Testing in $datapath" - sleep 1 - for file in $datapath; do - echo -n "$file" - echo -n -e "\t\t" - - if grep -q $file ../blacklist - then - echo 'SKIPPING' - else - output=`./test_one.pl "$file"` - ret=$? - - if [ $ret -eq 1 ] ; then - echo 'OK' - else - echo 'ERROR' - exitcode=1 - echo $output - fi - fi - done - sleep 1 - echo -} - -if [ -n "$1" ]; then - test_all "$DATAPATH/$1/*.m" -else - for i in $DATAPATH/*; do - test_all "$i/*.m" - done -fi -exit $exitcode diff --git a/tests/integration/perl_modular/test_one.pl b/tests/integration/perl_modular/test_one.pl deleted file mode 100755 index d1ef09661fa..00000000000 --- a/tests/integration/perl_modular/test_one.pl +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/perl -#TODO:PTZ121027 remove this hack by using use a Env{???} -use lib qw(. /usr/src/shogun/src/interfaces/perl_modular /usr/src/shogun/src/shogun); -use PDL; -use PDL::Char; - -use IO::File; - -import Devel::Trace 'trace'; - -use kernel; -use distance; -use classifier; -use clustering; -use distribution; -use regression; -use preprocessor; - -#... -our @SUPPORTED=('kernel', 'distance', 'classifier', 'clustering', 'distribution', - 'regression', 'preprocessor'); - -#use Shogun qw(Math_init_random); -use modshogun; - -sub _get_name_fun($) -{ - my $fnam = shift; - my $module; - if(my ($supported) = grep($fnam =~ /$_/, @SUPPORTED)) { - $module = $supported; - } - unless($module) { - printf('Module required for %s not supported yet!', $fnam); - return undef; - } - return $module . '::test'; -} - -=head3 _test_mfile - - a simple parser from m files to perl structures - -=cut - -sub _test_mfile { - my $fnam = shift; - my $mfile = IO::File->new($fnam, 'r') or return false; - my %indata = (); - - my $name_fun = &_get_name_fun($fnam); - unless($name_fun) { - return false; - } - while(my $line = <$mfile>) { - $line =~ s/[\s]//g; - $line =~ s/;$//; - (my $param = $line) =~ s/=.*//; - - if($param eq 'name') { - $indata{$param} = $line =~ m/.*='(.*)'/; - } elsif ($param eq 'kernel_symdata' or $param eq 'kernel_data') { - $indata{$param} = &_read_matrix($line); - } elsif ($param =~ /^kernel_matrix/ or - $param =~ /^distance_matrix/) { - $indata{$param} = &_read_matrix($line); - } elsif ($param =~ /data_train/ or $param =~ /data_test/) { - # data_{train,test} might be prepended by 'subkernelX_' - $indata{$param} = &_read_matrix($line); - } elsif ($param eq 'classifier_alphas' or $param eq 'classifier_support_vectors') { - ($indata{$param}) = $line =~ m/=(.*)$/; - unless($indata{$param}) { - # might be MultiClass SVM and hence matrix - $indata{$param} = &_read_matrix($line); - } - } elsif($param eq 'clustering_centers' or $param eq 'clustering_pairs') { - $indata{$param} = &_read_matrix($line); - } else { - unless($line =~ /'/) { - my $str_line; - if(($str_line) = $line =~ m/=\[(.*)\]$/g) { - $indata{$param} = &pdl(eval($str_line)); - } elsif(($str_line) = $line =~ m/=\{(.*)\}$/g) { - $indata{$param} = PDL::Char->new(eval($str_line)); - } else { - ($indata{$param}) = $line =~ m/=(.*)$/; - } - } else { - ($indata{$param}) = $line =~ m/='(.*)'/; - } - } - } - $mfile->close(); - my $fun = *{$name_fun}; - my $random = modshogun::Math->new(); - # seed random to constant value used at data file's creation - &modshogun::Math::init_random($indata{'init_random'}); - #is(&modshogun::Math::get_seed(), $indata{'init_random'}); - #$random->seed($indata{'init_random'}); - #= &modshogun::Math::random() - return $fun->(\%indata); -} - -sub _stringlike { - return 1 unless defined $_[0] && ref $_[0]; - return 1 if (blessed $_[0]) && - eval { - (("$_[0]" eq "$_[0]") && (($_[0] cmp $_[0])==0)) - }; - return; -} -sub _numberlike { - return 1, unless defined $_[0]; - - # L manpage notes that NaN != NaN, so we can verify that - # numeric conversion function works properly along with the - # comparison operator. - no warnings; - - return 1 if ((!ref $_[0]) || blessed($_[0])) && - eval { - ((0+$_[0]) == (0+$_[0])) && (($_[0] <=> $_[0])==0) - } && ($_[0] =~ /\d/); - return; -} -sub _read_matrix { - my $line = shift; - my $is_char = 0; - my ($str_line) = $line =~ m/\[(.*)\]/g; - unless($str_line) { - ($str_line) = $line =~ m/\{(.*)\}/g; - $is_char = 1; - } - my @lines = split(/;/, $str_line); - my @lis2d; - - foreach my $x (@lines) { - my @lis; - foreach my $y (split(/,/, $x)) { - if($y =~ /'/) { - $is_char = 1; - } - $y =~ s/'//g; - push(@lis, $y); - #$is_char ||= (&_stringlike($y) && !&_numberlike($y)); - } - push(@lis2d, \@lis); - } - if($is_char) { - my $m = PDL::Char->new(\@lis2d); - return $m; - } - PDL->new(\@lis2d); -} - -my $res = 1; -foreach my $filename (@ARGV) { - if($filename =~ /\.m$/) { - $res &&= &_test_mfile($filename); - } -} - -exit($res); - -__END__ - -=head1 - -Test one data file - -=cut diff --git a/tests/integration/perl_modular/test_svm_kernel_multiple.pl b/tests/integration/perl_modular/test_svm_kernel_multiple.pl deleted file mode 100755 index 043aaed2e95..00000000000 --- a/tests/integration/perl_modular/test_svm_kernel_multiple.pl +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env false -#!/usr/bin/env perl - -from numpy import random -from shogun.Classifier import LibSVM -from shogun.Features import RealFeatures, Labels -from shogun.Kernel import LinearKernel - - -num_feats=23 -num_vec=42 - -scale=2.1 -size_cache=10 - -C=0.017 -epsilon=1e-5 -tube_epsilon=1e-2 -svm=LibSVM() -svm.set_C(C, C) -svm.set_epsilon(epsilon) -svm.set_tube_epsilon(tube_epsilon) - -for i in xrange(3): - data_train=random.rand(num_feats, num_vec) - data_test=random.rand(num_feats, num_vec) - feats_train=RealFeatures(data_train) - feats_test=RealFeatures(data_test) - labels=Labels(random.rand(num_vec).round()*2-1) - - svm.set_kernel(LinearKernel(size_cache, scale)) - svm.set_labels(labels) - - kernel=svm.get_kernel() - print "kernel cache size: %s" % (kernel.get_cache_size()) - - kernel.init(feats_test, feats_test) - svm.train() - - kernel.init(feats_train, feats_test) - print svm.apply().get_labels() - - #kernel.remove_lhs_and_rhs() - - #import pdb - #pdb.set_trace() - diff --git a/tests/integration/perl_modular/tester.pl b/tests/integration/perl_modular/tester.pl deleted file mode 100755 index d626a4b6dde..00000000000 --- a/tests/integration/perl_modular/tester.pl +++ /dev/null @@ -1,272 +0,0 @@ -#!/usr/bin/perl -I ../../../src/interfaces/perl_modular -I . -#TODO:PTZ121027 remove this hack by using use a Env{???} -use lib qw(../../../src/interfaces/perl_modular); -use PDL; -use modshogun; -use generator qw(setup_tests get_fname blacklist get_test_mod run_test); - -use Test::More; -use Data::Dumper; -use File::Slurp; -use Getopt::Long; -use Pod::Usage; - -use Data::Dumper; -use Digest::MD5 qw(md5 md5_hex md5_base64); -use IO::String; -use IO::File; -use Data::Hexdumper qw(hexdump) ; -use File::Temp qw/tempfile tempdir mktemp/; -use File::Spec; -use Tie::IxHash; -#use MIME::Lite::HTML; -#use Algorithm::QuineMcCluskey::Util; -use Storable; -use Archive::Zip qw( :ERROR_CODES ); - - -sub typecheck -{ - my ($a, $b) = @_; - (my $rfa = ref($a)) =~ s/\(.*$//; - (my $rfb = ref($b)) =~ s/\(.*$//; - if($rfa =~ /Shogun.*Labels/ and $rfb =~ /Shogun.*Labels/) { - return 1; - } - return($rfa eq $rfb); -} - -#PTZ121006 there must be better ways -sub compare -{ - my($a, $b, $tolerance) = @_; - if(not &typecheck($a, $b)) { return 0;} - if(ref($a) =~ /PDL::ndarray/) { - if($tolerance) { - return (max(abs($a - $b)) < $tolerance); - } else { - return like($a,$b); - } - } elsif( &isinstance($a, modshogun::SGObject)) { - return like(Dumper($a), Dumper($b)); - } elsif(ref($a) =~ qr'ARRAY') { - if($#$a != $#$b) {return 0;} - while((my ($obj1) = pop(@$a)) && (my ($obj2) = pop(@$b))) { - if( not &compare($obj1, $obj2, $tolerance)) { - return 0; - } - } - return 1; - } - return $a <=> $b; -} - -sub compare_dbg { - my ($a, $b, $tolerance) = @_; - if(not &compare_dbg_helper($a, $b, $tolerance)) { -# import pdb; -# pdb.set_trace() - } -} - -sub compare_dbg_helper -{ - my ($a, $b, $tolerance) = @_; - my $rfa = ref($a); - my $rfb = ref($b); - if(not &typecheck($a, $b)) { - printf( "Type mismatch (type(a)=%s vs type(b)=%s)", ref($a), ref($b)); - return 0; - } - if(ref($a) =~ /PDL/) { - if ($tolerance){ - if (max(abs($a - $b)) < $tolerance) { - return 1; - }else{ - print "PDL Array mismatch > max_tol"; - print $a-$b; - return 0; - } - }else{ - if (&is_like($a, $b)){ - return 1; - }else{ - print "PDL Array mismatch"; - print $a-$b; - return 0; - } - } - } elsif(ref($a) =~ /modshogun::SGObject/){ - if(&like(Dumper($a), Dumper($b))) { - return 1; - } - print("a=", Dumper($a)); - print("b=", Dumper($b)); - return 0; - } elsif( $rfa =~ 'ARRAY') { - if($#a != $#b) { - printf( "Length mismatch (len(a)=%d vs len(b)=%d)", $#a, $#b); - return 0; - } - while(my ($obj1, $obj2) = each &zip($a,$b)) { - if( not &compare_dbg($obj1, $obj2, $tolerance)) { - return 0; - } - } - return 1; - } - - if ($a == $b) { - return 1; - } else { - print "a!=b"; - print "a=", $a; - print "b=", $b; - return 0; - } -} -sub tester -{ - my ($tests, $cmp_method, $tolerance, $failures, $missing) = @_; - foreach my $t (@$tests) { - my $n = 0; - my ($mod, $mod_name) = &get_test_mod($t); - if($mod && 0) { -#TODO::PTZ121109 parameter_list nowhere to be seen... maybe loadfile_parameter - $n = $#{$mod->{parameter_list}}; - unless($n >= 0) {next;} - if($@) { - warn( "%-60s ERROR (%s)", $t, $@); - next; - } - } - my $fname = ""; - foreach my $i (0..$n) { - $fname = &get_fname($mod_name, $i); - my $setting_str = sprintf("%s setting %d/%d", $t, $i+1, $n); - my $a = &run_test($mod, $mod_name, $i); - my $b = &read_file($fname);#slurp file... - if(&cmp_method($a, $b, $tolerance)) { - if(not $failures and not $missing) { - printf("%-60s OK", $setting_str); - } else { - if(not $missing) { - printf("%-60s ERROR", $setting_str); - } - } - #PTZ12106todo ... use ok(...); or so... - if($@) { - warn($setting_str, $@); - #except IOError, e: - if(not $failures) { - warn( "%-60s NO TEST", $setting_str); - } - #except Exception, e: - if (not $missing) { - printf("%-60s EXCEPTION %s", $setting_str, $e); - } - } - } - } - } -} - - -my $verbose = ''; # option variable with default value (false) -my $all = ''; # option variable with default value (false) -my $man = 0; -my $help = 0; - -my $debug = false; -my $failures = false; -my $tolerance = false; -my $missing = false; -my $cmp_method; - - -GetOptions ('verbose' => \$verbose, 'all' => \$all, - , 'debug!' => \$debug - , 'help|?' => \$help, man => \$man - , 'failures!' => \$failures - , 'tolerance!' => \$tolerance - , 'missing!' =>\$missing - ) or pod2usage(2); -pod2usage(1) if $help; -pod2usage(-exitstatus => 0, -verbose => 2) if $man; - -if($debug) { - $cmp_method= \&compare_dbg; -} else { - $cmp_method=\&compare; -} -my $tests = &setup_tests(\@ARGV); -&tester($tests, $cmp_method, $tolerance, $failures, $missing); - -__END__ - - -=head1 NAME - - tester.pl - Using Getopt::Long and Pod::Usage - - =head1 SYNOPSIS - - tester.pl [options] [ ...] - - Options: - --help brief help message - --man full documentation - --debug detailed debug output of objects that don't match - --failures show only failures - --missing show only missing tests - --tolerance tolerance used to estimate accuracy - -=head1 OPTIONS - -=over 8 - -=item B<--help> - - Print a brief help message and exits. - -=item B<--man> - - Prints the manual page and exits. - -=item B<--debug> - - detailed debug output of objects that don't match - -=item B<--failures> - show only failures - -=item B<--missing> - - show only missing tests - -=item B<--tolerance> - - tolerance used to estimate accuracy - -=back - -=head1 DESCRIPTION - - B will read the given input file(s) and do something - useful with the contents thereof. - - -=cut - - -=pod - - - #from Makefile.PL - - check-local: Makefile.perl - $(MAKE) -f $< test - - done - -=cut diff --git a/tests/integration/perl_modular/util.pm b/tests/integration/perl_modular/util.pm deleted file mode 100644 index 01912907279..00000000000 --- a/tests/integration/perl_modular/util.pm +++ /dev/null @@ -1,254 +0,0 @@ -package util; - -use modshogun; - -=pod - -use Shogun.Features;use Shogun.Preprocessor; -use Shogun.Distance;use Shogun.Kernel; - -=cut - -use PDL; -#use PDL::NiceSlice; - -use Data::Dumper; - -use base qw(Exporter); -our %EXPORT_TAGS = -( - #FIELDS => [ @EXPORT_OK, @EXPORT ] - feats => [qw(check_accuracy get_args get_features get_feats_simple get_feats_string get_feats_string_complex )] - , basic => [qw()] - , misc => [qw(add_preprocessor)] -); - -Exporter::export_tags(qw//); -Exporter::export_ok_tags(qw/feats basic misc/); - -sub check_accuracy { - my ($accuracy, $kwargs) = @_; - my $acc=&double($accuracy); - my @output; - while( my ($key, $val) = each(%$kwargs)) { - if($val) { - push(@output, sprintf('%s: %e', $key, $val)); - } - } - printf("\n". join(', ', @output) . ' <--- accuracy: %e', $accuracy); - foreach my $val (values(%$kwargs)) { - if($val > $acc) { - return 0;#false - } - } - return 1;#true -} - -sub get_args { - my ($indata, $prefix) = @_; - -=head3 get_args - - Slightly esoteric function to build a tuple to be used as argument to - constructor calls. - - Perl dicts are not ordered, so we have to look at the number in - the parameter's name and insert items appropriately into an - ordered list - -=cut - - my $ident=$prefix . 'arg'; - # need to pregenerate list for using indices in loop - my @args = (); - foreach my $i (keys(%$indata)) { - unless($i =~ /$ident/) { next;} - my ($idx) = $i =~ m/$ident(\d+)/; - unless($idx >= 0) { - warn( 'Wrong indata data %s: "%s"!' , $ident, $i); - } - if($i =~ /_distance/) { # DistanceKernel - $args[$idx] = eval('modshogun::' . $indata->{$i})->new(); - }else{ - $args[$idx] = eval($indata->{$i}); - #except TypeError: # no bool - if($@) { - $args[$idx] = $indata->{$i}; - } - #False and True... - if($args[$idx] eq 'False') { - $args[$idx] = 0;#$modshogun::False; - } elsif($args[$idx] eq 'True') { - $args[$idx] = 1;#$modshogun::True; - } - } - } - # weed out superfluous Nones - @args = grep(defined($_), @args); - return \@args; #&filter(lambda arg: arg is not None, $args); -} - -sub get_features -{ - my ($indata, $prefix) = @_; - my $fclass=$prefix.'feature_class'; - if($indata->{$fclass} eq 'simple') { - return &get_feats_simple($indata, $prefix); - } elsif($indata->{$fclass} eq 'string') { - return &get_feats_string($indata, $prefix); - }elsif ($indata->{$fclass} eq 'string_complex') { - return &get_feats_string_complex($indata, $prefix); - } elsif($indata->{$fclass} eq 'wd') { - return &get_feats_wd($indata, $prefix); - } else { - croak('Unknown feature class %s!', $indata->{$prefix.'feature_class'}); - } -} - -sub get_feats_simple { - my ($indata, $prefix) = @_; - - my $ftype=$indata->{$prefix.'feature_type'}; - - # have to explicitely set data type for numpy if not real - my %as_types=( - 'Byte'=> \&byte, - 'Real'=> \&double, - 'Word'=> \&ushort - , 'Char' => \&byte - ); - unless(defined($as_types{$ftype})) { - croak("PDL type conversion $ftype not found"); - } - my $data_train = $as_types{$ftype}->($indata->{$prefix.'data_train'}); - my $data_test = $as_types{$ftype}->($indata->{$prefix.'data_test'}); - my $ftrain; - my $ftest; - if($ftype eq 'Byte' or $ftype eq 'Char') { - my $alphabet= ${'modshogun::' . $indata->{$prefix.'alphabet'}}; - $ftrain=eval('modshogun::' . $ftype . 'Features')->new($alphabet); - $ftest =eval('modshogun::' . $ftype . 'Features')->new($alphabet); - $ftrain->copy_feature_matrix($data_train); - $ftest->copy_feature_matrix($data_test); - } else { -#tests (for 'Real' type) matrix_from_pdl(arg1, ST(0), PDL_D)) ; is_pdl_matrix(ST(0), PDL_D); - $ftrain = eval('modshogun::' . $ftype . 'Features')->new($data_train); - $ftest = eval('modshogun::' . $ftype . 'Features')->new($data_test); - } - if($indata->{$prefix.'name'} =~ /Sparse/ or ( - defined($indata->{'classifier_type'}) and - $indata->{'classifier_type'} eq 'linear') - ) { -# _v = is_pdl_sparse_matrix(ST(0), PDL_D); - my $sparse_train=eval('modshogun::' .'Sparse'.$ftype.'Features')->new(); - $sparse_train->obtain_from_simple($ftrain); - - my $sparse_test=eval('modshogun::' .'Sparse'.$ftype.'Features')->new(); - $sparse_test->obtain_from_simple($ftest); - - return {'train' => $sparse_train, 'test' => $sparse_test}; - }else{ - return {'train' => $ftrain, 'test' => $ftest}; - } -} - -sub get_feats_string { - my ($indata, $prefix) = @_; - my $ftype=$indata->{$prefix.'feature_type'}; - my $alphabet=${'modshogun::' . $indata->{$prefix.'alphabet'}}; - my %feats=( - 'train'=> eval('modshogun::' . 'String'.$ftype.'Features')->new($alphabet) - ,'test'=> eval('modshogun::' . 'String'.$ftype.'Features')->new($alphabet) - ); - #$feats{'train'}->set_features($indata->{$prefix.'data_train'}->slice('0:-1,(0)')); - #$feats{'test'}->set_features($indata->{$prefix.'data_test'}->slice('0:-1,(0)')); - $feats{'train'}->set_features($indata->{$prefix.'data_train'}); - $feats{'test'}->set_features($indata->{$prefix.'data_test'}); - return \%feats; -} -sub get_feats_string_complex -{ - my ($indata, $prefix) = @_; - my $alphabet = ${'modshogun::' . $indata->{$prefix.'alphabet'}}; - my %feats=( - 'train'=> modshogun::StringCharFeatures->new($alphabet) - , 'test'=> modshogun::StringCharFeatures->new($alphabet) - ); -#PTZ121011 ->server might be needed to be sure it is a dense object. - #my $data_train = $indata->{$prefix.'data_train'}->slice('0:-1,(0)'); #->nslice([0,-1], 0); - #my $data_test = $indata->{$prefix.'data_test'}->slice('0:-1,(0)'); #->nslice([0,-1], 0); - my $data_train = $indata->{$prefix.'data_train'}; - my $data_test = $indata->{$prefix.'data_test'}; - if($alphabet == $modshogun::CUBE) # data_{train,test} ints due to test.py:_read_matrix - { ##map { $a($_ - 1) .= $_; } (1..$a->nelem); # Lots of little ops - #map { $data_train->nslice($_) = chr($data_train->nslice($_)) } (0..$data_train->nelem - 1); - #map { $data_test->nslice($_) = chr($data_test->nslice($_)) } (0..$data_test->nelem - 1); - - } - $feats{'train'}->set_features($data_train); - $feats{'test'}->set_features($data_test); - - my $feat=eval('modshogun::' .'String'.$indata->{$prefix.'feature_type'}."Features")->new($alphabet); - $feat->obtain_from_char($feats{'train'}, $indata->{$prefix.'order'}-1, - $indata->{$prefix.'order'}, $indata->{$prefix.'gap'}, - ${'modshogun::' . $indata->{$prefix.'reverse'}}); - $feats{'train'}=$feat; - - $feat=eval('modshogun::' .'String'.$indata->{$prefix.'feature_type'}."Features")->new($alphabet); - $feat->obtain_from_char($feats{'test'}, $indata->{$prefix.'order'}-1, - $indata->{$prefix.'order'}, $indata->{$prefix.'gap'}, - ${'modshogun::' . $indata->{$prefix.'reverse'}}); - $feats{'test'}=$feat; - - if( $indata->{$prefix.'feature_type'} eq 'Word' or - $indata->{$prefix.'feature_type'} eq 'Ulong'){ - my $name = 'Sort' .$indata->{$prefix.'feature_type'}.'String'; - return &add_preprocessor($name, \%feats); - } else { - return \%feats; - } -} - -sub get_feats_wd -{ - my ($indata, $prefix) = @_; - my $order=$indata->{$prefix.'order'}; - my %feats; - - my $charfeat=&modshogun::StringCharFeatures(&modshogun::DNA); - my $charfeat->set_features(@{$indata->{$prefix.'data_train'}[0]}); - my $bytefeat=&modshogun::StringByteFeatures(&modshogun::RAWDNA); - $bytefeat->obtain_from_char($charfeat, 0, 1, 0, false); - $feats{'train'}=&modshogun::WDFeatures($bytefeat, $order, $order); - - $charfeat=&modshogun::StringCharFeatures(&modshogun::DNA); - $charfeat->set_features(@{$indata->{$prefix.'data_test'}[0]}); - $bytefeat=&modshogun::StringByteFeatures(&modshogun::RAWDNA); - $bytefeat->obtain_from_char($charfeat, 0, 1, 0, false); - $feats{'test'}=&modshogun::WDFeatures($bytefeat, $order, $order); - - return \%feats; -} - -sub add_preprocessor -{ - my ($name, $feats, $args) = @_; - #my $fun=*{$name}; - #my $preproc=*{$name.'::new'}->($name, @$args); - my $preproc=eval('modshogun::' . $name)->new(@$args); - $preproc->init($feats->{'train'}); - $feats->{'train'}->add_preprocessor($preproc); - $feats->{'train'}->apply_preprocessor(); - $feats->{'test'}->add_preprocessor($preproc); - $feats->{'test'}->apply_preprocessor(); - return $feats; -} - -1; -__END__ - -=head1 - -Utilities for testing - -=cut diff --git a/tests/integration/python_modular/CMakeLists.txt b/tests/integration/python_modular/CMakeLists.txt deleted file mode 100644 index b2bee8e956c..00000000000 --- a/tests/integration/python_modular/CMakeLists.txt +++ /dev/null @@ -1,41 +0,0 @@ -INCLUDE(PythonEnvironment) -GET_PYTHON_ENV() - -FOREACH(TEST_SRC ${INTEGRATION_TESTS}) - STRING(REGEX REPLACE ".*/(.*).m" "\\1" TEST_NAME ${TEST_SRC}) - SET(EXEC_EXECUTABLE ${PYTHON_EXECUTABLE}) - SET(EXEC_FILE ${CMAKE_CURRENT_SOURCE_DIR}/test_one.py) - SET(EXEC_ARGS ${TEST_SRC}) - add_test(NAME integration-python_modular-${TEST_NAME} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - COMMAND ${CMAKE_COMMAND} - -DEXECUTABLE=${EXEC_EXECUTABLE} - -DFILE=${EXEC_FILE} - -DARGS=${EXEC_ARGS} - -DENABLE_PYTHON_DEBUG=${ENABLE_PYTHON_DEBUG} - -DGDB_COMMAND=${GDB_COMMAND} - -DGDB_SCRIPT=${GDB_DEFAULT_SCRIPT} - -P ${CMAKE_MODULE_PATH}/checkgdb.cmake) - set_property(TEST integration-python_modular-${TEST_NAME} PROPERTY - ENVIRONMENT "${PYTHON_ENV_VARS}") -ENDFOREACH() - -FILE(GLOB PYTHON_MODULAR_INTEGRATION_TESTS tests/*.py) -FOREACH(TEST_SRC ${PYTHON_MODULAR_INTEGRATION_TESTS}) - STRING(REGEX REPLACE ".*/(.*).py" "\\1" TEST_NAME ${TEST_SRC}) - SET(EXEC_EXECUTABLE ${PYTHON_EXECUTABLE}) - SET(EXEC_FILE ${CMAKE_CURRENT_SOURCE_DIR}/tester.py) - SET(EXEC_ARGS ${TEST_NAME}.py) - add_test(NAME integration-python_modular-tester-${TEST_NAME} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - COMMAND ${CMAKE_COMMAND} - -DEXECUTABLE=${EXEC_EXECUTABLE} - -DFILE=${EXEC_FILE} - -DARGS=${EXEC_ARGS} - -DENABLE_PYTHON_DEBUG=${ENABLE_PYTHON_DEBUG} - -DGDB_COMMAND=${GDB_COMMAND} - -DGDB_SCRIPT=${GDB_DEFAULT_SCRIPT} - -P ${CMAKE_MODULE_PATH}/checkgdb.cmake) - set_property(TEST integration-python_modular-tester-${TEST_NAME} PROPERTY - ENVIRONMENT "${PYTHON_ENV_VARS}") -ENDFOREACH() diff --git a/tests/integration/python_modular/README b/tests/integration/python_modular/README deleted file mode 100644 index 4f1b3e9cfe2..00000000000 --- a/tests/integration/python_modular/README +++ /dev/null @@ -1,8 +0,0 @@ -This directory contains mostly legacy regression test scripts that will soon no -longer be used. The new way to check for regressions is via the following two -scripts: - - generator.py - to generate regression tests and - tester.py - to check for regressions - -So tester.py is sufficient. diff --git a/tests/integration/python_modular/classifier.py b/tests/integration/python_modular/classifier.py deleted file mode 100644 index 91885fa7ab6..00000000000 --- a/tests/integration/python_modular/classifier.py +++ /dev/null @@ -1,172 +0,0 @@ -""" -Test Classifier -""" - -from numpy import double -from modshogun import BinaryLabels -from modshogun import * -from modshogun import * -from modshogun import * - -import util - -def _get_machine (indata, prefix, feats): - if indata[prefix+'type']=='kernel': - pre='kernel_' - kargs=util.get_args(indata, pre) - kfun=eval(indata[pre+'name']+'Kernel') - machine=kfun(feats['train'], feats['train'], *kargs) - - if indata[pre+'name']=='Linear': - normalizer=eval(indata[pre+'normalizer']+'()') - machine.set_normalizer(normalizer) - machine.init(feats['train'], feats['train']) - - machine.parallel.set_num_threads(indata[prefix+'num_threads']) - elif indata[prefix+'type']=='knn': - pre='distance_' - dargs=util.get_args(indata, pre) - dfun=eval(indata[pre+'name']) - machine=dfun(feats['train'], feats['train'], *dargs) - machine.parallel.set_num_threads(indata[prefix+'num_threads']) - else: - machine=None - - return machine - - -def _get_results_alpha_and_sv(indata, prefix, classifier): - if prefix+'alpha_sum' not in indata and \ - prefix+'sv_sum' not in indata: - return None, None - - a=0 - sv=0 - if prefix+'label_type' in indata and \ - indata[prefix+'label_type']=='series': - for i in range(classifier.get_num_svms()): - subsvm=classifier.get_svm(i) - for item in subsvm.get_alphas().tolist(): - a+=item - for item in subsvm.get_support_vectors().tolist(): - sv+=item - - a=abs(a-indata[prefix+'alpha_sum']) - sv=abs(sv-indata[prefix+'sv_sum']) - else: - for item in classifier.get_alphas().tolist(): - a+=item - a=abs(a-indata[prefix+'alpha_sum']) - for item in classifier.get_support_vectors().tolist(): - sv+=item - sv=abs(sv-indata[prefix+'sv_sum']) - - return a, sv - - -def _get_results (indata, prefix, classifier, machine=None, feats=None): - res={ - 'alphas':0, - 'bias':0, - 'sv':0, - 'accuracy':indata[prefix+'accuracy'], - } - - if prefix+'bias' in indata: - res['bias']=abs(classifier.get_bias()-indata[prefix+'bias']) - - res['alphas'], res['sv']=_get_results_alpha_and_sv( - indata, prefix, classifier) - - ctype=indata[prefix+'type'] - if ctype=='kernel' or ctype=='knn': - machine.init(feats['train'], feats['test']) - else: - ctypes=('linear', 'perceptron', 'lda', 'wdsvmocas') - if ctype in ctypes: - classifier.set_features(feats['test']) - - res['classified']=max(abs( - classifier.apply().get_values()-indata[prefix+'classified'])) - return res - - -def _evaluate (indata): - prefix='classifier_' - ctype=indata[prefix+'type'] - if indata[prefix+'name']=='KNN': - feats=util.get_features(indata, 'distance_') - elif ctype=='kernel': - feats=util.get_features(indata, 'kernel_') - else: - feats=util.get_features(indata, prefix) - - machine=_get_machine(indata, prefix, feats) - - try: - fun=eval(indata[prefix+'name']) - except NameError as e: - print("%s is disabled/unavailable!"%indata[prefix+'name']) - return False - - # cannot refactor into function, because labels is unrefed otherwise - if prefix+'labels' in indata: - labels=BinaryLabels(double(indata[prefix+'labels'])) - if ctype=='kernel': - classifier=fun(indata[prefix+'C'], machine, labels) - elif ctype=='linear': - classifier=fun(indata[prefix+'C'], feats['train'], labels) - elif ctype=='knn': - classifier=fun(indata[prefix+'k'], machine, labels) - elif ctype=='lda': - classifier=fun(indata[prefix+'gamma'], feats['train'], labels) - elif ctype=='perceptron': - classifier=fun(feats['train'], labels) - elif ctype=='wdsvmocas': - classifier=fun(indata[prefix+'C'], indata[prefix+'degree'], - indata[prefix+'degree'], feats['train'], labels) - else: - return False - else: - classifier=fun(indata[prefix+'C'], machine) - - if classifier.get_name() == 'LibLinear': - print(classifier.get_name(), "yes") - classifier.set_liblinear_solver_type(L2R_LR) - - classifier.parallel.set_num_threads(indata[prefix+'num_threads']) - if ctype=='linear': - if prefix+'bias' in indata: - classifier.set_bias_enabled(True) - else: - classifier.set_bias_enabled(False) - if ctype=='perceptron': - classifier.set_learn_rate=indata[prefix+'learn_rate'] - classifier.set_max_iter=indata[prefix+'max_iter'] - if prefix+'epsilon' in indata: - try: - classifier.set_epsilon(indata[prefix+'epsilon']) - except AttributeError: - pass - if prefix+'max_train_time' in indata: - classifier.set_max_train_time(indata[prefix+'max_train_time']) - if prefix+'linadd_enabled' in indata: - classifier.set_linadd_enabled(indata[prefix+'linadd_enabled']) - if prefix+'batch_enabled' in indata: - classifier.set_batch_computation_enabled(indata[prefix+'batch_enabled']) - - classifier.train() - - res=_get_results(indata, prefix, classifier, machine, feats) - return util.check_accuracy(res['accuracy'], - alphas=res['alphas'], bias=res['bias'], sv=res['sv'], - classified=res['classified']) - - -######################################################################## -# public -######################################################################## - -def test (indata): - return _evaluate(indata) - diff --git a/tests/integration/python_modular/clustering.py b/tests/integration/python_modular/clustering.py deleted file mode 100644 index 0dc00d8bf39..00000000000 --- a/tests/integration/python_modular/clustering.py +++ /dev/null @@ -1,50 +0,0 @@ -""" -Test Clustering -""" - -from modshogun import EuclideanDistance -from modshogun import * - -import util - - -def _evaluate (indata): - if 'clustering_k' in indata: - first_arg=indata['clustering_k'] - elif 'clustering_merges' in indata: - first_arg=indata['clustering_merges'] - else: - return False - - feats=util.get_features(indata, 'distance_') - dfun=eval(indata['distance_name']) - distance=dfun(feats['train'], feats['train']) - - cfun=eval(indata['clustering_name']) - clustering=cfun(first_arg, distance) - clustering.train() - - if 'clustering_radi' in indata: - radi=max(abs(clustering.get_radiuses()-indata['clustering_radi'])) - centers=max(abs(clustering.get_cluster_centers().flatten() - \ - indata['clustering_centers'].flat)) - return util.check_accuracy(indata['clustering_accuracy'], - radi=radi, centers=centers) - elif 'clustering_merge_distance' in indata: - merge_distance=max(abs(clustering.get_merge_distances()- \ - indata['clustering_merge_distance'])) - pairs=max(abs(clustering.get_cluster_pairs()- \ - indata['clustering_pairs']).flat) - return util.check_accuracy(indata['clustering_accuracy'], - merge_distance=merge_distance, pairs=pairs) - else: - return util.check_accuracy(indata['clustering_accuracy']) - - -######################################################################## -# public -######################################################################## - -def test (indata): - return _evaluate(indata) - diff --git a/tests/integration/python_modular/distance.py b/tests/integration/python_modular/distance.py deleted file mode 100644 index e67a13188ee..00000000000 --- a/tests/integration/python_modular/distance.py +++ /dev/null @@ -1,34 +0,0 @@ -""" -Test Distance -""" - -from modshogun import * - -import util - - -def _evaluate (indata): - prefix='distance_' - feats=util.get_features(indata, prefix) - - dfun=eval(indata[prefix+'name']) - dargs=util.get_args(indata, prefix) - distance=dfun(feats['train'], feats['train'], *dargs) - - dm_train=max(abs( - indata[prefix+'matrix_train']-distance.get_distance_matrix()).flat) - distance.init(feats['train'], feats['test']) - dm_test=max(abs( - indata[prefix+'matrix_test']-distance.get_distance_matrix()).flat) - - return util.check_accuracy( - indata[prefix+'accuracy'], dm_train=dm_train, dm_test=dm_test) - - -######################################################################## -# public -######################################################################## - -def test (indata): - return _evaluate(indata) - diff --git a/tests/integration/python_modular/distribution.py b/tests/integration/python_modular/distribution.py deleted file mode 100644 index ee7c0246549..00000000000 --- a/tests/integration/python_modular/distribution.py +++ /dev/null @@ -1,62 +0,0 @@ -""" -Test Distribution -""" -from numpy import inf, nan -from modshogun import * - -import util - -def _evaluate (indata): - prefix='distribution_' - feats=util.get_features(indata, prefix) - - if indata[prefix+'name']=='HMM': - distribution=HMM(feats['train'], indata[prefix+'N'], - indata[prefix+'M'], indata[prefix+'pseudo']) - distribution.train() - distribution.baum_welch_viterbi_train(BW_NORMAL) - else: - dfun=eval(indata[prefix+'name']) - distribution=dfun(feats['train']) - distribution.train() - - likelihood=distribution.get_log_likelihood_sample() - num_examples=feats['train'].get_num_vectors() - num_param=distribution.get_num_model_parameters() - derivatives=0 - for i in range(num_param): - for j in range(num_examples): - val=distribution.get_log_derivative(i, j) - if val!=-inf and val!=nan: # only consider sparse matrix! - derivatives+=val - - derivatives=abs(derivatives-indata[prefix+'derivatives']) - likelihood=abs(likelihood-indata[prefix+'likelihood']) - - if indata[prefix+'name']=='HMM': - best_path=0 - best_path_state=0 - for i in range(indata[prefix+'num_examples']): - best_path+=distribution.best_path(i) - for j in range(indata[prefix+'N']): - best_path_state+=distribution.get_best_path_state(i, j) - - best_path=abs(best_path-indata[prefix+'best_path']) - best_path_state=abs(best_path_state-\ - indata[prefix+'best_path_state']) - - return util.check_accuracy(indata[prefix+'accuracy'], - derivatives=derivatives, likelihood=likelihood, - best_path=best_path, best_path_state=best_path_state) - else: - return util.check_accuracy(indata[prefix+'accuracy'], - derivatives=derivatives, likelihood=likelihood) - - -######################################################################## -# public -######################################################################## - -def test (indata): - return _evaluate(indata) - diff --git a/tests/integration/python_modular/generator.py b/tests/integration/python_modular/generator.py deleted file mode 100755 index 4778735729f..00000000000 --- a/tests/integration/python_modular/generator.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python - -import os -import sys -import pickle - -example_dir = '../../../examples/undocumented/python_modular' -test_dir = '../../../tests/integration/data/python%d-tests' % sys.version_info[0] -blacklist = ("__init__.py", - "classifier_libsvm_minimal_modular.py", - "classifier_multiclassocas_modular.py", - "modelselection_grid_search_kernel.py", - "serialization_string_kernels_modular.py", - "kernel_fisher_modular.py", - "kernel_top_modular.py", - "distribution_hmm_modular.py", - "kernel_director_linear_modular.py", - "features_director_dot_modular.py", - "converter_tdistributedstochasticneighborembedding_modular.py", - "evaluation_director_contingencytableevaluation_modular.py", - "distance_director_euclidean_modular.py", - "classifier_multiclass_ecoc_random.py", - "statistics_hsic.py", - "statistics_quadratic_time_mmd.py", - "transfer_multitask_clustered_logistic_regression.py", - "mathematics_logdet.py", - "classifier_svmlight_batch_linadd_modular.py", - "preprocessor_randomfouriergausspreproc_modular.py", -#the tests below all fail on travis but work fine on our buildbot -# "classifier_lda_modular.py", -# "classifier_liblinear_modular.py", -# "converter_tdistributedstochasticneighborembedding_modular.py", -# "distance_mahalanobis_modular.py", -# "mathematics_sparseinversecovariance_modular.py", -# "preprocessor_dimensionreductionpreprocessor_modular.py", -# "preprocessor_kernelpca_modular.py", -# "preprocessor_pca_modular.py", -# "regression_kernel_ridge_modular.py", -# "regression_least_squares_modular.py", -# "regression_linear_ridge_modular.py", -# "regression_svrlight_modular.py", -# "features_dense_protocols_modular.py", -# "features_dense_zero_copy_modular.py", -# "transfer_multitask_l12_logistic_regression.py", -# "transfer_multitask_trace_logistic_regression.py", - ) - -def get_fname(mod_name, i): - return os.path.join(test_dir, mod_name + str(i) + '.txt') - -def setup_tests(tests=[]): - os.chdir(example_dir) - - if not len(tests): - tests = os.listdir(".") - tests.sort() - else: - tests = [ os.path.basename(t) for t in tests ] - - sys.path.insert(0, '.') - return tests - -def check_for_function(fname): - for l in open(fname).readlines(): - if l.startswith("def "): - return True - return False - -def get_test_mod(t): - if t.endswith(".py") and not t.startswith('.') and t not in blacklist: - mod_name = t[:-3] - - if not check_for_function(t): - raise Exception("ERROR (no function)") - - return __import__(mod_name), mod_name - -def run_test(mod, mod_name, i): - fname = get_fname(mod_name, i) - par=mod.parameter_list[i] - a = getattr(mod, mod_name)(*par) - return a - -def generator(tests): - for t in tests: - try: - mod, mod_name = get_test_mod(t) - except TypeError: - continue - except Exception as e: - print(t, e) - continue - fname = "" - - print("%-60s" % mod_name) - #print("%+60s" % "...") - try: - for i in range(len(mod.parameter_list)): - fname = get_fname(mod_name, i) - a = run_test(mod, mod_name, i) - pickle.dump(a,open(fname, "wb"),0) - print("OK") - except Exception as e: - print("ERROR generating '%s' using '%s'" % (fname,t)) - print(e) - continue - -if __name__=='__main__': - from optparse import OptionParser - op=OptionParser() - op.set_usage("[ ...]") - (opts, args)=op.parse_args() - tests = setup_tests(args) - generator(tests) diff --git a/tests/integration/python_modular/kernel.py b/tests/integration/python_modular/kernel.py deleted file mode 100644 index dbfbeb05178..00000000000 --- a/tests/integration/python_modular/kernel.py +++ /dev/null @@ -1,227 +0,0 @@ -""" -Test Kernel -""" - -from modshogun import * -from modshogun import * -from modshogun import * -from modshogun import * -from modshogun import PluginEstimate -from modshogun import HMM, BW_NORMAL -from numpy import array, ushort, ubyte, double - -import util - -######################################################################## -# kernel computation -######################################################################## - -def _evaluate (indata, prefix): - feats=util.get_features(indata, prefix) - kfun=eval(indata[prefix+'name']+'Kernel') - kargs=util.get_args(indata, prefix) - kernel=kfun(*kargs) - if prefix+'normalizer' in indata: - kernel.set_normalizer(eval(indata[prefix+'normalizer']+'()')) - - kernel.init(feats['train'], feats['train']) - km_train=max(abs( - indata[prefix+'matrix_train']-kernel.get_kernel_matrix()).flat) - kernel.init(feats['train'], feats['test']) - km_test=max(abs( - indata[prefix+'matrix_test']-kernel.get_kernel_matrix()).flat) - - return util.check_accuracy( - indata[prefix+'accuracy'], km_train=km_train, km_test=km_test) - - -def _get_subkernels (indata, prefix): - subkernels={} - prefix=prefix+'subkernel' - len_prefix=len(prefix) - - # loop through indata (unordered dict) to gather subkernel data - for key in indata: - if key.find(prefix)==-1: - continue - - # get subkernel's number - try: - num=key[len_prefix] - except ValueError: - raise ValueError('Cannot find number for subkernel: "%s"!' % data) - - # get item's name - name=key[len_prefix+2:] - - # append new item - if num not in subkernels: - subkernels[num]={} - subkernels[num][name]=indata[key] - - # got all necessary information in new structure, now create a kernel - # object for each subkernel - for num, data in subkernels.items(): - fun=eval(data['name']+'Kernel') - args=util.get_args(data, '') - subkernels[num]['kernel']=fun(*args) - - return subkernels - - -def _evaluate_combined (indata, prefix): - kernel=CombinedKernel() - feats={'train':CombinedFeatures(), 'test':CombinedFeatures()} - - subkernels=_get_subkernels(indata, prefix) - for subk in subkernels.values(): - feats_subk=util.get_features(subk, '') - feats['train'].append_feature_obj(feats_subk['train']) - feats['test'].append_feature_obj(feats_subk['test']) - kernel.append_kernel(subk['kernel']) - - kernel.init(feats['train'], feats['train']) - km_train=max(abs( - indata['kernel_matrix_train']-kernel.get_kernel_matrix()).flat) - kernel.init(feats['train'], feats['test']) - km_test=max(abs( - indata['kernel_matrix_test']-kernel.get_kernel_matrix()).flat) - - return util.check_accuracy(indata[prefix+'accuracy'], - km_train=km_train, km_test=km_test) - - -def _evaluate_auc (indata, prefix): - subk=_get_subkernels(indata, prefix)['0'] - feats_subk=util.get_features(subk, '') - subk['kernel'].init(feats_subk['train'], feats_subk['test']) - - feats={ - 'train': WordFeatures(indata[prefix+'data_train'].astype(ushort)), - 'test': WordFeatures(indata[prefix+'data_test'].astype(ushort)) - } - kernel=AUCKernel(10, subk['kernel']) - - kernel.init(feats['train'], feats['train']) - km_train=max(abs( - indata[prefix+'matrix_train']-kernel.get_kernel_matrix()).flat) - kernel.init(feats['train'], feats['test']) - km_test=max(abs( - indata[prefix+'matrix_test']-kernel.get_kernel_matrix()).flat) - - return util.check_accuracy(indata[prefix+'accuracy'], - km_train=km_train, km_test=km_test) - - -def _evaluate_custom (indata, prefix): - feats={ - 'train': RealFeatures(indata[prefix+'data']), - 'test': RealFeatures(indata[prefix+'data']) - } - - symdata=indata[prefix+'symdata'] - lowertriangle=array([symdata[(x,y)] for x in range(symdata.shape[1]) - for y in range(symdata.shape[0]) if y<=x]) - kernel=CustomKernel() - kernel.set_triangle_kernel_matrix_from_triangle(lowertriangle) - triangletriangle=max(abs( - indata[prefix+'matrix_triangletriangle']-kernel.get_kernel_matrix()).flat) - kernel.set_triangle_kernel_matrix_from_full(indata[prefix+'symdata']) - fulltriangle=max(abs( - indata[prefix+'matrix_fulltriangle']-kernel.get_kernel_matrix()).flat) - kernel.set_full_kernel_matrix_from_full(indata[prefix+'data']) - fullfull=max(abs( - indata[prefix+'matrix_fullfull']-kernel.get_kernel_matrix()).flat) - - return util.check_accuracy(indata[prefix+'accuracy'], - triangletriangle=triangletriangle, fulltriangle=fulltriangle, - fullfull=fullfull) - - -def _evaluate_pie (indata, prefix): - pie=PluginEstimate() - feats=util.get_features(indata, prefix) - labels=BinaryLabels(double(indata['classifier_labels'])) - pie.set_labels(labels) - pie.set_features(feats['train']) - pie.train() - - fun=eval(indata[prefix+'name']+'Kernel') - kernel=fun(feats['train'], feats['train'], pie) - km_train=max(abs( - indata[prefix+'matrix_train']-kernel.get_kernel_matrix()).flat) - - kernel.init(feats['train'], feats['test']) - pie.set_features(feats['test']) - km_test=max(abs( - indata[prefix+'matrix_test']-kernel.get_kernel_matrix()).flat) - classified=max(abs( - pie.apply().get_values()-indata['classifier_classified'])) - - return util.check_accuracy(indata[prefix+'accuracy'], - km_train=km_train, km_test=km_test, classified=classified) - - -def _evaluate_top_fisher (indata, prefix): - feats={} - wordfeats=util.get_features(indata, prefix) - - pos_train=HMM(wordfeats['train'], indata[prefix+'N'], indata[prefix+'M'], - indata[prefix+'pseudo']) - pos_train.train() - pos_train.baum_welch_viterbi_train(BW_NORMAL) - neg_train=HMM(wordfeats['train'], indata[prefix+'N'], indata[prefix+'M'], - indata[prefix+'pseudo']) - neg_train.train() - neg_train.baum_welch_viterbi_train(BW_NORMAL) - pos_test=HMM(pos_train) - pos_test.set_observations(wordfeats['test']) - neg_test=HMM(neg_train) - neg_test.set_observations(wordfeats['test']) - - if indata[prefix+'name']=='TOP': - feats['train']=TOPFeatures(10, pos_train, neg_train, False, False) - feats['test']=TOPFeatures(10, pos_test, neg_test, False, False) - else: - feats['train']=FKFeatures(10, pos_train, neg_train) - feats['train'].set_opt_a(-1) #estimate prior - feats['test']=FKFeatures(10, pos_test, neg_test) - feats['test'].set_a(feats['train'].get_a()) #use prior from training data - - prefix='kernel_' - args=util.get_args(indata, prefix) - kernel=PolyKernel(feats['train'], feats['train'], *args) -# kernel=PolyKernel(*args) -# kernel.init(feats['train'], feats['train']) - km_train=max(abs( - indata[prefix+'matrix_train']-kernel.get_kernel_matrix()).flat) - kernel.init(feats['train'], feats['test']) - km_test=max(abs( - indata[prefix+'matrix_test']-kernel.get_kernel_matrix()).flat) - - return util.check_accuracy(indata[prefix+'accuracy'], - km_train=km_train, km_test=km_test) - - -######################################################################## -# public -######################################################################## - -def test (indata): - prefix='topfk_' - if prefix+'name' in indata: - return _evaluate_top_fisher(indata, prefix) - - prefix='kernel_' - names=['Combined', 'AUC', 'Custom'] - for name in names: - if indata[prefix+'name']==name: - return eval('_evaluate_'+name.lower()+'(indata, prefix)') - - names=['HistogramWordString', 'SalzbergWordString'] - for name in names: - if indata[prefix+'name']==name: - return _evaluate_pie(indata, prefix) - - return _evaluate(indata, prefix) - diff --git a/tests/integration/python_modular/preprocessor.py b/tests/integration/python_modular/preprocessor.py deleted file mode 100644 index f8fe2ee2a44..00000000000 --- a/tests/integration/python_modular/preprocessor.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -Test Preprocessor -""" - -from modshogun import * - -import util - - -######################################################################## -# kernel computation -######################################################################## - -def _evaluate (indata): - prefix='kernel_' - feats=util.get_features(indata, prefix) - kfun=eval(indata[prefix+'name']+'Kernel') - kargs=util.get_args(indata, prefix) - - prefix='preprocessor_' - pargs=util.get_args(indata, prefix) - feats=util.add_preprocessor(indata[prefix+'name'], feats, *pargs) - - prefix='kernel_' - kernel=kfun(feats['train'], feats['train'], *kargs) - km_train=max(abs( - indata[prefix+'matrix_train']-kernel.get_kernel_matrix()).flat) - kernel.init(feats['train'], feats['test']) - km_test=max(abs( - indata[prefix+'matrix_test']-kernel.get_kernel_matrix()).flat) - - return util.check_accuracy( - indata[prefix+'accuracy'], km_train=km_train, km_test=km_test) - - -######################################################################## -# public -######################################################################## - -def test (indata): - return _evaluate(indata) - diff --git a/tests/integration/python_modular/regression.py b/tests/integration/python_modular/regression.py deleted file mode 100644 index 9071920b999..00000000000 --- a/tests/integration/python_modular/regression.py +++ /dev/null @@ -1,74 +0,0 @@ -""" -Test Regression -""" - -from numpy import double -from modshogun import RegressionLabels -from modshogun import * -from modshogun import * - -import util - -def _evaluate (indata): - prefix='kernel_' - feats=util.get_features(indata, prefix) - kargs=util.get_args(indata, prefix) - fun=eval(indata[prefix+'name']+'Kernel') - kernel=fun(feats['train'], feats['train'], *kargs) - - prefix='regression_' - kernel.parallel.set_num_threads(indata[prefix+'num_threads']) - - try: - name = indata[prefix+'name'] - if (name=='KERNELRIDGEREGRESSION'): - name = 'KernelRidgeRegression' - - rfun=eval(name) - except NameError as e: - print("%s is disabled/unavailable!"%indata[prefix+'name']) - return False - - labels=RegressionLabels(double(indata[prefix+'labels'])) - if indata[prefix+'type']=='svm': - regression=rfun( - indata[prefix+'C'], indata[prefix+'epsilon'], kernel, labels) - elif indata[prefix+'type']=='kernelmachine': - regression=rfun(indata[prefix+'tau'], kernel, labels) - else: - return False - - regression.parallel.set_num_threads(indata[prefix+'num_threads']) - if prefix+'tube_epsilon' in indata: - regression.set_tube_epsilon(indata[prefix+'tube_epsilon']) - - regression.train() - - alphas=0 - bias=0 - sv=0 - if prefix+'bias' in indata: - bias=abs(regression.get_bias()-indata[prefix+'bias']) - if prefix+'alphas' in indata: - for item in regression.get_alphas().tolist(): - alphas+=item - alphas=abs(alphas-indata[prefix+'alphas']) - if prefix+'support_vectors' in indata: - for item in inregression.get_support_vectors().tolist(): - sv+=item - sv=abs(sv-indata[prefix+'support_vectors']) - - kernel.init(feats['train'], feats['test']) - classified=max(abs( - regression.apply().get_labels()-indata[prefix+'classified'])) - - return util.check_accuracy(indata[prefix+'accuracy'], alphas=alphas, - bias=bias, support_vectors=sv, classified=classified) - -######################################################################## -# public -######################################################################## - -def test (indata): - return _evaluate(indata) - diff --git a/tests/integration/python_modular/test_all.sh b/tests/integration/python_modular/test_all.sh deleted file mode 100755 index de670fd4822..00000000000 --- a/tests/integration/python_modular/test_all.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash - -if test -z "$PYTHON" -then - PYTHON=python -fi - -DATAPATH='../data' -exitcode=0 - -function test_all () { - datapath="$1" - - if echo "$datapath" | grep -q '\.\./data/tests' - then - continue - fi - - echo "*** Testing in $datapath" - sleep 1 - for file in $datapath; do - echo -n "$file" - echo -n -e "\t\t" - - if grep -q $file ../blacklist - then - echo 'SKIPPING' - else - output=`${PYTHON} test_one.py "$file"` - ret=$? - - if [ $ret -eq 0 ] ; then - echo 'OK' - else - exitcode=1 - echo 'ERROR' - echo $output - fi - fi - done - sleep 1 - echo -} - -if [ -n "$1" ]; then - test_all "$DATAPATH/$1/*.m" -else - for i in $DATAPATH/*; do - test_all "$i/*.m" - done -fi -exit $exitcode diff --git a/tests/integration/python_modular/test_one.py b/tests/integration/python_modular/test_one.py deleted file mode 100755 index 8973ea55a78..00000000000 --- a/tests/integration/python_modular/test_one.py +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env python -""" -Test one data file -""" - -from numpy import * -import sys - -import kernel -import distance -import classifier -import clustering -import distribution -import regression -import preprocessor -from modshogun import Math_init_random - -SUPPORTED=['kernel', 'distance', 'classifier', 'clustering', 'distribution', - 'regression', 'preprocessor'] - -def _get_name_fun (fnam): - module=None - - for supported in SUPPORTED: - if fnam.find(supported)>-1: - module=supported - break - - if module is None: - print('Module required for %s not supported yet!' % fnam) - return None - - return module+'.test' - -def _test_mfile (fnam): - try: - mfile=open(fnam, mode='r') - except IOError as e: - print(e) - return False - - indata={} - - name_fun=_get_name_fun(fnam) - if name_fun is None: - return False - - for line in mfile: - line=line.strip(" \t\n;") - param = line.split('=')[0].strip() - - if param=='name': - name=line.split('=')[1].strip().split("'")[1] - indata[param]=name - elif param=='kernel_symdata' or param=='kernel_data': - indata[param]=_read_matrix(line) - elif param.startswith('kernel_matrix') or \ - param.startswith('distance_matrix'): - indata[param]=_read_matrix(line) - elif param.find('data_train')>-1 or param.find('data_test')>-1: - # data_{train,test} might be prepended by 'subkernelX_' - indata[param]=_read_matrix(line) - elif param=='classifier_alphas' or param=='classifier_support_vectors': - try: - indata[param]=eval(line.split('=')[1]) - except SyntaxError: # might be MultiClass SVM and hence matrix - indata[param]=_read_matrix(line) - elif param=='clustering_centers' or param=='clustering_pairs': - indata[param]=_read_matrix(line) - else: - if (line.find("'")==-1): - indata[param]=eval(line.split('=')[1]) - else: - indata[param]=line.split('=')[1].strip().split("'")[1] - - mfile.close() - fun=eval(name_fun) - - # seed random to constant value used at data file's creation - Math_init_random(indata['init_random']) - random.seed(indata['init_random']) - - return fun(indata) - -def _read_matrix (line): - try: - str_line=(line.split('[')[1]).split(']')[0] - except IndexError: - str_line=(line.split('{')[1]).split('}')[0] - - lines=str_line.split(';') - lis2d=list() - - for x in lines: - lis=list() - for y in x.split(','): - y=y.replace("'","").strip() - if(y.isalpha()): - lis.append(y) - else: - if y.find('.')!=-1: - lis.append(float(y)) - else: - try: - lis.append(int(y)) - except ValueError: # not int, RAWDNA? - lis.append(y) - - lis2d.append(lis) - - return array(lis2d) - -for filename in sys.argv: - if (filename.endswith('.m')): - res=_test_mfile(filename) - if res: - sys.exit(0) - else: - sys.exit(1) diff --git a/tests/integration/python_modular/test_svm_kernel_multiple.py b/tests/integration/python_modular/test_svm_kernel_multiple.py deleted file mode 100644 index 8abacee9d89..00000000000 --- a/tests/integration/python_modular/test_svm_kernel_multiple.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python - -from numpy import random -from modshogun import LibSVM -from modshogun import RealFeatures, Labels -from modshogun import LinearKernel - - -num_feats=23 -num_vec=42 - -scale=2.1 -size_cache=10 - -C=0.017 -epsilon=1e-5 -tube_epsilon=1e-2 -svm=LibSVM() -svm.set_C(C, C) -svm.set_epsilon(epsilon) -svm.set_tube_epsilon(tube_epsilon) - -for i in range(3): - data_train=random.rand(num_feats, num_vec) - data_test=random.rand(num_feats, num_vec) - feats_train=RealFeatures(data_train) - feats_test=RealFeatures(data_test) - labels=Labels(random.rand(num_vec).round()*2-1) - - svm.set_kernel(LinearKernel(size_cache, scale)) - svm.set_labels(labels) - - kernel=svm.get_kernel() - print("kernel cache size: %s" % (kernel.get_cache_size())) - - kernel.init(feats_test, feats_test) - svm.train() - - kernel.init(feats_train, feats_test) - print(svm.apply().get_labels()) - - #kernel.remove_lhs_and_rhs() - - #import pdb - #pdb.set_trace() - diff --git a/tests/integration/python_modular/tester.py b/tests/integration/python_modular/tester.py deleted file mode 100755 index 162e550deb7..00000000000 --- a/tests/integration/python_modular/tester.py +++ /dev/null @@ -1,216 +0,0 @@ -#!/usr/bin/env python - -import modshogun -import pickle -import os -import filecmp -import numpy -import sys -import difflib - -from generator import setup_tests, get_fname, blacklist, get_test_mod, run_test - -def is_python2(): - return sys.version_info[0]==2 - -def typecheck(a, b): - if str(type(a)).find('shogun')>=0 and \ - str(type(a)).find('Labels')>=0 and \ - str(type(b)).find('shogun')>=0 and \ - str(type(b)).find('Labels')>=0: - return True - return type(a) == type(b) - -def compare_floats(a,b, tolerance): - import numpy as np - # we will see underflow errors and whatnot / mute them - np.seterr(all='ignore') - # inspired by http://floating-point-gui.de/errors/comparison/ - diff = np.abs(a - b) - - ok_idx = a == b - near_zero_idx = ((a * b) == 0) & ~ok_idx - other_idx = ~ (ok_idx | near_zero_idx) - - ok_idx[near_zero_idx] = np.abs(a[near_zero_idx]-b[near_zero_idx]) < tolerance**2 - ok_idx[other_idx] = np.abs(a[other_idx]-b[other_idx]) / (np.abs(a)[other_idx] + np.abs(b)[other_idx]) < tolerance - #if not np.all(ok_idx == True): - # bad=ok_idx == False - # print("a", a[bad]) - # print("b", a[bad]) - # import pdb - # pdb.set_trace() - - return np.all(ok_idx == True) - - -def compare(a, b, tolerance, sgtolerance): - if not typecheck(a,b): - return False - - if type(a) == numpy.ndarray: - if a.dtype != b.dtype: - return False - if a.dtype != numpy.floating or tolerance == 0: - return numpy.all(a==b) - return compare_floats(a,b, tolerance) - elif isinstance(a, modshogun.SGObject): - a.io.set_loglevel(modshogun.MSG_ERROR) - if pickle.dumps(a) == pickle.dumps(b): - result = True - else: - result = a.equals(b, sgtolerance, True) - - # print debug output in case of failure - if not result: - pickle.dump(a, open('/tmp/a','wb')) - pickle.dump(b, open('/tmp/b','wb')) - print("Equals failed with debug output") - old_loglevel=a.io.get_loglevel() - a.io.set_loglevel(modshogun.MSG_INFO) - a.equals(b, sgtolerance, True) - a.io.set_loglevel(old_loglevel) - - return result - elif type(a) in (tuple,list): - if len(a) != len(b): return False - for obj1, obj2 in zip(a,b): - if not compare(obj1, obj2, tolerance, sgtolerance): - return False - return True - elif type(a) == float: - return compare_floats(numpy.array([a]), numpy.array([b]), tolerance) - - return a == b - -def compare_dbg(a, b, tolerance, sgtolerance): - if not compare(a, b, tolerance, sgtolerance): - import pdb - pdb.set_trace() - -def get_fail_string(a): - failed_string = [] - if type(a) in (tuple,list): - for i in range(len(a)): - failed_string.append(get_fail_string(a[i])) - elif isinstance(a, modshogun.SGObject): - failed_string.append(pickle.dumps(a)) - else: - failed_string.append(str(a)) - return failed_string - -def get_split_string(a): - strs=[] - for l in a: - if type(l) in (tuple,list): - e=str(l[0]) - else: - e=str(l) - strs.extend(e.replace('\\n','\n').splitlines()) - return strs - -def tester(tests, cmp_method, opts): - failed=[] - sgtolerance = opts.sgtolerance - tolerance = opts.tolerance - failures = opts.failures - missing = opts.missing - - for t in tests: - try: - mod, mod_name = get_test_mod(t) - n=len(mod.parameter_list) - except TypeError: - continue - except Exception as e: - print("%-60s ERROR (%s)" % (t,e)) - failed.append(t) - continue - fname = "" - - for i in range(n): - fname = get_fname(mod_name, i) - setting_str = "%s setting %d/%d" % (t,i+1,n) - try: - a = run_test(mod, mod_name, i) - - try: - b = pickle.load(open(fname)) - except: - - try: - b = pickle.load(open(fname, 'rb')) - - except: - with open(fname, 'rb') as f: - p = pickle._Unpickler(f) - p.encoding = 'latin1' - b = p.load() - - - try: - if cmp_method(a,b,tolerance,sgtolerance): - if not failures and not missing: - print("%-60s OK" % setting_str) - else: - if not missing: - failed.append((setting_str, get_fail_string(a), get_fail_string(b))) - print("%-60s ERROR" % setting_str) - except Exception as e: - print(setting_str, e) - except IOError as e: - if not failures: - print("%-60s NO TEST (%s)" % (setting_str, e)) - except Exception as e: - failed.append(setting_str) - if not missing: - print("%-60s EXCEPTION %s" % (setting_str,e)) - return failed - -if __name__=='__main__': - import sys - from optparse import OptionParser - op=OptionParser() - op.add_option("-d", "--debug", action="store_true", default=False, - help="detailed debug output of objects that don't match") - op.add_option("-f", "--failures", action="store_true", default=False, - help="show only failures") - op.add_option("-m", "--missing", action="store_true", default=False, - help="show only missing tests") - op.add_option("-t", "--tolerance", action="store", default=1e-13, - help="tolerance used to compare numbers") - op.add_option("-s", "--sgtolerance", action="store", default=1e-5, - help="shogun tolerance used to compare numbers in shogun objects") - - op.set_usage("[ ...]") - (opts, args)=op.parse_args() - if opts.debug: - cmp_method=compare_dbg - else: - cmp_method=compare - tests = setup_tests(args) - failed = tester(tests, cmp_method, opts) - if failed: - print("The following tests failed!") - for f in failed: - print("\t" + f[0]) - - if is_python2(): - print("Detailed failures:") - for f in failed: - print("\t" + f[0]) - got=get_split_string(f[1]) - expected=get_split_string(f[2]) - #print "=== EXPECTED ==========" - #import pdb - #pdb.set_trace() - #print '\n'.join(expected) - #print "=== GOT ===============" - #print '\n'.join(got) - print("====DIFF================") - print('\n'.join(difflib.unified_diff(expected, got, fromfile='expected', tofile='got'))) - print("====EOT================") - print("\n\n\n") - - sys.exit(1) - sys.exit(0) diff --git a/tests/integration/python_modular/tests b/tests/integration/python_modular/tests deleted file mode 120000 index 5cc6418731d..00000000000 --- a/tests/integration/python_modular/tests +++ /dev/null @@ -1 +0,0 @@ -../../../examples/undocumented/python_modular \ No newline at end of file diff --git a/tests/integration/python_modular/util.py b/tests/integration/python_modular/util.py deleted file mode 100644 index 3ef8a5bd089..00000000000 --- a/tests/integration/python_modular/util.py +++ /dev/null @@ -1,194 +0,0 @@ -""" -Utilities for testing -""" - -from modshogun import * -from modshogun import * -from modshogun import * -from modshogun import * -from numpy import * - - -def check_accuracy (accuracy, **kwargs): - acc=double(accuracy) - output=[] - - for key, val in kwargs.items(): - if val is not None: - output.append('%s: %e' % (key, val)) - print(', '.join(output)+' <--- accuracy: %e' % accuracy) - - for val in kwargs.values(): - if val is not None and val>acc: - return False - - return True - - -def get_args (indata, prefix=''): - """ - Slightly esoteric function to build a tuple to be used as argument to - constructor calls. - - Python dicts are not ordered, so we have to look at the number in - the parameter's name and insert items appropriately into an - ordered list - """ - - ident=prefix+'arg' - # need to pregenerate list for using indices in loop - args=len(indata)*[None] - - for i in indata: - if i.find(ident)==-1: - continue - - try: - idx=int(i[len(ident)]) - except ValueError: - raise ValueError('Wrong indata data %s: "%s"!' % (ident, i)) - - if i.find('_distance')!=-1: # DistanceKernel - args[idx]=eval(indata[i]+'()') - else: - try: - args[idx]=eval(indata[i]) - except TypeError: # no bool - args[idx]=indata[i] - - # weed out superfluous Nones - return [arg for arg in args if arg is not None] - - -def get_features(indata, prefix=''): - fclass=prefix+'feature_class' - if indata[fclass]=='simple': - return get_feats_simple(indata, prefix) - elif indata[fclass]=='string': - return get_feats_string(indata, prefix) - elif indata[fclass]=='string_complex': - return get_feats_string_complex(indata, prefix) - elif indata[fclass]=='wd': - return get_feats_wd(indata, prefix) - else: - raise ValueError('Unknown feature class %s!'%indata[prefix+'feature_class']) - - -def get_feats_simple (indata, prefix=''): - ftype=indata[prefix+'feature_type'] - - # have to explicitely set data type for numpy if not real - as_types={ - 'Byte': ubyte, - 'Real': double, - 'Word': ushort - } - data_train=indata[prefix+'data_train'].astype(as_types[ftype]) - data_test=indata[prefix+'data_test'].astype(as_types[ftype]) - - if ftype=='Byte' or ftype=='Char': - alphabet=eval(indata[prefix+'alphabet']) - ftrain=eval(ftype+'Features(alphabet)') - ftest=eval(ftype+'Features(alphabet)') - ftrain.copy_feature_matrix(data_train) - ftest.copy_feature_matrix(data_test) - else: - ftrain=eval(ftype+'Features(data_train)') - ftest=eval(ftype+'Features(data_test)') - - if (indata[prefix+'name'].find('Sparse')!=-1 or ( - 'classifier_type' in indata and \ - indata['classifier_type']=='linear')): - sparse_train=eval('Sparse'+ftype+'Features()') - sparse_train.obtain_from_simple(ftrain) - - sparse_test=eval('Sparse'+ftype+'Features()') - sparse_test.obtain_from_simple(ftest) - - return {'train':sparse_train, 'test':sparse_test} - else: - return {'train':ftrain, 'test':ftest} - - -def get_feats_string (indata, prefix=''): - ftype=indata[prefix+'feature_type'] - alphabet=eval(indata[prefix+'alphabet']) - feats={ - 'train': eval('String'+ftype+'Features(alphabet)'), - 'test': eval('String'+ftype+'Features(alphabet)') - } - feats['train'].set_features(list(indata[prefix+'data_train'][0])) - feats['test'].set_features(list(indata[prefix+'data_test'][0])) - - return feats - - -def get_feats_string_complex (indata, prefix=''): - alphabet=eval(indata[prefix+'alphabet']) - feats={ - 'train': StringCharFeatures(alphabet), - 'test': StringCharFeatures(alphabet) - } - - if alphabet==CUBE: # data_{train,test} ints due to test.py:_read_matrix - data_train=[str(x) for x in list(indata[prefix+'data_train'][0])] - data_test=[str(x) for x in list(indata[prefix+'data_test'][0])] - else: - data_train=list(indata[prefix+'data_train'][0]) - data_test=list(indata[prefix+'data_test'][0]) - - feats['train'].set_features(data_train) - feats['test'].set_features(data_test) - - feat=eval('String'+indata[prefix+'feature_type']+ \ - "Features(alphabet)") - feat.obtain_from_char(feats['train'], indata[prefix+'order']-1, - indata[prefix+'order'], indata[prefix+'gap'], - eval(indata[prefix+'reverse'])) - feats['train']=feat - - feat=eval('String'+indata[prefix+'feature_type']+ \ - "Features(alphabet)") - feat.obtain_from_char(feats['test'], indata[prefix+'order']-1, - indata[prefix+'order'], indata[prefix+'gap'], - eval(indata[prefix+'reverse'])) - feats['test']=feat - - if indata[prefix+'feature_type']=='Word' or \ - indata[prefix+'feature_type']=='Ulong': - name='Sort'+indata[prefix+'feature_type']+'String' - return add_preprocessor(name, feats) - else: - return feats - - -def get_feats_wd (indata, prefix=''): - order=indata[prefix+'order'] - feats={} - - charfeat=StringCharFeatures(DNA) - charfeat.set_features(list(indata[prefix+'data_train'][0])) - bytefeat=StringByteFeatures(RAWDNA) - bytefeat.obtain_from_char(charfeat, 0, 1, 0, False) - feats['train']=WDFeatures(bytefeat, order, order) - - charfeat=StringCharFeatures(DNA) - charfeat.set_features(list(indata[prefix+'data_test'][0])) - bytefeat=StringByteFeatures(RAWDNA) - bytefeat.obtain_from_char(charfeat, 0, 1, 0, False) - feats['test']=WDFeatures(bytefeat, order, order) - - return feats - - -def add_preprocessor(name, feats, *args): - fun=eval(name) - preproc=fun(*args) - preproc.init(feats['train']) - feats['train'].add_preprocessor(preproc) - feats['train'].apply_preprocessor() - feats['test'].add_preprocessor(preproc) - feats['test'].apply_preprocessor() - - return feats - diff --git a/tests/integration/python_static/classifier.py b/tests/integration/python_static/classifier.py deleted file mode 100644 index 26c6c6343bd..00000000000 --- a/tests/integration/python_static/classifier.py +++ /dev/null @@ -1,140 +0,0 @@ -""" -Test Classifier -""" - -from numpy import double -from sg import sg - -import util - - -def _set_classifier (indata, prefix): - if indata.has_key(prefix+'labels'): - sg('set_labels', 'TRAIN', double(indata[prefix+'labels'])) - - cname=util.fix_classifier_name_inconsistency(indata[prefix+'name']) - sg('new_classifier', cname) - - if indata.has_key(prefix+'bias'): - sg('svm_use_bias', True) - else: - sg('svm_use_bias', False) - - if indata.has_key(prefix+'epsilon'): - sg('svm_epsilon', indata[prefix+'epsilon']) - if indata.has_key(prefix+'max_train_time'): - sg('svm_max_train_time', indata[prefix+'max_train_time']) - if indata.has_key(prefix+'linadd_enabled'): - sg('use_linadd', True) - if indata.has_key(prefix+'batch_enabled'): - sg('use_batch_computation', True) - if indata.has_key(prefix+'num_threads'): - sg('threads', indata[prefix+'num_threads']) - - -def _train (indata, prefix): - if indata[prefix+'type']=='knn': - sg('train_classifier', indata[prefix+'k']) - elif indata[prefix+'type']=='lda': - sg('train_classifier', indata[prefix+'gamma']) - elif indata[prefix+'type']=='perceptron': - # does not converge - try: - sg('train_classifier') - except RuntimeError: - import sys - sys.exit(0) - else: - if indata.has_key(prefix+'C'): - sg('c', double(indata[prefix+'C'])) - sg('train_classifier') - - -def _get_alpha_and_sv(indata, prefix): - if not indata.has_key(prefix+'alpha_sum') and \ - not indata.has_key(prefix+'sv_sum'): - return None, None - - a=0 - sv=0 - if indata.has_key(prefix+'label_type') and \ - indata[prefix+'label_type']=='series': - for i in xrange(sg('get_num_svms')): - [dump, weights]=sg('get_svm', i) - weights=weights.T - for item in weights[0].tolist(): - a+=item - for item in weights[1].tolist(): - sv+=item - a=abs(a-indata[prefix+'alpha_sum']) - sv=abs(sv-indata[prefix+'sv_sum']) - else: - [dump, weights]=sg('get_svm') - weights=weights.T - for item in weights[0].tolist(): - a+=item - a=abs(a-indata[prefix+'alpha_sum']) - for item in weights[1].tolist(): - sv+=item - sv=abs(sv-indata[prefix+'sv_sum']) - - return a, sv - - -def _evaluate (indata, prefix): - alphas=0 - bias=0 - sv=0 - - if indata[prefix+'type']=='lda': - pass - else: - if indata.has_key(prefix+'label_type') and \ - indata[prefix+'label_type'] != 'series' and \ - indata.has_key(prefix+'bias'): - [b, weights]=sg('get_svm') - weights=weights.T - bias=abs(b-indata[prefix+'bias']) - - alphas, sv=_get_alpha_and_sv(indata, prefix) - - classified=max(abs(sg('classify')-indata[prefix+'classified'])) - - return util.check_accuracy(indata[prefix+'accuracy'], - alphas=alphas, bias=bias, support_vectors=sv, classified=classified) - -######################################################################## -# public -######################################################################## - -def test (indata): - prefix='classifier_' - - if indata[prefix+'type']=='kernel': - feature_prefix='kernel_' - elif indata[prefix+'type']=='knn': - feature_prefix='distance_' - else: - feature_prefix='classifier_' - - try: - util.set_features(indata, feature_prefix) - except NotImplementedError, e: - print e - return True - - if indata[prefix+'type']=='kernel': - util.set_and_train_kernel(indata) - elif indata[prefix+'type']=='knn': - util.set_and_train_distance(indata) - - try: - _set_classifier(indata, prefix) - except RuntimeError: - print "%s is disabled/unavailable!" % indata[prefix+'name'] - return True - - _train(indata, prefix) - - return _evaluate(indata, prefix) - diff --git a/tests/integration/python_static/clustering.py b/tests/integration/python_static/clustering.py deleted file mode 100644 index 3e5f3ba07c2..00000000000 --- a/tests/integration/python_static/clustering.py +++ /dev/null @@ -1,72 +0,0 @@ -""" -Test Clustering -""" - -from sg import sg -import util - - -def _set_clustering (indata): - cname=util.fix_clustering_name_inconsistency(indata['clustering_name']) - sg('new_clustering', cname) - - -def _train (indata, prefix): - if indata.has_key(prefix+'max_iter'): - max_iter=indata[prefix+'max_iter'] - else: - max_iter=1000 - - if indata.has_key(prefix+'k'): - first_arg=indata[prefix+'k'] - elif indata.has_key(prefix+'merges'): - first_arg=indata[prefix+'merges'] - else: - raise StandardError, 'Incomplete clustering data.' - - sg('train_clustering', first_arg, max_iter) - - -def _evaluate (indata, prefix): - if indata.has_key(prefix+'radi'): - [radi, centers]=sg('get_clustering') - radi=max(abs(radi.T[0]-indata[prefix+'radi'])) - centers=max(abs(centers-indata[prefix+'centers']).flat) - - return util.check_accuracy(indata[prefix+'accuracy'], - radi=radi, centers=centers) - - elif indata.has_key(prefix+'merge_distance'): - [merge_distances, pairs]=sg('get_clustering') - merge_distances=max(abs(merge_distances.T[0]- \ - indata[prefix+'merge_distance'])) - pairs=max(abs(pairs-indata[prefix+'pairs']).flat) - - return util.check_accuracy(indata[prefix+'accuracy'], - merge_distances=merge_distances, pairs=pairs) - - else: - raise StandardError, 'Incomplete clustering data.' - -######################################################################## -# public -######################################################################## - -def test (indata): - try: - util.set_features(indata, 'distance_') - except NotImplementedError, e: - print e - return True - - util.set_and_train_distance(indata) - _set_clustering(indata) - - prefix='clustering_' - try: - _train(indata, prefix) - return _evaluate(indata, prefix) - except StandardError, e: - print e - return False - diff --git a/tests/integration/python_static/distance.py b/tests/integration/python_static/distance.py deleted file mode 100644 index 21ae67fbfca..00000000000 --- a/tests/integration/python_static/distance.py +++ /dev/null @@ -1,35 +0,0 @@ -""" -Test Distance -""" - -from sg import sg -import util - - -def _evaluate (indata, prefix): - dmatrix=sg('get_distance_matrix', 'TRAIN') - dm_train=max(abs(indata['distance_matrix_train']-dmatrix).flat) - - dmatrix=sg('get_distance_matrix', 'TEST') - dm_test=max(abs(indata['distance_matrix_test']-dmatrix).flat) - - return util.check_accuracy( - indata[prefix+'accuracy'], dm_train=dm_train, dm_test=dm_test) - - -######################################################################## -# public -######################################################################## - -def test (indata): - prefix='distance_' - try: - util.set_features(indata, prefix) - except NotImplementedError, e: - print e - return True - - util.set_and_train_distance(indata) - - return _evaluate(indata, prefix) - diff --git a/tests/integration/python_static/distribution.py b/tests/integration/python_static/distribution.py deleted file mode 100644 index 38e113ca835..00000000000 --- a/tests/integration/python_static/distribution.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -Test Distribution -""" -from numpy import inf, nan -from sg import sg -import util - -def _evaluate (indata): - prefix='distribution_' - # what is sg('likelihood')? - likelihood=abs(sg('hmm_likelihood')-indata[prefix+'likelihood']) - return util.check_accuracy(indata[prefix+'accuracy'], - likelihood=likelihood) - - # best path? which? no_b_trans? trans? trans_deriv? - if indata['name']=='HMM': - best_path=0 - best_path_state=0 - for i in xrange(indata[prefix+'num_examples']): - best_path+=distribution.best_path(i) - for j in xrange(indata[prefix+'N']): - best_path_state+=distribution.get_best_path_state(i, j) - - best_path=abs(best_path-indata[prefix+'best_path']) - best_path_state=abs(best_path_state-\ - indata[prefix+'best_path_state']) - - return util.check_accuracy(indata[prefix+'accuracy'], - derivatives=derivatives, likelihood=likelihood, - best_path=best_path, best_path_state=best_path_state) - else: - return util.check_accuracy(indata[prefix+'accuracy'], - derivatives=derivatives, likelihood=likelihood) - - -def _set_distribution (indata): - prefix='distribution_' - if indata[prefix+'name']=='HMM': - sg('new_hmm', indata[prefix+'N'], indata[prefix+'M']) - sg('bw') - else: - raise NotImplementedError, 'Can\'t yet train other distributions than HMM in static interface.' - - -######################################################################## -# public -######################################################################## - -def test (indata): - try: - util.set_features(indata, 'distribution_') - _set_distribution(indata) - except NotImplementedError, e: - print e - return True - - return _evaluate(indata) - diff --git a/tests/integration/python_static/kernel.py b/tests/integration/python_static/kernel.py deleted file mode 100644 index 5dbb5b0f60a..00000000000 --- a/tests/integration/python_static/kernel.py +++ /dev/null @@ -1,203 +0,0 @@ -""" -Test Kernel -""" - -from sg import sg -from numpy import double -import util - - -def _evaluate_kernel (indata, prefix, kernel_is_set=False): - if not kernel_is_set: - util.set_and_train_kernel(indata) - - kmatrix=sg('get_kernel_matrix', 'TRAIN') - km_train=max(abs(indata[prefix+'matrix_train']-kmatrix).flat) - - kmatrix=sg('get_kernel_matrix', 'TEST') - km_test=max(abs(indata[prefix+'matrix_test']-kmatrix).flat) - - return util.check_accuracy( - indata[prefix+'accuracy'], km_train=km_train, km_test=km_test) - - -def _get_subkernels (indata, prefix): - subkernels={} - prefix=prefix+'subkernel' - len_prefix=len(prefix) - - # loop through indata (unordered dict) to gather subkernel data - for key in indata: - if key.find(prefix)==-1: - continue - - # get subkernel's number - try: - num=key[len_prefix] - except ValueError: - raise ValueError, 'Cannot find number for subkernel: "%s"!' % data - - # get item's name - name=key[len_prefix+2:] - - # append new item - if not subkernels.has_key(num): - subkernels[num]={} - subkernels[num][name]=indata[key] - - # got all necessary information in new structure, now create a kernel - # object for each subkernel - for num, data in subkernels.iteritems(): - subkernels[num]['args']=util.get_args(data, '') - subkernels[num]['name']=util.fix_kernel_name_inconsistency(data['name']) - - return subkernels - - -def _evaluate_combined (indata, prefix): - sg('set_kernel', 'COMBINED', util.SIZE_CACHE) - - subkernels=_get_subkernels(indata, prefix) - for subk in subkernels.itervalues(): - sg('add_kernel', 1., subk['name'], subk['feature_type'].upper(), - *subk['args']) - - if subk.has_key('alphabet'): - sg('add_features', 'TRAIN', - list(subk['data_train'][0]), subk['alphabet']) - sg('add_features', 'TEST', - list(subk['data_test'][0]), subk['alphabet']) - else: - sg('add_features', 'TRAIN', subk['data_train'][0]) - sg('add_features', 'TEST', subk['data_test'][0]) - - return _evaluate_kernel(indata, prefix, True) - - -def _evaluate_auc (indata, prefix): - raise NotImplementedError, 'AUC kernel not yet supported in static interfaces.' - - subk=_get_subkernels(indata, prefix)[0] - subk['kernel'].init(feats_subk['train'], feats_subk['test']) - - feats={ - 'train': WordFeatures(indata['data_train'].astype(ushort)), - 'test': WordFeatures(indata['data_test'].astype(ushort))} - kernel=AUCKernel(10, subk['kernel']) - - return _evaluate_kernel(indata, prefix, True) - - -def _evaluate_custom (indata, prefix): - raise NotImplementedError, 'Custom kernel not yet implemented in static interfaces.' - - symdata=indata[prefix+'symdata'] - lowertriangle=array([symdata[(x,y)] for x in xrange(symdata.shape[1]) - for y in xrange(symdata.shape[0]) if y<=x]) - - sg('set_kernel', 'CUSTOM') - sg('set_triangle_kernel_matrix_from_triangle', lowertriangle) - triangletriangle=max(abs( - indata[prefix+'matrix_triangletriangle']-sg('get_kernel_matrix')).flat) - - sg('set_triangle_kernel_matrix_from_full', indata[prefix+'symdata']) - fulltriangle=max(abs( - indata[prefix+'matrix_fulltriangle']-sg('get_kernel_matrix')).flat) - - sg('set_full_kernel_matrix_from_full', indata[prefix+'data']) - fullfull=max(abs( - indata[prefix+'matrix_fullfull']-sg('get_kernel_matrix')).flat) - - return util.check_accuracy( - indata[prefix+'accuracy'], - triangletriangle=triangletriangle, - fulltriangle=fulltriangle, - fullfull=fullfull - ) - - -def _evaluate_pie (indata, prefix): - pseudo_pos=1e-10 - pseudo_neg=1e-10 - - sg('new_plugin_estimator', pseudo_pos, pseudo_neg) - - sg('set_labels', 'TRAIN', double(indata['classifier_labels'])) - sg('train_estimator') - - return _evaluate_kernel(indata, prefix) - - -def _evaluate_top_fisher (indata, prefix): - raise NotImplementedError, 'TOP/Fisher not yet supported in static interfaces.' - - sg('new_hmm', indata[prefix+'N'], indata[prefix+'M']) - pos=HMM(wordfeats['train'], indata[prefix+'N'], indata[prefix+'M'], - indata[prefix+'pseudo']) - pos.train() - pos.baum_welch_viterbi_train(BW_NORMAL) - neg=HMM(wordfeats['train'], indata[prefix+'N'], indata[prefix+'M'], - indata[prefix+'pseudo']) - neg.train() - neg.baum_welch_viterbi_train(BW_NORMAL) - pos_clone=HMM(pos) - neg_clone=HMM(neg) - pos_clone.set_observations(wordfeats['test']) - neg_clone.set_observations(wordfeats['test']) - - if indata[prefix+'type']=='TOP': - feats['train']=TOPFeatures(10, pos, neg, False, False) - feats['test']=TOPFeatures(10, pos_clone, neg_clone, False, False) - else: - feats['train']=FKFeatures(10, pos, neg) - feats['train'].set_opt_a(-1) #estimate prior - feats['test']=FKFeatures(10, pos_clone, neg_clone) - feats['test'].set_a(feats['train'].get_a()) #use prior from training data - - prefix='kernel_' - args=util.get_args(indata, prefix) - kernel=PolyKernel(feats['train'], feats['train'], *args) - km_train=max(abs( - indata[prefix+'matrix_train']-kernel.get_kernel_matrix()).flat) - kernel.init(feats['train'], feats['test']) - km_test=max(abs( - indata[prefix+'matrix_test']-kernel.get_kernel_matrix()).flat) - - return util.check_accuracy(indata[prefix+'accuracy'], - km_train=km_train, km_test=km_test) - - -######################################################################## -# public -######################################################################## - -def test (indata): - prefix='topfk_' - if indata.has_key(prefix+'name'): - try: - util.set_features(indata, prefix) - return _evaluate_top_fisher(indata, prefix) - except NotImplementedError, e: - print e - return True - - prefix='kernel_' - try: - util.set_features(indata, prefix) - names=['Combined', 'AUC', 'Custom'] - for name in names: - if indata[prefix+'name']==name: - return eval('_evaluate_'+name.lower()+'(indata, prefix)') - - names=['HistogramWordString', 'SalzbergWordString'] - for name in names: - if indata[prefix+'name']==name: - return _evaluate_pie(indata, prefix) - - # pretty normal kernel - return _evaluate_kernel(indata, prefix) - except NotImplementedError, e: - print e - return True - - diff --git a/tests/integration/python_static/preprocessor.py b/tests/integration/python_static/preprocessor.py deleted file mode 100644 index 87cc8d4127f..00000000000 --- a/tests/integration/python_static/preprocessor.py +++ /dev/null @@ -1,46 +0,0 @@ -""" -Test Preprocessor -""" - -from sg import sg -import util - - -def _evaluate (indata, prefix): - util.set_and_train_kernel(indata) - - kmatrix=sg('get_kernel_matrix', 'TRAIN') - km_train=max(abs(indata['kernel_matrix_train']-kmatrix).flat) - - kmatrix=sg('get_kernel_matrix', 'TEST') - km_test=max(abs(indata['kernel_matrix_test']-kmatrix).flat) - - return util.check_accuracy( - indata[prefix+'accuracy'], km_train=km_train, km_test=km_test) - - -def _set_preprocessor(indata, prefix): - pname=util.fix_preproc_name_inconsistency(indata[prefix+'name']) - args=util.get_args(indata, prefix) - - sg('add_preproc', pname, *args) - sg('attach_preproc', 'TRAIN') - sg('attach_preproc', 'TEST') - - -######################################################################## -# public -######################################################################## - -def test (indata): - prefix='kernel_' - try: - util.set_features(indata, prefix) - except NotImplementedError, e: - print e - return True - - _set_preprocessor(indata, 'preprocessor_') - - return _evaluate(indata, prefix) - diff --git a/tests/integration/python_static/regression.py b/tests/integration/python_static/regression.py deleted file mode 100644 index 5c1e2210a33..00000000000 --- a/tests/integration/python_static/regression.py +++ /dev/null @@ -1,81 +0,0 @@ -""" -Test Regression -""" - -from sg import sg -from numpy import double -import util - - -def _set_regression (indata): - prefix='regression_' - sg('threads', indata[prefix+'num_threads']) - sg('set_labels', 'TRAIN', double(indata[prefix+'labels'])) - - rname=util.fix_regression_name_inconsistency(indata[prefix+'name']) - print rname - sg('new_regression', rname) - - -def _train (indata): - if indata['regression_type']=='svm': - sg('c', double(indata['regression_C'])) - sg('svm_epsilon', indata['regression_epsilon']) - sg('svr_tube_epsilon', indata['regression_tube_epsilon']) - elif indata['regression_type']=='kernelmachine': - sg('krr_tau', indata['regression_tau']) - else: - raise StandardError, 'Incomplete regression data.' - - sg('train_regression') - - -def _evaluate (indata): - alphas=0 - bias=0 - sv=0 - - if indata.has_key('regression_bias'): - [bias, weights]=sg('get_svm') - weights=weights.T - bias=abs(bias-indata['regression_bias']) - for item in weights[0].tolist(): - alphas+=item - alphas=abs(alphas-indata['regression_alpha_sum']) - for item in weights[1].tolist(): - sv+=item - sv=abs(sv-indata['regression_sv_sum']) - - classified=max(abs(sg('classify')-indata['regression_classified'])) - - return util.check_accuracy(indata['regression_accuracy'], - alphas=alphas, bias=bias, support_vectors=sv, classified=classified) - - -######################################################################## -# public -######################################################################## - -def test (indata): - try: - util.set_features(indata, 'kernel_') - except NotImplementedError, e: - print e - return True - - util.set_and_train_kernel(indata) - - try: - _set_regression(indata) - except RuntimeError, e: - print "%s is disabled/unavailable!" % indata['name'] - return True - - try: - _train(indata) - except StandardError, e: - print e - return False - - return _evaluate(indata) - diff --git a/tests/integration/python_static/test_all.sh b/tests/integration/python_static/test_all.sh deleted file mode 100755 index 6b2319f1050..00000000000 --- a/tests/integration/python_static/test_all.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash - -DATAPATH='../data' -exitcode=0 - -[ -z "$PYTHON" ] && PYTHON=python - -function test_all () { - datapath="$1" - - if echo "$datapath" | grep -q '\.\./data/tests' - then - continue - fi - - echo "*** Testing in $datapath" - sleep 1 - for file in $datapath; do - echo -n "$file" - echo -n -e "\t\t" - - if grep -q $file ../blacklist - then - echo 'SKIPPING' - else - output=`${PYTHON} test_one.py "$file"` - ret=$? - - if [ $ret -eq 0 ] ; then - echo 'OK' - else - exitcode=1 - echo 'ERROR' - echo $output - fi - fi - done - sleep 1 - echo -} - -if [ -n "$1" ]; then - test_all "$DATAPATH/$1/*.m" -else - for i in $DATAPATH/*; do - test_all "$i/*.m" - done -fi - -exit $exitcode diff --git a/tests/integration/python_static/test_one.py b/tests/integration/python_static/test_one.py deleted file mode 100755 index 3f04c52e427..00000000000 --- a/tests/integration/python_static/test_one.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env python -""" -Test one data file -""" - -from numpy import * -import sys - -import kernel -import distance -import classifier -import clustering -import distribution -import regression -import preprocessor -from sg import sg - -SUPPORTED=['kernel', 'distance', 'classifier', 'clustering', 'distribution', - 'regression', 'preprocessor'] - -def _get_name_fun (fnam): - module=None - - for supported in SUPPORTED: - if fnam.find(supported)>-1: - module=supported - break - - if module is None: - print 'Module required for %s not supported yet!' % fnam - return None - - return module+'.test' - -def _test_mfile (fnam): - try: - mfile=open(fnam, mode='r') - except IOError, e: - print e - return False - - indata={} - - name_fun=_get_name_fun(fnam) - if name_fun is None: - return False - - for line in mfile: - line=line.strip(" \t\n;") - param = line.split('=')[0].strip() - - if param=='name': - name=line.split('=')[1].strip().split("'")[1] - indata[param]=name - elif param=='kernel_symdata' or param=='kernel_data': - indata[param]=_read_matrix(line) - elif param.startswith('kernel_matrix') or \ - param.startswith('distance_matrix'): - indata[param]=_read_matrix(line) - elif param.find('data_train')>-1 or param.find('data_test')>-1: - # data_{train,test} might also be prepended by *subkernel* - indata[param]=_read_matrix(line) - elif param=='clustering_centers' or param=='clustering_pairs': - indata[param]=_read_matrix(line) - else: - if (line.find("'")==-1): - indata[param]=eval(line.split('=')[1]) - else: - indata[param]=line.split('=')[1].strip().split("'")[1] - - mfile.close() - fun=eval(name_fun) - - # seed random to constant value used at data file's creation - sg('init_random', indata['init_random']) - random.seed(indata['init_random']) - - return fun(indata) - -def _read_matrix (line): - try: - str_line=(line.split('[')[1]).split(']')[0] - except IndexError: - str_line=(line.split('{')[1]).split('}')[0] - - lines=str_line.split(';') - lis2d=list() - - for x in lines: - lis=list() - for y in x.split(','): - y=y.replace("'","").strip() - if(y.isalpha()): - lis.append(y) - else: - if y.find('.')!=-1: - lis.append(float(y)) - else: - try: - lis.append(int(y)) - except ValueError: # not int, RAWDNA? - lis.append(y) - lis2d.append(lis) - - return array(lis2d) - -for filename in sys.argv: - if (filename.endswith('.m')): - res=_test_mfile(filename) - if res: - sys.exit(0) - else: - sys.exit(1) diff --git a/tests/integration/python_static/test_svm_kernel_multiple.py b/tests/integration/python_static/test_svm_kernel_multiple.py deleted file mode 100644 index 040608a29a7..00000000000 --- a/tests/integration/python_static/test_svm_kernel_multiple.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python - -from numpy import random -from shogun.Classifier import LibSVM -from shogun.Features import RealFeatures, Labels -from shogun.Kernel import LinearKernel - - -num_feats=23 -num_vec=42 - -scale=2.1 -size_cache=10 - -C=0.017 -epsilon=1e-5 -tube_epsilon=1e-2 -svm=LibSVM() -svm.set_C(C, C) -svm.set_epsilon(epsilon) -svm.set_tube_epsilon(tube_epsilon) - -for i in xrange(3): - data_train=random.rand(num_feats, num_vec) - data_test=random.rand(num_feats, num_vec) - feats_train=RealFeatures(data_train) - feats_test=RealFeatures(data_test) - labels=Labels(random.rand(num_vec).round()*2-1) - - svm.set_kernel(LinearKernel(size_cache, scale)) - svm.set_labels(labels) - - kernel=svm.get_kernel() - print "kernel cache size: %s" % (kernel.get_cache_size()) - - kernel.init(feats_test, feats_test) - svm.train() - - kernel.init(feats_train, feats_test) - print svm.classify().get_labels() - - #kernel.remove_lhs_and_rhs() - - #import pdb - #pdb.set_trace() - diff --git a/tests/integration/python_static/util.py b/tests/integration/python_static/util.py deleted file mode 100644 index 30ede13aaed..00000000000 --- a/tests/integration/python_static/util.py +++ /dev/null @@ -1,219 +0,0 @@ -""" -Utilities for testing -""" - -from numpy import double, ushort, ubyte, matrix -from sg import sg - -SIZE_CACHE=10 - - -def check_accuracy (accuracy, **kwargs): - acc=double(accuracy) - output=[] - - for key, val in kwargs.iteritems(): - if val is not None: - output.append('%s: %e' % (key, val)) - print ', '.join(output)+' <--- accuracy: %e' % accuracy - - for val in kwargs.itervalues(): - if val>acc: - return False - - return True - - -def get_args (indata, prefix=''): - # python dicts are not ordered, so we have to look at the number in - # the parameter's name and insert items appropriately into an - # ordered list - - # need to pregenerate list for using indices in loop - ident=prefix+'arg' - args=len(indata)*[None] - - for i in indata: - if i.find(ident)==-1: - continue - - try: - idx=int(i[len(ident)]) - except ValueError: - raise ValueError, 'Wrong indata data %s: "%s"!' % (ident, i) - - if i.find('_distance')!=-1: # DistanceKernel - args[idx]=indata[i] - else: - try: - args[idx]=eval(indata[i]) - except TypeError: # no bool - args[idx]=indata[i] - - # weed out superfluous Nones - return filter(lambda arg: arg is not None, args) - - -def set_features (indata, prefix): - uppername=indata[prefix+'name'].upper() - if uppername=='COMBINED' or uppername=='CUSTOM': - return - - if uppername.startswith('SPARSE'): - raise NotImplementedError, 'Sparse features not supported yet.' - elif indata.has_key(prefix+'type') and \ - indata[prefix+'type']=='linear': - raise NotImplementedError, 'Linear classifiers with sparse features not supported yet.' - - indata_train=indata[prefix+'data_train'] - indata_test=indata[prefix+'data_test'] - - if indata.has_key(prefix+'alphabet'): - alphabet=indata[prefix+'alphabet'] - if alphabet=='RAWBYTE': - raise NotImplementedError, 'Alphabet RAWBYTE not supported yet.' - - if alphabet=='RAWDNA': - data_train=list(indata_train[0]) - sg('set_features', 'TRAIN', data_train, 'DNA') - sg('convert', 'TRAIN', 'STRING', 'CHAR', 'STRING', 'BYTE') - data_test=list(indata_test[0]) - sg('set_features', 'TEST', data_test, 'DNA') - sg('convert', 'TEST', 'STRING', 'CHAR', 'STRING', 'BYTE') - elif alphabet=='CUBE': - data_train=[str(x) for x in list(indata_train[0])] - sg('set_features', 'TRAIN', data_train, alphabet) - data_test=[str(x) for x in list(indata_test[0])] - sg('set_features', 'TEST', data_test, alphabet) - else: - data_train=list(indata_train[0]) - sg('set_features', 'TRAIN', data_train, alphabet) - data_test=list(indata_test[0]) - sg('set_features', 'TEST', data_test, alphabet) - - elif indata.has_key('data'): # CustomKernel - sg('set_features', 'TRAIN', - indata[prefix+'data'].astype(ushort)) - sg('set_features', 'TEST', - indata[prefix+'data'].astype(ushort)) - - else: - as_types={ - 'Word': ushort, - 'Real': double, - 'Byte': ubyte - } - as_type=as_types[indata[prefix+'feature_type']] - sg('set_features', 'TRAIN', indata_train.astype(as_type)) - sg('set_features', 'TEST', indata_test.astype(as_type)) - - convert_features_and_add_preproc(indata, prefix) - - -def set_and_train_distance (indata, do_train=True): - prefix='distance_' - dargs=get_args(indata, prefix) - dname=fix_distance_name_inconsistency(indata[prefix+'name']) - sg('set_distance', dname, indata[prefix+'feature_type'].upper(), *dargs) - -def set_and_train_kernel (indata, do_train=True): - prefix='kernel_' - kargs=get_args(indata, prefix) - kname=fix_kernel_name_inconsistency(indata[prefix+'name']) - - if indata.has_key(prefix+'arg0_size'): - size=kargs[0] - kargs=kargs[1:] - else: - size=SIZE_CACHE - - if kname=='POLY' and indata.has_key(prefix+'normalizer'): - kargs.append(True) - - if kname=='DISTANCE': - dname=fix_distance_name_inconsistency(kargs.pop()) - # FIXME: REAL is cheating and will break in the future - sg('set_distance', dname, 'REAL') - sg('set_kernel', kname, size, *kargs) - else: - sg('set_kernel', kname, indata[prefix+'feature_type'].upper(), size, - *kargs) - -def convert_features_and_add_preproc (indata, prefix): - # having order implies having gap and reverse - if not indata.has_key(prefix+'order'): - return - - if indata[prefix+'feature_type']=='Ulong': - type='ULONG' - elif indata[prefix+'feature_type']=='Word': - type='WORD' - else: - return - - order=indata[prefix+'order'] - sg('add_preproc', 'SORT'+type+'STRING') - sg('convert', 'TRAIN', 'STRING', 'CHAR', 'STRING', type, - order, order-1, indata[prefix+'gap'], indata[prefix+'reverse']) - sg('attach_preproc', 'TRAIN') - - sg('convert', 'TEST', 'STRING', 'CHAR', 'STRING', type, - order, order-1, indata[prefix+'gap'], indata[prefix+'reverse']) - sg('attach_preproc', 'TEST') - - -# fix inconsistency in modular/static interfaces -def fix_kernel_name_inconsistency (kname): - kname=kname.upper() - if kname=='LOCALITYIMPROVEDSTRING': - return 'LIK' - elif kname=='SIMPLELOCALITYIMPROVEDSTRING': - return 'SLIK' - elif kname=='WORDMATCH': - return 'MATCH' - elif kname=='WEIGHTEDDEGREEPOSITIONSTRING': - return 'WEIGHTEDDEGREEPOS' - elif kname=='COMMULONGSTRING': - return 'COMMSTRING' - elif kname=='COMMWORDSTRING': - return 'COMMSTRING' - elif kname=='WEIGHTEDCOMMWORDSTRING': - return 'WEIGHTEDCOMMSTRING' - elif kname.endswith('WORDSTRING'): - return kname.split('WORDSTRING')[0] - elif kname.endswith('STRING'): - return kname.split('STRING')[0] - elif kname.endswith('WORD'): - return kname.split('WORD')[0] - else: - return kname - - -def fix_distance_name_inconsistency (dname): - dname=dname.upper() - if dname.endswith('WORDDISTANCE'): - return dname.split('WORDDISTANCE')[0] - elif dname.endswith('DISTANCE'): - return dname.split('DISTANCE')[0] - elif dname.endswith('METRIC'): - return dname.split('METRIC')[0] - else: - return dname - - -def fix_classifier_name_inconsistency (cname): - cname=cname.upper() - if cname.startswith('LIBSVM') and len(cname)>len('LIBSVM'): - return 'LIBSVM_'+cname.split('LIBSVM')[1] - else: - return cname - - -def fix_clustering_name_inconsistency (cname): - return cname.upper() - -def fix_preproc_name_inconsistency (pname): - return pname.upper() - -def fix_regression_name_inconsistency (rname): - return rname.upper() diff --git a/tests/integration/r_modular/classifier.R b/tests/integration/r_modular/classifier.R deleted file mode 100644 index 131a4dacd76..00000000000 --- a/tests/integration/r_modular/classifier.R +++ /dev/null @@ -1,156 +0,0 @@ -classifier <- function(filename) { - source('util/get_features.R') - source('util/get_kernel.R') - source('util/get_distance.R') - source('util/check_accuracy.R') - source('util/tobool.R') - - if (regexpr('kernel', classifier_type)>0) { - feats <- get_features('kernel_') - if (typeof(feats)=='logical') { - return(TRUE) - } - kernel <- get_kernel(feats) - if (typeof(kernel)=='logical') { - return(TRUE) - } - } else if (regexpr('knn', classifier_type)>0) { - feats <- get_features('distance_') - if (typeof(feats)=='logical') { - return(TRUE) - } - distance <- get_distance(feats) - if (typeof(kernel)=='logical') { - return(TRUE) - } - } else { - feats <- get_features('classifier_') - if (typeof(feats)=='logical') { - return(TRUE) - } - } - - if (exists('classifier_labels')) { - lab <- Labels(as.double(classifier_labels)) - } - - if (regexpr('GMNPSVM', classifier_name)>0) { - classifier <- GMNPSVM(classifier_C, kernel, lab) - } else if (regexpr('GPBTSVM', classifier_name)>0) { - classifier <- GPBTSVM(classifier_C, kernel, lab) - } else if (regexpr('KNN', classifier_name)>0) { - classifier <- KNN(as.integer(classifier_k), distance, lab) - } else if (regexpr('LDA', classifier_name)>0) { - classifier <- LDA(classifier_gamma, feats[[1]], lab) - } else if (regexpr('LibLinear', classifier_name)>0) { - classifier <- LibLinear(classifier_C, feats[[1]], lab) - classifier$set_solver_type(classifier, L2R_LR) - } else if (regexpr('LibSVMMultiClass', classifier_name)>0) { - classifier <- LibSVMMultiClass(classifier_C, kernel, lab) - } else if (regexpr('LibSVMOneClass', classifier_name)>0) { - classifier <- LibSVMOneClass(classifier_C, kernel) - } else if (regexpr('LibSVM', classifier_name)>0) { - classifier <- LibSVM(classifier_C, kernel, lab) - } else if (regexpr('MPDSVM', classifier_name)>0) { - classifier <- MPDSVM(classifier_C, kernel, lab) - } else if (regexpr('Perceptron', classifier_name)>0) { - classifier <- Perceptron(feats[[1]], lab) - classifier$set_learn_rate(classifier, classifier_learn_rate) - classifier$set_max_iter(classifier, as.integer(classifier_max_iter)) - } else if (regexpr('SVMLight', classifier_name)>0) { - try(classifier <- SVMLight(classifier_C, kernel, lab)) - } else if (regexpr('SVMLin', classifier_name)>0) { - classifier <- SVMLin(classifier_C, feats[[1]], lab) - } else if (regexpr('SVMOcas', classifier_name)>0) { - classifier <- SVMOcas(classifier_C, feats[[1]], lab) - } else if (regexpr('SVMSGD', classifier_name)>0) { - classifier <- SVMSGD(classifier_C, feats[[1]], lab) - } else if (regexpr('SubGradientSVM', classifier_name)>0) { - classifier <- SubGradientSVM(classifier_C, feats[[1]], lab) - } else { - print(paste('Unsupported classifier', classifier_name)) - return(FALSE) - } - - classifier$parallel$set_num_threads( - classifier$parallel, as.integer(classifier_num_threads)) - - if (regexpr('linear', classifier_type)>0 && exists('classifier_bias')) { - classifier$set_bias_enabled(classifier, TRUE) - } - if (exists('classifier_epsilon')) { - if (regexpr('SVMSGD', classifier_name)<=0) { - classifier$set_epsilon(classifier, classifier_epsilon) - } - } - if (exists('classifier_max_train_time')) { - classifier$set_max_train_time(classifier, classifier_max_train_time) - } - if (exists('classifier_linadd_enabled')) { - classifier$set_linadd_enabled( - classifier, tobool(classifier_linadd_enabled)) - } - if (exists('classifier_batch_enabled')) { - classifier$set_batch_computation_enabled( - classifier, tobool(classifier_batch_enabled)) - } - - classifier$train() - - bias <- 0 - if (exists('classifier_bias')) { - # get_bias yields true instead of a float??? - #print(classifier$get_bias()) - #print(classifier$get_bias(classifier)) - #bias <- classifier$get_bias(classifier) - #bias <- abs(bias-classifier_bias) - } - - alphas <- 0 - sv <- 0 - if (exists('classifier_alpha_sum') && exists('classifier_sv_sum')) { - if (exists('classifier_label_type') && regexpr('series', classifier_label_type)>0) { - for (i in 0:(classifier$get_num_svms()-1)) { - subsvm <- classifier$get_svm(classifier, i) - tmp <- subsvm$get_alphas() - for (j in 1:length(tmp)) { - alphas <- alphas + tmp[j] - } - tmp <- subsvm$get_support_vectors() - for (j in 1:length(tmp)) { - sv <- sv + tmp[j] - } - } - alphas <- abs(alphas-classifier_alpha_sum) - sv <- abs(sv-classifier_sv_sum) - } else { - tmp <- classifier$get_alphas() - for (i in 1:length(tmp)) { - alphas <- alphas + tmp[i] - } - alphas <- abs(alphas-classifier_alpha_sum) - tmp <- classifier$get_support_vectors() - for (i in 1:length(tmp)) { - sv <- sv + tmp[i] - } - sv <- abs(sv-classifier_sv_sum) - } - } - - if (regexpr('knn', classifier_type)>0) { - distance$init(distance, feats[[1]], feats[[2]]) - } else if (regexpr('kernel', classifier_type)>0) { - kernel$init(kernel, feats[[1]], feats[[2]]) - } else if (regexpr('lda', classifier_type)>0 || - regexpr('linear', classifier_type)>0 || - regexpr('perceptron', classifier_type)>0) { - classifier$set_features(classifier, feats[[2]]) - } - - classified <- classifier$classify(classifier) - classified <- max(abs( - classified$get_labels(classified)-classifier_classified)) - - data <- list(alphas, bias, sv, classified) - return(check_accuracy(classifier_accuracy, 'classifier', data)) -} diff --git a/tests/integration/r_modular/clustering.R b/tests/integration/r_modular/clustering.R deleted file mode 100644 index 22bc76b0017..00000000000 --- a/tests/integration/r_modular/clustering.R +++ /dev/null @@ -1,44 +0,0 @@ -clustering <- function(filename) { - source('util/get_features.R') - source('util/get_distance.R') - source('util/check_accuracy.R') - - feats <- get_features('distance_') - if (typeof(feats)=='logical') { - return(TRUE) - } - - distance <- get_distance(feats) - if (typeof(distance)=='logical') { - return(TRUE) - } - - if (regexpr('KMeans', clustering_name)>0) { - clustering <- KMeans(as.integer(clustering_k), distance); - clustering$train() - - distance$init(distance, feats[[1]], feats[[2]]) - centers <- clustering$get_cluster_centers() - centers <- max(max(abs(centers-clustering_centers))) - radi <- clustering$get_radiuses() - radi <- max(abs(radi-clustering_radi)) - data <- list(centers, radi) - - return(check_accuracy(clustering_accuracy, 'kmeans', data)) - } else if (regexpr('Hierarchical', clustering_name)>0) { - clustering <- Hierarchical(as.integer(clustering_merges), distance); - clustering$train() - - distance$init(distance, feats[[1]], feats[[2]]) - merge_distances <- clustering$get_merge_distances() - merge_distances <- max(abs(merge_distances-clustering_merge_distance)) - pairs <- clustering$get_cluster_pairs() - pairs <- max(max(abs(pairs-clustering_pairs))) - data <- list(merge_distances, pairs) - - return(check_accuracy(clustering_accuracy, 'hierarchical', data)) - } else { - print('Incomplete clustering data!') - return(FALSE) - } -} diff --git a/tests/integration/r_modular/distance.R b/tests/integration/r_modular/distance.R deleted file mode 100644 index 49a7636c672..00000000000 --- a/tests/integration/r_modular/distance.R +++ /dev/null @@ -1,25 +0,0 @@ -distance <- function(filename) { - source('util/get_features.R') - source('util/get_distance.R') - source('util/check_accuracy.R') - - feats <- get_features('distance_') - if (typeof(feats)=='logical') { - return(TRUE) - } - - distance <- get_distance(feats) - if (typeof(distance)=='logical') { - return(TRUE) - } - - dmatrix <- distance$get_distance_matrix() - dm_train <- max(max(abs(distance_matrix_train-dmatrix))) - - distance$init(distance, feats[[1]], feats[[2]]) - dmatrix <- distance$get_distance_matrix() - dm_test <- max(max(abs(distance_matrix_test-dmatrix))) - - data <- list(dm_train, dm_test) - return(check_accuracy(distance_accuracy, 'distance', data)) -} diff --git a/tests/integration/r_modular/distribution.R b/tests/integration/r_modular/distribution.R deleted file mode 100644 index ea6cb833804..00000000000 --- a/tests/integration/r_modular/distribution.R +++ /dev/null @@ -1,46 +0,0 @@ -distribution <- function(filename) { - source('util/get_features.R') - source('util/check_accuracy.R') - - feats <- get_features('distribution_') - if (typeof(feats)=='logical') { - return(TRUE) - } - - if (regexpr('^HMM', distribution_name)>0) { - distribution <- HMM(feats[[1]], - as.integer(distribution_N), as.integer(distribution_M), - distribution_pseudo) - distribution$train(distribution) - distribution$baum_welch_viterbi_train(distribution, 'BW_NORMAL') - } - else if (regexpr('^Histogram', distribution_name)>0) { - distribution <- Histogram(feats[[1]]) - distribution$train(distribution) - } else if (regexpr('^LinearHMM', distribution_name)>0) { - distribution <- LinearHMM(feats[[1]]) - distribution$train(distribution) - } else { - print(paste('Unknown distribution', distribution_name)) - return(TRUE) - } - - likelihood <- max(abs( - distribution$get_log_likelihood_sample()-distribution_likelihood)) - num_examples <- feats[[1]]$get_num_vectors() - num_param <- distribution$get_num_model_parameters() - derivatives <- 0 - for (i in 0:(num_param-1)) { - for (j in 0:(num_examples-1)) { - val <- distribution$get_log_derivative( - distribution, as.integer(i), as.integer(j)) - if (abs(val)!=Inf) { - derivatives <- derivatives + val - } - } - } - derivatives <- max(abs(derivatives-distribution_derivatives)) - - data=list(likelihood, derivatives) - return(check_accuracy(distribution_accuracy, 'distribution', data)) -} diff --git a/tests/integration/r_modular/kernel.R b/tests/integration/r_modular/kernel.R deleted file mode 100644 index f884f075da3..00000000000 --- a/tests/integration/r_modular/kernel.R +++ /dev/null @@ -1,24 +0,0 @@ -kernel <- function() { - source('util/get_features.R') - source('util/get_kernel.R') - source('util/check_accuracy.R') - - feats <- get_features('kernel_') - if (typeof(feats)=='logical') { - return(TRUE) - } - - kernel <- get_kernel(feats) - if (typeof(kernel)=='logical') { - return(TRUE) - } - - kmatrix <- kernel$get_kernel_matrix() - km_train <- max(max(abs(kernel_matrix_train-kmatrix))) - - kernel$init(kernel, feats[[1]], feats[[2]]) - kmatrix <- kernel$get_kernel_matrix() - km_test <- max(max(abs(kernel_matrix_test-kmatrix))) - data <- list(km_train, km_test) - return(check_accuracy(kernel_accuracy, 'kernel', data)) -} diff --git a/tests/integration/r_modular/preproc.R b/tests/integration/r_modular/preproc.R deleted file mode 100644 index 1d5c91d4429..00000000000 --- a/tests/integration/r_modular/preproc.R +++ /dev/null @@ -1,46 +0,0 @@ -preproc <- function(filename) { - source('util/get_features.R') - source('util/get_kernel.R') - source('util/check_accuracy.R') - source('util/tobool.R') - - feats <- get_features('kernel_') - if (typeof(feats)=='logical') { - return(TRUE) - } - - if (regexpr('LogPlusOne', preproc_name)>0) { - preproc <- LogPlusOne() - } else if (regexpr('NormOne', preproc_name)>0) { - preproc <- NormOne() - } else if (regexpr('PruneVarSubMean', preproc_name)>0) { - preproc <- PruneVarSubMean(tobool(preproc_arg0_divide)) - } else if (regexpr('SortUlongString', preproc_name)>0) { - preproc <- SortUlongString() - } else if (regexpr('SortWordString', preproc_name)>0) { - preproc <- SortWordString() - } else { - print(paste('Unsupported preproc', preproc_name)) - return(FALSE) - } - - preproc$init(preproc, feats[[1]]) - feats[[1]]$add_preproc(feats[[1]], preproc) - feats[[1]]$apply_preproc(feats[[1]]) - feats[[2]]$add_preproc(feats[[2]], preproc) - feats[[2]]$apply_preproc(feats[[2]]) - - kernel <- get_kernel(feats) - if (typeof(kernel)=='logical') { - return(TRUE) - } - - kmatrix <- kernel$get_kernel_matrix() - km_train <- max(max(abs(kernel_matrix_train-kmatrix))) - - kernel$init(kernel, feats[[1]], feats[[2]]) - kmatrix <- kernel$get_kernel_matrix() - km_test <- max(max(abs(kernel_matrix_test-kmatrix))) - data <- list(km_train, km_test) - return(check_accuracy(kernel_accuracy, 'kernel', data)) -} diff --git a/tests/integration/r_modular/regression.R b/tests/integration/r_modular/regression.R deleted file mode 100644 index 8696824defe..00000000000 --- a/tests/integration/r_modular/regression.R +++ /dev/null @@ -1,66 +0,0 @@ -regression <- function(filename) { - source('util/get_features.R') - source('util/get_kernel.R') - source('util/check_accuracy.R') - source('util/tobool.R') - - feats <- get_features('kernel_') - if (typeof(feats)=='logical') { - return(TRUE) - } - kernel <- get_kernel(feats) - if (typeof(kernel)=='logical') { - return(TRUE) - } - - regression_num_threads <- as.integer(regression_num_threads) - kernel$parallel$set_num_threads(kernel$parallel, regression_num_threads) - lab <- Labels(as.double(regression_labels)) - - if (regexpr('KRR', regression_name)>0) { - regression <- KRR(regression_tau, kernel, lab) - } else if (regexpr('LibSVR', regression_name)>0) { - regression <- LibSVR(regression_C, regression_epsilon, kernel, lab) - regression$set_tube_epsilon(regression, regression_tube_epsilon) - } else if (regexpr('SVRLight', regression_name)) { - try(regression <- SVRLight( - regression_C, regression_epsilon, kernel, lab)) - regression$set_tube_epsilon(regression, regression_tube_epsilon) - } else { - print(paste('Unsupported regression', regression_name)) - return(FALSE) - } - - regression$parallel$set_num_threads( - regression$parallel, regression_num_threads) - regression$train(regression) - - bias <- 0 - if (exists('regression_bias')) { - bias <- abs(regression$get_bias()-regression_bias) - } - - alphas <- 0 - sv <- 0 - if (exists('regression_alpha_sum')) { - tmp <- regression$get_alphas() - for (i in 1:length(tmp) ){ - alphas <= alphas + tmp[i] - } - alphas <- abs(alphas-regression_alpha_sum) - - tmp <- regression$get_support_vectors() - for (i in 1:length(tmp)) { - sv <- sv + tmp[i] - } - sv <- abs(sv-regression_sv_sum) - } - - kernel$init(kernel, feats[[1]], feats[[2]]) - classified <- regression$classify(regression) - classified <- max(abs( - classified$get_labels(classified)-regression_classified)) - - data <- list(alphas, bias, sv, classified) - return(check_accuracy(regression_accuracy, 'classifier', data)) -} diff --git a/tests/integration/r_modular/test_all.sh b/tests/integration/r_modular/test_all.sh deleted file mode 100755 index 05ea9f5d4f9..00000000000 --- a/tests/integration/r_modular/test_all.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash - -DATAPATH='../data' -exitcode=0 - -function test_all () { - datapath=${1} - - if echo "$datapath" | grep -q '\.\./data/tests' - then - continue - fi - - echo "*** Testing in ${datapath}" - sleep 1 - for file in ${datapath}; do - if grep -q $file ../blacklist - then - echo 'SKIPPING' - else - dump=`echo ${file} | grep WDSVMOcas` - if [ $? -eq 0 ] - then - echo WDSVMOcas totally unsupported: ERROR - else - echo -n "${file}" - echo -n -e "\t\t" - - output=`R --no-save --no-restore --no-readline --slave ${file} < test_one.R` - if [ $? -ne 0 ]; then - exitcode=1 - echo ERROR - echo ${output} - else - echo OK - fi - fi - fi - done - sleep 1 - echo -} - -if [ -n "${1}" ]; then - test_all "${DATAPATH}/${1}/*.m" -else - for i in ${DATAPATH}/*; do - test_all "${i}/*.m" - done -fi -exit $exitcode diff --git a/tests/integration/r_modular/test_one.R b/tests/integration/r_modular/test_one.R deleted file mode 100644 index 8effa4440db..00000000000 --- a/tests/integration/r_modular/test_one.R +++ /dev/null @@ -1,22 +0,0 @@ -library(shogun) - -source("util/read_mfile.R") -filename <- commandArgs()[6] -indata <- read_mfile(filename) -for (i in 1:length(indata)) { - eval(parse(text=indata[i])) -} - -# there is some randomness involved, alas this is -# not working correctly in R -dump <- Math_init_random(init_random) -set.seed(init_random) - -path=strsplit(filename, '/', fixed=TRUE) -module=path[[1]][3] - -source(paste(module, '.R', sep='')) -res <- eval(parse(text=paste(module, '()', sep=''))) - -# res==TRUE is 1; have to negate to get shell-compatible truth value -quit(status=!res) diff --git a/tests/integration/r_modular/test_one.sh b/tests/integration/r_modular/test_one.sh deleted file mode 100755 index 4b7c0ffd4da..00000000000 --- a/tests/integration/r_modular/test_one.sh +++ /dev/null @@ -1,7 +0,0 @@ -dump=`echo ${1} | grep WDSVMOcas` -if [ $? -eq 0 ]; then - echo "WDSVMOcas totally unsupported" - exit 1 -fi - -R --no-save --no-restore --no-readline --slave ${1} < test_one.R diff --git a/tests/integration/r_modular/util/check_accuracy.R b/tests/integration/r_modular/util/check_accuracy.R deleted file mode 100644 index 7cce7a7b341..00000000000 --- a/tests/integration/r_modular/util/check_accuracy.R +++ /dev/null @@ -1,29 +0,0 @@ -check_accuracy <- function(accuracy, type, data) { - if (regexpr(type, 'classifier')>0) { - print(paste('alphas:', data[1], ', bias:', data[2], ', sv:', data[3], ', classified:', data[4], '<--- accuracy ', accuracy)) - } else if (regexpr(type, 'distance')>0) { - print(paste('dtrain:', data[1], ', dtest:', data[2], '<--- accuracy ', accuracy)) - } else if (regexpr(type, 'distribution')>0) { - print(paste('likelihood:', data[1], ', derivatives:', data[2], '<--- accuracy ', accuracy)) - } else if (regexpr(type, 'hierarchical')>0) { - print(paste('merge_distances:', data[1], ', pairs:', data[2], '<--- accuracy', accuracy)) - } else if (regexpr(type, 'kernel')>0) { - print(paste('ktrain:', data[1], ', ktest:', data[2], '<--- accuracy', accuracy)) - } else if (regexpr(type, 'custom')>0) { - print(paste('triangletriangle:', data[1], ', fulltriangle:', data[2], - ', fullfull:', data[3], '<--- accuracy', accuracy)) - } else if (regexpr(type, 'kmeans')>0) { - print(paste('centers:', data[1], ', radi:', data[2], '<--- accuracy', accuracy)) - } else { - print(paste('Unknown accuracy type', type, '!')) - return(FALSE) - } - - for (i in 1:length(data)) { - if (data[i]>accuracy) { - return(FALSE) - } - } - - return(TRUE) -} diff --git a/tests/integration/r_modular/util/convert_features_and_add_preproc.R b/tests/integration/r_modular/util/convert_features_and_add_preproc.R deleted file mode 100644 index a365260df0a..00000000000 --- a/tests/integration/r_modular/util/convert_features_and_add_preproc.R +++ /dev/null @@ -1,35 +0,0 @@ -convert_features_and_add_preproc <- function(char_train, char_test, prefix) { - source('util/tobool.R') - - if (!exists(paste(prefix, 'order', sep=''), mode='numeric')) { - return(FALSE) - } - - ftype <- eval(parse(text=paste(prefix, 'feature_type', sep=''))) - if (regexpr('Ulong', ftype)>0) { - train <- StringUlongFeatures(char_train$get_alphabet()) - test <- StringUlongFeatures(char_test$get_alphabet()) - preproc <- SortUlongString() - } else if (regexpr('Word', ftype)>0) { - train <- StringWordFeatures(char_train$get_alphabet()) - test <- StringWordFeatures(char_test$get_alphabet()) - preproc <- SortWordString() - } else { - return(FALSE) - } - - order <- as.integer(eval(parse(text=paste(prefix, 'order', sep='')))) - start <- as.integer(order-1) - gap <- as.integer(eval(parse(text=paste(prefix, 'gap', sep='')))) - reverse <- tobool(eval(parse(text=paste(prefix, 'reverse', sep='')))) - train$obtain_from_char(train, char_train, start, order, gap, reverse) - test$obtain_from_char(test, char_test, start, order, gap, reverse) - - preproc$init(preproc, train) - train$add_preproc(train, preproc) - train$apply_preproc(train) - test$add_preproc(test, preproc) - test$apply_preproc(test) - - return(list(train, test)) -} diff --git a/tests/integration/r_modular/util/get_distance.R b/tests/integration/r_modular/util/get_distance.R deleted file mode 100644 index 75c1720e420..00000000000 --- a/tests/integration/r_modular/util/get_distance.R +++ /dev/null @@ -1,21 +0,0 @@ -get_distance <- function(feats) { - source('util/tobool.R') - - if (exists('kernel_arg2_distance')) { - dname <- kernel_arg2_distance - } else { - dname <- distance_name - } - - if (regexpr('HammingWordDistance', dname)>0) { - distance <- HammingWordDistance(feats[[1]], feats[[1]], - tobool(distance_arg0_use_sign)) - } else if (regexpr('MinkowskiMetric', dname)>0) { - distance <- MinkowskiMetric(feats[[1]], feats[[1]], distance_arg0_k) - } else { - dfun <- eval(parse(text=dname)) - distance <- dfun(feats[[1]], feats[[1]]) - } - - return(distance) -} diff --git a/tests/integration/r_modular/util/get_features.R b/tests/integration/r_modular/util/get_features.R deleted file mode 100644 index 88b1fe9f186..00000000000 --- a/tests/integration/r_modular/util/get_features.R +++ /dev/null @@ -1,137 +0,0 @@ -get_features <- function(prefix) { - source('util/convert_features_and_add_preproc.R') - source('util/get_posneg.R') - - size_cache <- 10 - name <- eval(parse(text=paste(prefix, 'name', sep=''))) - - if (regexpr('Combined', name)>0) { - # this will break when subkernels in data file are changed - # it blindly assumes that all features are StringChar - train <- CombinedFeatures() - test <- CombinedFeatures() - - feat <- StringCharFeatures('DNA') - feat$set_string_features(feat, kernel_subkernel0_data_train) - train$append_feature_obj(train, feat) - feat <- StringCharFeatures('DNA') - feat$set_string_features(feat, kernel_subkernel0_data_test) - test$append_feature_obj(test, feat) - - feat <- StringCharFeatures('DNA') - feat$set_string_features(feat, kernel_subkernel1_data_train) - train$append_feature_obj(train, feat) - feat <- StringCharFeatures('DNA') - feat$set_string_features(feat, kernel_subkernel1_data_test) - test$append_feature_obj(test, feat) - - feat <- StringCharFeatures('DNA') - feat$set_string_features(feat, kernel_subkernel2_data_train) - train$append_feature_obj(train, feat) - feat <- StringCharFeatures('DNA') - feat$set_string_features(feat, kernel_subkernel2_data_test) - test$append_feature_obj(test, feat) - } else if (regexpr('Custom', name)>0) { - train <- RealFeatures(kernel_data) - test <- RealFeatures(kernel_data) - } else if (exists('topfk_name')) { - posneg <- get_posneg('topfk_') - if (typeof(posneg)=='logical') { - return(posneg) - } - - if (regexpr('FK', topfk_name)>0) { - train <- FKFeatures(size_cache, posneg[1][1], posneg[1][2]) - train$set_opt_a(-1) # estimate prior - test <- FKFeatures(size_cache, posneg[2][1], posneg[2][2]) - test$set_a(train$get_a()) - } else { - train <- TOPFeatures(size_cache, posneg[1][1], posneg[1][2]) - test <- TOPFeatures(size_cache, posneg[2][1], posneg[2][2]) - } - } else { - data_train <- eval(parse(text=paste(prefix, 'data_train', sep=''))) - data_test <- eval(parse(text=paste(prefix, 'data_test', sep=''))) - feature_type <- eval(parse(text=paste(prefix, 'feature_type', sep=''))) - feature_class <- eval(parse( - text=paste(prefix, 'feature_class', sep=''))) - - if (regexpr('Sparse', name)>0) { - is_sparse <- TRUE - } else if (exists('classifier_type') && - regexpr('linear', classifier_type)>0) { - is_sparse <- TRUE - } else { - is_sparse <- FALSE - } - - if (regexpr('simple', feature_class)>0) { - if (regexpr('Real', feature_type)>0) { - train <- RealFeatures(data_train) - test <- RealFeatures(data_test) - - if (is_sparse) { - sparse <- SparseRealFeatures() - sparse$obtain_from_simple(sparse, train) - train <- sparse - sparse <- SparseRealFeatures() - sparse$obtain_from_simple(sparse, test) - test <- sparse - } - } else if (regexpr('Word', feature_type)) { - print('Datatype Word not yet supported in R') - return(FALSE) - train <- WordFeatures(data_train) - test <- WordFeatures(data_test) - } else if (regexpr('Byte', feature_type)) { - print('Datatype Byte not yet supported in R') - return(FALSE) - train <- ByteFeatures(RAWBYTE) - train$copy_feature_matrix(data_train) - test <- ByteFeatures(RAWBYTE) - test$copy_feature_matrix(data_test) - } else { - print(paste('Simple feature type not supported yet:', - feature_type)) - } - } else if (regexpr('string', feature_class)>0) { - alphabet <- eval(parse(text=paste(prefix, 'alphabet', sep=''))) - if (grep('DNA', alphabet)==0 && grep('CUBE', alphabet)==0) { - print(paste('Alphabet not supported yet:', alphabet)) - return(FALSE) - } - - train <- StringCharFeatures(alphabet) - train$set_string_features(train, data_train) - test <- StringCharFeatures(alphabet) - test$set_string_features(test, data_test) - - if (regexpr('string_complex', feature_class)>0) { - return(convert_features_and_add_preproc(train, test, prefix)) - } - } else if (regexpr('wd', feature_class)>0) { - order <- as.integer(eval(parse(text=paste( - prefix, 'order', sep='')))) - - charfeat <- StringCharFeatures('DNA') - charfeat$set_string_features(charfeat, data_train) - bytefeat <- StringByteFeatures('RAWDNA') - bytefeat$obtain_from_char(bytefeat, charfeat, - as.integer(0), as.integer(1), as.integer(0), FALSE) - train <- WDFeatures(bytefeat, order, order) - - charfeat <- StringCharFeatures('DNA') - charfeat$set_string_features(charfeat, data_test) - bytefeat <- StringByteFeatures('RAWDNA') - bytefeat$obtain_from_char(bytefeat, charfeat, - as.integer(0), as.integer(1), as.integer(0), FALSE) - test <- WDFeatures(bytefeat, order, order) - } - else { - print(paste('Unknown feature class', feature_class)) - return(FALSE) - } - } - - return(list(train, test)) -} diff --git a/tests/integration/r_modular/util/get_kernel.R b/tests/integration/r_modular/util/get_kernel.R deleted file mode 100644 index d0a25e70365..00000000000 --- a/tests/integration/r_modular/util/get_kernel.R +++ /dev/null @@ -1,275 +0,0 @@ -get_kernel <- function(feats) { - source('util/tobool.R') - source('util/get_distance.R') - source('util/get_pie.R') - - if (exists('kernel_arg0_size')) { - size_cache <- as.integer(kernel_arg0_size) - } - else if (exists('kernel_arg1_size')) { - size_cache <- as.integer(kernel_arg1_size) - } else { - size_cache <- as.integer(10) - } - - if (exists('kernel_name')) { - kname <- kernel_name - } else { - kname <- name - } - - if (regexpr('^AUC', kname)>0) { - if (regexpr('^Gaussian', kernel_subkernel0_name)==0) { - print(paste( - 'Dunno how to handle AUC subkernel', kernel_subkernel0_name)) - } - subfeats_train <- RealFeatures(kernel_subkernel0_data_train) - subfeats_test <- RealFeatures(kernel_subkernel0_data_test) - subkernel <- GaussianKernel( - subfeats_train, subfeats_test, kernel_subkernel0_arg1_width) - return( - AUCKernel(feats[[1]], feats[[1]], kernel_arg1_width, size_cache)) - } - - else if (regexpr('^Chi2', kname)>0) { - return(Chi2Kernel( - feats[[1]], feats[[1]], kernel_arg1_width, size_cache)) - } - - else if (regexpr('^Combined', kname)>0) { - # this will break when structure of test file is changed! - kernel <- CombinedKernel() - subkernel <- FixedDegreeStringKernel( - as.integer(kernel_subkernel0_arg0_size), - as.integer(kernel_subkernel0_arg1_degree)) - kernel$append_kernel(kernel, subkernel) - subkernel <- PolyMatchStringKernel( - as.integer(kernel_subkernel1_arg0_size), - as.integer(kernel_subkernel1_arg1_degree), - tobool(kernel_subkernel1_arg2_inhomogene)) - kernel$append_kernel(kernel, subkernel) - subkernel <- LocalAlignmentStringKernel( - as.integer(kernel_subkernel2_arg0_size)) - kernel$append_kernel(kernel, subkernel) - kernel$init(kernel, feats[[1]], feats[[1]]) - return(kernel) - } - - else if (regexpr('^CommUlongString', kname)>0) { - if (exists('kernel_arg1_use_sign')) { - return(CommUlongStringKernel(feats[[1]], feats[[1]], - tobool(kernel_arg1_use_sign))) - } else { - return(CommUlongStringKernel(feats[[1]], feats[[1]])) - } - } - - else if (regexpr('^CommWordString', kname)>0) { - if (exists('kernel_arg1_use_sign')) { - return(CommWordStringKernel(feats[[1]], feats[[1]], - tobool(kernel_arg1_use_sign))) - } else { - return(CommWordStringKernel(feats[[1]], feats[[1]])) - } - } - - else if (regexpr('^Const', kname)>0) { - return(ConstKernel(feats[[1]], feats[[1]], kernel_arg0_c)) - } - - else if (regexpr('^Custom', kname)>0) { - return(TRUE) # silently unsupported - } - - else if (regexpr('^Diag', kname)>0) { - return(DiagKernel(feats[[1]], feats[[1]], kernel_arg1_diag)) - } - - else if (regexpr('^Distance', kname)>0) { - distance <- get_distance(feats) - if (typeof(distance)=='logical') { - return(distance) - } - return(DistanceKernel( - feats[[1]], feats[[1]], kernel_arg1_width, distance)) - } - - else if (regexpr('^FixedDegreeString', kname)>0) { - return(FixedDegreeStringKernel( - feats[[1]], feats[[1]], as.integer(kernel_arg1_degree))) - } - - else if (regexpr('^Gaussian$', kname)>0) { - if (exists('kernel_arg0_width')) { - width <- kernel_arg0_width - } else { - width <- kernel_arg1_width - } - return(GaussianKernel(feats[[1]], feats[[1]], width)) - } - - else if (regexpr('^GaussianShift', kname)>0) { - return(GaussianShiftKernel( - feats[[1]], feats[[1]], - kernel_arg1_width, - as.integer(kernel_arg2_max_shift), - as.integer(kernel_arg3_shift_step))) - } - - else if (regexpr('^HistogramWordString', kname)>0) { - pie <- get_pie(feats[[1]]) - if (typeof(pie)=='logical') { - return(TRUE) - } - return(HistogramWordStringKernel(feats[[1]], feats[[1]], pie)) - } - - else if (regexpr('^Linear$', kname)>0) { - kernel <- LinearKernel() - if (exists('kernel_arg0_scale')) { - normalizer <- AvgDiagKernelNormalizer(kernel_arg0_scale) - kernel$set_normalizer(kernel, normalizer) - } else { - kernel$set_normalizer(kernel, AvgDiagKernelNormalizer()) - } - kernel$init(kernel, feats[[1]], feats[[1]]) - return(kernel) - } - - else if (regexpr('^LinearByte', kname)>0) { - kernel <- LinearByteKernel() - kernel$set_normalizer(kernel, AvgDiagKernelNormalizer()) - kernel$init(kernel, feats[[1]], feats[[1]]) - return(kernel) - } - - else if (regexpr('^LinearString', kname)>0) { - kernel <- LinearStringKernel() - kernel$set_normalizer(kernel, AvgDiagKernelNormalizer()) - kernel$init(kernel, feats[[1]], feats[[1]]) - return(kernel) - } - - else if (regexpr('^LinearWord', kname)>0) { - kernel <- LinearWordKernel() - kernel$set_normalizer(kernel, AvgDiagKernelNormalizer()) - kernel$init(kernel, feats[[1]], feats[[1]]) - return(kernel) - } - - else if (regexpr('^LocalAlignmentString', kname)>0) { - return(LocalAlignmentStringKernel(feats[[1]], feats[[1]])) - } - - else if (regexpr('^MatchWordString', kname)>0) { - return(MatchWordStringKernel( - feats[[1]], feats[[1]], as.integer(kernel_arg1_degree))) - } - - else if (regexpr('^Oligo', kname)>0) { - kernel <- OligoKernel(size_cache, - as.integer(kernel_arg1_k), kernel_arg2_width) - kernel$init(kernel, feats[[1]], feats[[1]]) - return(kernel) - } - - - else if (regexpr('^PolyMatchString', kname)>0) { - return(PolyMatchStringKernel( - feats[[1]], feats[[1]], - as.integer(kernel_arg1_degree), tobool(kernel_arg2_inhomogene))) - } - - else if (regexpr('^PolyMatchWordString', kname)>0) { - return(PolyMatchWordStringKernel(feats[[1]], feats[[1]], - as.integer(kernel_arg1_degree), tobool(kernel_arg2_inhomogene))) - } - - else if (regexpr('^Poly', kname)>0) { - if (exists('kernel_normalizer')) { - kernel <- PolyKernel(size_cache, - as.integer(kernel_arg1_degree), - tobool(kernel_arg2_inhomogene)) - normalizer <- eval(parse( - text=paste(kernel_normalizer, '()', sep=''))) - kernel$set_normalizer(kernel, normalizer) - kernel$init(kernel, feats[[1]], feats[[1]]) - } else { - kernel <- PolyKernel( - feats[[1]], feats[[1]], - as.integer(kernel_arg1_degree), tobool(kernel_arg2_inhomogene)) - } - return(kernel) - } - - else if (regexpr('^SalzbergWordString', kname)>0) { - pie <- get_pie(feats[[1]]) - if (typeof(pie)=='logical') { - return(TRUE) - } - return(SalzbergWordStringKernel(feats[[1]], feats[[1]], pie)) - } - - else if (regexpr('^Sigmoid', kname)>0) { - return(SigmoidKernel( - feats[[1]], feats[[1]], - size_cache, kernel_arg1_gamma, kernel_arg2_coef0)) - } - - else if (regexpr('^SimpleLocalityImprovedString', kname)>0) { - return(SimpleLocalityImprovedStringKernel( - feats[[1]], feats[[1]], - as.integer(kernel_arg1_length), - as.integer(kernel_arg2_inner_degree), - as.integer(kernel_arg3_outer_degree))) - } - - else if (regexpr('^SparseGaussian', kname)>0) { - return(SparseGaussianKernel(feats[[1]], feats[[1]], kernel_arg1_width)) - } - - else if (regexpr('^SparseLinear', kname)>0) { - kernel <- SparseLinearKernel() - if (exists('kernel_arg0_scale')) { - normalizer <- AvgDiagKernelNormalizer(kernel_arg0_scale) - kernel$set_normalizer(kernel, normalizer) - } else { - kernel$set_normalizer(kernel, AvgDiagKernelNormalizer()) - } - kernel$init(kernel, feats[[1]], feats[[1]]) - return(kernel) - } - - else if (regexpr('^SparsePoly', kname)>0) { - kernel <- SparsePolyKernel( - size_cache, as.integer(kernel_arg1_degree), - tobool(kernel_arg2_inhomogene)) - kernel$init(kernel, feats[[1]], feats[[1]]) - return(kernel) - } - - else if (regexpr('^WeightedCommWordString', kname)>0) { - return(WeightedCommWordStringKernel( - feats[[1]], feats[[1]], tobool(kernel_arg1_use_sign))) - } - - else if (regexpr('^WeightedDegreePositionString', kname)>0) { - if (exists('kernel_arg0_degree')) { - degree <- as.integer(kernel_arg0_degree) - } else { - degree <- as.integer(kernel_arg1_degree) - } - return(WeightedDegreePositionStringKernel( - feats[[1]], feats[[1]], degree)) - } - - else if (regexpr('^WeightedDegreeString', kname)>0) { - return(WeightedDegreeStringKernel( - feats[[1]], feats[[1]], as.integer(kernel_arg0_degree))) - } - - else { - print(paste('Unsupported kernel:', kname)) - return(TRUE) - } -} diff --git a/tests/integration/r_modular/util/get_pie.R b/tests/integration/r_modular/util/get_pie.R deleted file mode 100644 index c134d209b49..00000000000 --- a/tests/integration/r_modular/util/get_pie.R +++ /dev/null @@ -1,9 +0,0 @@ -get_pie <- function(feats) { - pie <- PluginEstimate() - lab <- as.double(classifier_labels) - labels <- Labels(lab) - pie$set_labels(pie, labels) - pie$set_features(pie, feats) - pie$train() - return(pie) -} diff --git a/tests/integration/r_modular/util/get_posneg.R b/tests/integration/r_modular/util/get_posneg.R deleted file mode 100644 index 470e91a6f16..00000000000 --- a/tests/integration/r_modular/util/get_posneg.R +++ /dev/null @@ -1,45 +0,0 @@ -get_posneg <- function(prefix) { - print('Datatype Word not yet supported in R') - return(FALSE) # data type word not yet supported in R - N <- as.integer(eval(parse(text=paste(prefix, 'N', sep='')))) - M <- as.integer(eval(parse(text=paste(prefix, 'M', sep='')))) - pseudo <- eval(parse(text=paste(prefix, 'pseudo', sep=''))) - order <- as.integer(eval(parse(text=paste(prefix, 'order', sep='')))) - start <- as.integer(order-1) - gap <- as.integer(eval(parse(text=paste(prefix, 'gap', sep='')))) - reverse <- tobool(paste(prefix, 'reverse', sep='')) - data_train <- eval(parse(text=paste(prefix, 'data_train', sep=''))) - data_test <- eval(parse(text=paste(prefix, 'data_test', sep=''))) - - charfeat <- StringCharFeatures(CUBE) - charfeat$set_string_features(charfeat, data_train) - wordfeats_train <- StringWordFeatures(charfeat$get_alphabet()) - wordfeats_train$obtain_from_char( - wordfeats_train, charfeat, start, order, gap, reverse) - preproc <- SortWordString() - preproc$init(preproc, wordfeats_train) - wordfeats_train$add_preproc(wordfeats_train, preproc) - wordfeats_train$apply_preproc(wordfeats_train) - - charfeat <- StringCharFeatures(CUBE) - charfeat$set_string_features(charfeat, data_test) - wordfeats_test <- StringWordFeatures(charfeat$get_alphabet()) - wordfeats_test$obtain_from_char( - wordfeats_test, charfeat, start, order, gap, reverse) - wordfeats_test$add_preproc(wordfeats_test, preproc) - wordfeats_test$apply_preproc(wordfeats_test) - - pos_train <- HMM(wordfeats_train, N, M, pseudo) - pos_train$train(pos_train) - pos_train$baum_welch_viterbi_train(pos_train, 'BW_NORMAL') - neg_train <- HMM(wordfeats_train, N, M, pseudo) - neg_train$train(neg_train) - neg_train$baum_welch_viterbi_train(neg_train, 'BW_NORMAL') - pos_test=HMM(pos_train) - pos_test$set_observations(pos_test, wordfeats_test) - neg_test=HMM(neg_train) - neg_test$set_observations(neg_test, wordfeats_test) - - return(list(list(pos_train, neg_train), list(pos_test, neg_test))) -} - diff --git a/tests/integration/r_modular/util/read_mfile.R b/tests/integration/r_modular/util/read_mfile.R deleted file mode 100644 index 863d2498b31..00000000000 --- a/tests/integration/r_modular/util/read_mfile.R +++ /dev/null @@ -1,35 +0,0 @@ -read_mfile <- function(filename) { - res <- c() - f <- file(filename) - lines <- readLines(f) - - for (line in lines) { - if (regexpr("[[]",line)>0) { - n <- nchar(line) - line <- gsub(";","::",line) - rows <- nchar(line)-n - #rows <- nchar(line)-n+1 - line <- gsub("::",",",line) - line <- gsub("=", "<-",line) - line <- gsub("[[]","matrix(c(",line) - line <- gsub("[]]",paste("),nrow=",rows, ",byrow=TRUE)"),line) - line <- gsub(";",",",line) - line <- gsub(",$", "", line) # remove trailing comma - res <- cbind(res, line) - #tcon <- textConnection(line) - #source(tcon) - #close(tcon) - } else { - line <- gsub("=","<- ",line) - line <- gsub("[{]", "c(",line) - line <- gsub("[}]", ')', line) - res <- cbind(res, line) - #tcon <- textConnection(line) - #source(tcon) - #close(tcon) - } - } - - close(f) - return(res) -} diff --git a/tests/integration/r_modular/util/tobool.R b/tests/integration/r_modular/util/tobool.R deleted file mode 100644 index 9e2ee7ac08d..00000000000 --- a/tests/integration/r_modular/util/tobool.R +++ /dev/null @@ -1,9 +0,0 @@ -tobool <- function(value) { - evaluated=eval(parse(text=toupper(value))) - if (typeof(evaluated)=='logical') { - return(evaluated) - } else { - print(paste('Could not make a bool out of', value)) - return(FALSE) - } -} diff --git a/tests/integration/r_static/classifier.R b/tests/integration/r_static/classifier.R deleted file mode 100644 index 1815d26d7c9..00000000000 --- a/tests/integration/r_static/classifier.R +++ /dev/null @@ -1,115 +0,0 @@ -classifier <- function(filename) { - source('util/set_features.R') - source('util/set_kernel.R') - source('util/set_distance.R') - source('util/check_accuracy.R') - source('util/fix_classifier_name_inconsistency.R') - - if (regexpr('Perceptron', classifier_name)>0) { # b0rked, skip it - return(TRUE) - } - - if (regexpr('kernel', classifier_type)>0) { - if (!set_features('kernel_')) { - return(TRUE) - } - if (!set_kernel()) { - return(TRUE) - } - } else if (regexpr('knn', classifier_type)>0) { - if (!set_features('distance_')) { - return(TRUE) - } - if (!set_distance()) { - return(TRUE) - } - } else { - if (!set_features('classifier_')) { - return(TRUE) - } - } - - if (exists('classifier_labels')) { - sg('set_labels', 'TRAIN', classifier_labels) - } - - cname <- fix_classifier_name_inconsistency(classifier_name) - try(sg('new_classifier', cname)) - - if (exists('classifier_bias')) { - sg('svm_use_bias', TRUE) - } else { - sg('svm_use_bias', FALSE) - } - - if (exists('classifier_epsilon')) { - sg('svm_epsilon', classifier_epsilon) - } - if (exists('classifier_max_train_time')) { - sg('svm_max_train_time', classifier_max_train_time) - } - if (exists('classifier_linadd_enabled')) { - sg('use_linadd', TRUE) - } - if (exists('classifier_batch_enabled')) { - sg('use_batch_computation', TRUE) - } - if (exists('classifier_num_threads')) { - sg('threads', classifier_num_threads) - } - - if (regexpr('knn', classifier_type)>0) { - sg('train_classifier', classifier_k) - } else if (regexpr('lda', classifier_type)>0) { - sg('train_classifier', classifier_gamma) - } else { - if (exists('classifier_C')) { - sg('c', classifier_C) - } - sg('train_classifier') - } - - alphas <- 0 - bias <- 0 - sv <- 0 - - if (regexpr('lda', classifier_type)>0) { - 0 # nop - } else { - if (exists('classifier_bias') && exists('classifier_label_type') && regexpr('series', classifier_label_type)<=0) { - res <- sg('get_svm') - bias <- abs(res[[1]]-classifier_bias) - } - - if (exists('classifier_alpha_sum') && exists('classifier_sv_sum')) { - if (exists('classifier_label_type') && regexpr('series', classifier_label_type)>0) { - for (i in 0:(sg('get_num_svms')-1)) { - weights <- t(sg('get_svm', i)[[2]]) - for (j in 1:length(weights[1,])) { - alphas <- alphas + weights[1, j] - } - for (j in 1:length(weights[2,])) { - sv <- sv + weights[2, j] - } - } - alphas <- abs(alphas-classifier_alpha_sum) - sv <- abs(sv-classifier_sv_sum) - } else { - weights <- t(sg('get_svm')[[2]]) - for (i in 1:length(weights[1,]) ){ - alphas <- alphas + weights[1, i] - } - alphas <- abs(alphas-classifier_alpha_sum) - for (i in 1:length(weights[2,])) { - sv <- sv + weights[2, i] - } - sv <- abs(sv-classifier_sv_sum) - } - } - } - - classified <- max(abs(sg('classify')-classifier_classified)) - - data <- list(alphas, bias, sv, classified) - return(check_accuracy(classifier_accuracy, 'classifier', data)) -} diff --git a/tests/integration/r_static/clustering.R b/tests/integration/r_static/clustering.R deleted file mode 100644 index e9efc83c85e..00000000000 --- a/tests/integration/r_static/clustering.R +++ /dev/null @@ -1,58 +0,0 @@ -clustering <- function(filename) { - source('util/set_features.R') - source('util/set_distance.R') - source('util/check_accuracy.R') - source('util/fix_clustering_name_inconsistency.R') - - if (!set_features('distance_')) { - return(TRUE) - } - - if (!set_distance()) { - return(TRUE) - } - - cname <- fix_clustering_name_inconsistency(clustering_name) - sg('new_clustering', cname) - - if (exists('clustering_max_iter')) { - max_iter <- clustering_max_iter - } else { - max_iter <- 1000 - } - - if (exists('clustering_k')) { - first_arg <- clustering_k - } else if (exists('clustering_merges')) { - first_arg <- clustering_merges - } else { - print('Incomplete clustering data!') - return(FALSE) - } - - sg('train_clustering', first_arg, max_iter) - - if (exists('clustering_radi')) { - res <- sg('get_clustering') - radi <- t(res[[1]]) - radi <- max(abs(radi-clustering_radi)) - centers <- max(max(abs(res[[2]]-clustering_centers))) - - data <- list(centers, radi); - return(check_accuracy(clustering_accuracy, 'kmeans', data)) - - } else if (exists('clustering_merge_distance')) { - res <- sg('get_clustering') - merge_distances <- t(res[[1]]) - merge_distances <- max(abs( - merge_distances-clustering_merge_distance)) - pairs <- max(max(abs(res[[2]]-clustering_pairs))) - - data <- list(merge_distances, pairs) - return(check_accuracy(clustering_accuracy, 'hierarchical', data)) - - } else { - print('Incomplete clustering data!') - return(FALSE) - } -} diff --git a/tests/integration/r_static/distance.R b/tests/integration/r_static/distance.R deleted file mode 100644 index 0e0a85199e6..00000000000 --- a/tests/integration/r_static/distance.R +++ /dev/null @@ -1,22 +0,0 @@ -distance <- function(filename) { - source('util/set_features.R') - source('util/set_distance.R') - source('util/check_accuracy.R') - - if (!set_features('distance_')) { - return(TRUE) - } - - if (!set_distance()) { - return(TRUE) - } - - dmatrix <- sg('get_distance_matrix', 'TRAIN') - dm_train <- max(max(abs(distance_matrix_train-dmatrix))) - - dmatrix <- sg('get_distance_matrix', 'TEST') - dm_test <- max(max(abs(distance_matrix_test-dmatrix))) - - data <- list(dm_train, dm_test) - return(check_accuracy(distance_accuracy, 'distance', data)) -} diff --git a/tests/integration/r_static/distribution.R b/tests/integration/r_static/distribution.R deleted file mode 100644 index 9beb1eb9026..00000000000 --- a/tests/integration/r_static/distribution.R +++ /dev/null @@ -1,23 +0,0 @@ -distribution <- function(filename) { - source('util/set_features.R') - source('util/check_accuracy.R') - - sg('init_random', init_random) - set.seed(init_random) - - if (!set_features('distribution_')) { - return(TRUE) - } - - if (regexpr('^HMM', distribution_name)>0) { - sg('new_hmm', distribution_N, distribution_M) - sg('bw') - } else { - print('Cannot yet train other distributions than HMM!') - return(TRUE) - } - - likelihood <- abs(sg('hmm_likelihood')-distribution_likelihood) - data <- list(likelihood, 0) - return(check_accuracy(distribution_accuracy, 'distribution', data)) -} diff --git a/tests/integration/r_static/kernel.R b/tests/integration/r_static/kernel.R deleted file mode 100644 index eb405cd7ab6..00000000000 --- a/tests/integration/r_static/kernel.R +++ /dev/null @@ -1,24 +0,0 @@ -kernel <- function() { - source('util/set_features.R') - source('util/set_kernel.R') - source('util/check_accuracy.R') - - if (!set_features('kernel_')) { - return(TRUE) - } - - if (!set_kernel()) { - return(TRUE) - } - - kmatrix <- sg('get_kernel_matrix', 'TRAIN') - km_train <- max(max(abs(kernel_matrix_train-kmatrix))) - - kmatrix <- sg('get_kernel_matrix', 'TEST') - km_test <- max(max(abs(kernel_matrix_test-kmatrix))) - - data <- list(km_train, km_test) - return(check_accuracy(kernel_accuracy, 'kernel', data)) -} - -# vim: set filetype=R diff --git a/tests/integration/r_static/preprocessor.R b/tests/integration/r_static/preprocessor.R deleted file mode 100644 index c77218976c0..00000000000 --- a/tests/integration/r_static/preprocessor.R +++ /dev/null @@ -1,34 +0,0 @@ -preprocessor <- function(filename) { - source('util/set_features.R') - source('util/set_kernel.R') - source('util/check_accuracy.R') - source('util/tobool.R') - source('util/fix_preproc_name_inconsistency.R'); - - if (!set_features('kernel_')) { - return(TRUE) - } - - pname <- fix_preproc_name_inconsistency(preprocessor_name) - if (regexpr('PRUNEVARSUBMEAN', pname)>0) { - sg('add_preproc', pname, tobool(preprocessor_arg0_divide)) - } else { - sg('add_preproc', pname) - } - - sg('attach_preproc', 'TRAIN') - sg('attach_preproc', 'TEST') - - if (!set_kernel()) { - return(TRUE); - } - - kmatrix <- sg('get_kernel_matrix', 'TRAIN') - km_train <- max(max(abs(kernel_matrix_train-kmatrix))) - - kmatrix <- sg('get_kernel_matrix', 'TEST') - km_test <- max(max(abs(kernel_matrix_test-kmatrix))) - - data <- list(km_train, km_test) - return(check_accuracy(kernel_accuracy, 'kernel', data)) -} diff --git a/tests/integration/r_static/regression.R b/tests/integration/r_static/regression.R deleted file mode 100644 index 818fb1e2040..00000000000 --- a/tests/integration/r_static/regression.R +++ /dev/null @@ -1,57 +0,0 @@ -regression <- function(filename) { - source('util/set_features.R') - source('util/set_kernel.R') - source('util/check_accuracy.R') - source('util/tobool.R') - source('util/fix_regression_name_inconsistency.R') - - if (!set_features('kernel_')) { - return(TRUE) - } - - if (!set_kernel()) { - return(TRUE); - } - - sg('threads', regression_num_threads) - sg('set_labels', 'TRAIN', regression_labels) - - rname <- fix_regression_name_inconsistency(regression_name) - try(sg('new_regression', rname)) - - if (regexpr('svm', regression_type)>0) { - sg('c', regression_C) - sg('svm_epsilon', regression_epsilon) - sg('svr_tube_epsilon', regression_tube_epsilon) - } else if (regexpr('kernelmachine', regression_type)>0) { - sg('krr_tau', regression_tau) - } else { - print('Incomplete regression data!') - } - - sg('train_regression') - - alphas <- 0 - bias <- 0 - sv <- 0 - if (exists('regression_bias')) { - res <- sg('get_svm') - - bias <- abs(res[[1]]-regression_bias) - - weights <- t(res[[2]]) - for (i in 1:length(weights[1,]) ){ - alphas <= alphas + weights[1, i] - } - alphas <- abs(alphas-regression_alpha_sum) - for (i in 1:length(weights[2,])) { - sv <- sv + weights[2, i] - } - sv <- abs(sv-regression_sv_sum) - } - - classified <- max(abs(sg('classify')-regression_classified)) - - data <- list(alphas, bias, sv, classified) - return(check_accuracy(regression_accuracy, 'classifier', data)) -} diff --git a/tests/integration/r_static/test_all.sh b/tests/integration/r_static/test_all.sh deleted file mode 100755 index 23d168f17b3..00000000000 --- a/tests/integration/r_static/test_all.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash - -DATAPATH='../data' -exitcode=0 - -function test_all () { - datapath=${1} - - if echo "$datapath" | grep -q '\.\./data/tests' - then - continue - fi - - echo "*** Testing in ${datapath}" - - sleep 1 - for file in ${datapath}; do - if grep -q $file ../blacklist - then - echo 'SKIPPING' - else - dump=`echo ${file} | grep WDSVMOcas` - if [ $? -eq 0 ]; then - echo WDSVMOcas totally unsupported: ERROR - else - echo -n "${file}" - echo -n -e "\t\t" - - output=`R --no-save --no-restore --no-readline --slave ${file} < test_one.R` - if [ $? -ne 0 ]; then - exitcode=1 - echo ERROR - echo ${output} - else - echo OK - fi - fi - fi - done - sleep 1 - echo -} - - -if [ -n "${1}" ]; then - test_all "${DATAPATH}/${1}/*.m" -else - for i in ${DATAPATH}/*; do - test_all "${i}/*.m" - done -fi -exit $exitcode diff --git a/tests/integration/r_static/test_one.R b/tests/integration/r_static/test_one.R deleted file mode 100644 index 6844a35e3f9..00000000000 --- a/tests/integration/r_static/test_one.R +++ /dev/null @@ -1,22 +0,0 @@ -library(sg) - -source("util/read_mfile.R") -filename <- commandArgs()[6] -indata <- read_mfile(filename) -for (i in 1:length(indata)) { - eval(parse(text=indata[i])) -} - -# there is some randomness involved, alas this is -# not working correctly in R -dump <- sg('init_random', init_random) -set.seed(init_random) - -path=strsplit(filename, '/', fixed=TRUE) -module=path[[1]][3] - -source(paste(module, '.R', sep='')) -res <- eval(parse(text=paste(module, '()', sep=''))) - -# res==TRUE is 1; have to negate to get shell-compatible truth value -quit(status=!res) diff --git a/tests/integration/r_static/test_one.sh b/tests/integration/r_static/test_one.sh deleted file mode 100755 index 4b7c0ffd4da..00000000000 --- a/tests/integration/r_static/test_one.sh +++ /dev/null @@ -1,7 +0,0 @@ -dump=`echo ${1} | grep WDSVMOcas` -if [ $? -eq 0 ]; then - echo "WDSVMOcas totally unsupported" - exit 1 -fi - -R --no-save --no-restore --no-readline --slave ${1} < test_one.R diff --git a/tests/integration/r_static/util/check_accuracy.R b/tests/integration/r_static/util/check_accuracy.R deleted file mode 100644 index 69d17312475..00000000000 --- a/tests/integration/r_static/util/check_accuracy.R +++ /dev/null @@ -1,26 +0,0 @@ -check_accuracy <- function(accuracy, type, data) { - if (regexpr(type, 'classifier')>0) { - print(paste('alphas:', data[1], ', bias:', data[2], ', sv:', data[3], ', classified:', data[4], '<--- accuracy ', accuracy)) - } else if (regexpr(type, 'distance')>0) { - print(paste('dtrain:', data[1], ', dtest:', data[2], '<--- accuracy ', accuracy)) - } else if (regexpr(type, 'distribution')>0) { - print(paste('likelihood:', data[1], ', derivatives:', data[2], '<--- accuracy ', accuracy)) - } else if (regexpr(type, 'hierarchical')>0) { - print(paste('merge_distances:', data[1], ', pairs:', data[2], '<--- accuracy', accuracy)) - } else if (regexpr(type, 'kernel')>0) { - print(paste('ktrain:', data[1], ', ktest:', data[2], '<--- accuracy', accuracy)) - } else if (regexpr(type, 'kmeans')>0) { - print(paste('centers:', data[1], ', radi:', data[2], '<--- accuracy', accuracy)) - } else { - print(paste('Unknown accuracy type', type, '!')) - return(FALSE) - } - - for (i in 1:length(data)) { - if (data[i]>accuracy) { - return(FALSE) - } - } - - return(TRUE) -} diff --git a/tests/integration/r_static/util/convert_features_and_add_preproc.R b/tests/integration/r_static/util/convert_features_and_add_preproc.R deleted file mode 100644 index 667309c1a63..00000000000 --- a/tests/integration/r_static/util/convert_features_and_add_preproc.R +++ /dev/null @@ -1,28 +0,0 @@ -convert_features_and_add_preproc <- function(prefix) { - if (!exists(paste(prefix, 'order', sep=''), mode='numeric')) { - return(FALSE) - } - - ftype <- eval(parse(text=paste(prefix, 'feature_type', sep=''))) - if (regexpr('Ulong', ftype)>0) { - type <- 'ULONG' - } else if (regexpr('Word', ftype)>0) { - type <- 'WORD' - } else { - return(FALSE) - } - - order <- eval(parse(text=paste(prefix, 'order', sep=''))) - gap <- eval(parse(text=paste(prefix, 'gap', sep=''))) - reverse <- eval(parse(text=paste(prefix, 'reverse', sep=''))) - sg('add_preproc', paste('SORT', type, 'STRING', sep='')) - sg('convert', 'TRAIN', 'STRING', 'CHAR', 'STRING', type, - order, order-1, gap, reverse) - sg('attach_preproc', 'TRAIN') - - sg('convert', 'TEST', 'STRING', 'CHAR', 'STRING', type, - order, order-1, gap, reverse) - sg('attach_preproc', 'TEST') - - return(TRUE) -} diff --git a/tests/integration/r_static/util/fix_classifier_name_inconsistency.R b/tests/integration/r_static/util/fix_classifier_name_inconsistency.R deleted file mode 100644 index dc9ca0d9325..00000000000 --- a/tests/integration/r_static/util/fix_classifier_name_inconsistency.R +++ /dev/null @@ -1,12 +0,0 @@ -fix_classifier_name_inconsistency <- function(cname) { - cname <- toupper(cname) - - if (regexpr('LIBSVM', cname)>0) { - if (nchar(cname)>nchar('LIBSVM')) { - ending <- strsplit(cname, 'LIBSVM')[[1]][2] - return(paste('LIBSVM_', ending, sep='')) - } - } - - return(cname) -} diff --git a/tests/integration/r_static/util/fix_clustering_name_inconsistency.R b/tests/integration/r_static/util/fix_clustering_name_inconsistency.R deleted file mode 100644 index 64244355dea..00000000000 --- a/tests/integration/r_static/util/fix_clustering_name_inconsistency.R +++ /dev/null @@ -1,3 +0,0 @@ -fix_clustering_name_inconsistency <- function(cname) { - return(toupper(cname)) -} diff --git a/tests/integration/r_static/util/fix_distance_name_inconsistency.R b/tests/integration/r_static/util/fix_distance_name_inconsistency.R deleted file mode 100644 index 2957e0263cb..00000000000 --- a/tests/integration/r_static/util/fix_distance_name_inconsistency.R +++ /dev/null @@ -1,12 +0,0 @@ -fix_distance_name_inconsistency <- function(dname) { - dname=toupper(dname) - if (regexpr('WORDDISTANCE', dname)>0) { - return(strsplit(dname, 'WORDDISTANCE')[[1]]) - } else if (regexpr('DISTANCE', dname)>0) { - return(strsplit(dname, 'DISTANCE')[[1]]) - } else if (regexpr('METRIC', dname)>0) { - return(strsplit(dname, 'METRIC')[[1]]) - } else { - return(dname); - } -} diff --git a/tests/integration/r_static/util/fix_kernel_name_inconsistency.R b/tests/integration/r_static/util/fix_kernel_name_inconsistency.R deleted file mode 100644 index b4fa5661586..00000000000 --- a/tests/integration/r_static/util/fix_kernel_name_inconsistency.R +++ /dev/null @@ -1,38 +0,0 @@ -fix_kernel_name_inconsistency = function(kname) { - kname=toupper(kname); - - if (regexpr('SIMPLELOCALITYIMPROVEDSTRING', kname)>0) { - return('SLIK') - } - else if (regexpr('LOCALITYIMPROVEDSTRING', kname)>0) { - return('LIK') - } - else if (regexpr('WORDMATCH', kname)>0) { - return('MATCH') - } - else if (regexpr('WEIGHTEDDEGREEPOSITIONSTRING', kname)>0) { - return('WEIGHTEDDEGREEPOS') - } - else if (regexpr('WEIGHTEDCOMMWORDSTRING', kname)>0) { - return('WEIGHTEDCOMMSTRING') - } - else if (regexpr('COMMULONGSTRING', kname)>0) { - return('COMMSTRING') - } - else if (regexpr('COMMWORDSTRING', kname)>0) { - return('COMMSTRING') - } - else if (regexpr('WORDSTRING', kname)>0) { - return(strsplit(kname, 'WORDSTRING')[[1]]) - } - else if (regexpr('STRING', kname)>0) { - return(strsplit(kname, 'STRING')[[1]]) - } - else if (regexpr('WORD', kname)>0) { - return(strsplit(kname, 'WORD')[[1]]) - } - else { - return(kname); - } - -} diff --git a/tests/integration/r_static/util/fix_normalizer_name_inconsistency.R b/tests/integration/r_static/util/fix_normalizer_name_inconsistency.R deleted file mode 100644 index 5c728e7b872..00000000000 --- a/tests/integration/r_static/util/fix_normalizer_name_inconsistency.R +++ /dev/null @@ -1,13 +0,0 @@ -fix_normalizer_name_inconsistency <- function(name) { - if (regexpr('Identity', name)>0) { - return('IDENTITY') - } else if (regexpr('AvgDiag', name)>0) { - return('AVGDIAG') - } else if (regexpr('SqrtDiag', name)>0) { - return('SQRTDIAG') - } else if (regexpr('FirstElement', name)>0) { - return('FIRSTELEMENT') - } else { - return(FALSE) - } -} diff --git a/tests/integration/r_static/util/fix_preproc_name_inconsistency.R b/tests/integration/r_static/util/fix_preproc_name_inconsistency.R deleted file mode 100644 index e9dae11fe63..00000000000 --- a/tests/integration/r_static/util/fix_preproc_name_inconsistency.R +++ /dev/null @@ -1,3 +0,0 @@ -fix_preproc_name_inconsistency <- function(pname) { - return(toupper(pname)) -} diff --git a/tests/integration/r_static/util/fix_regression_name_inconsistency.R b/tests/integration/r_static/util/fix_regression_name_inconsistency.R deleted file mode 100644 index 1f23304e490..00000000000 --- a/tests/integration/r_static/util/fix_regression_name_inconsistency.R +++ /dev/null @@ -1,3 +0,0 @@ -fix_regression_name_inconsistency <- function(rname) { - return(toupper(rname)) -} diff --git a/tests/integration/r_static/util/read_mfile.R b/tests/integration/r_static/util/read_mfile.R deleted file mode 100644 index 863d2498b31..00000000000 --- a/tests/integration/r_static/util/read_mfile.R +++ /dev/null @@ -1,35 +0,0 @@ -read_mfile <- function(filename) { - res <- c() - f <- file(filename) - lines <- readLines(f) - - for (line in lines) { - if (regexpr("[[]",line)>0) { - n <- nchar(line) - line <- gsub(";","::",line) - rows <- nchar(line)-n - #rows <- nchar(line)-n+1 - line <- gsub("::",",",line) - line <- gsub("=", "<-",line) - line <- gsub("[[]","matrix(c(",line) - line <- gsub("[]]",paste("),nrow=",rows, ",byrow=TRUE)"),line) - line <- gsub(";",",",line) - line <- gsub(",$", "", line) # remove trailing comma - res <- cbind(res, line) - #tcon <- textConnection(line) - #source(tcon) - #close(tcon) - } else { - line <- gsub("=","<- ",line) - line <- gsub("[{]", "c(",line) - line <- gsub("[}]", ')', line) - res <- cbind(res, line) - #tcon <- textConnection(line) - #source(tcon) - #close(tcon) - } - } - - close(f) - return(res) -} diff --git a/tests/integration/r_static/util/set_distance.R b/tests/integration/r_static/util/set_distance.R deleted file mode 100644 index 61f4d565f8e..00000000000 --- a/tests/integration/r_static/util/set_distance.R +++ /dev/null @@ -1,17 +0,0 @@ -set_distance <- function() { - source('util/tobool.R') - source('util/fix_distance_name_inconsistency.R') - - dname <- fix_distance_name_inconsistency(distance_name) - ftype <- toupper(distance_feature_type) - - if (regexpr('HAMMING', dname)>0) { - sg('set_distance', dname, ftype, tobool(distance_arg0_use_sign)); - } else if (regexpr('MINKOWSKI', dname)>0) { - sg('set_distance', dname, ftype, distance_arg0_k) - } else { - sg('set_distance', dname, ftype) - } - - return(TRUE) -} diff --git a/tests/integration/r_static/util/set_features.R b/tests/integration/r_static/util/set_features.R deleted file mode 100644 index 9aad113a89f..00000000000 --- a/tests/integration/r_static/util/set_features.R +++ /dev/null @@ -1,96 +0,0 @@ -set_features <- function(prefix) { - source('util/convert_features_and_add_preproc.R') - - name <- eval(parse(text=paste(prefix, 'name', sep=''))) - if (exists(paste(prefix, 'alphabet', sep=''))) { - alphabet <- eval(parse(text=paste(prefix, 'alphabet', sep=''))) - } - if (exists(paste(prefix, 'data_train', sep=''))) { - data_train <- eval(parse(text=paste(prefix, 'data_train', sep=''))) - data_test <- eval(parse(text=paste(prefix, 'data_test', sep=''))) - } - - if (regexpr('Sparse', name)>0) { - print('Sparse features not supported yet!') - return(FALSE) - } - - if (exists('classifier_type')) { - if (regexpr('linear', classifier_type)>0) { - print('Linear classifiers with sparse features not supported yet!') - return(FALSE) - } - } - - if (exists('alphabet')) { - if (regexpr('RAWBYTE', alphabet)>0) { - print('Alphabet RAWBYTE not supported yet!') - return(FALSE) - } - } - - if (exists('topfk_name')) { - print(paste('Features', topfk_name, 'not supported yet!')) - return(FALSE) - } - - sg('clean_features', 'TRAIN') - sg('clean_features', 'TEST') - - if (regexpr('Combined', name)>0) { - if (exists('kernel_subkernel0_alphabet')) { - sg('add_features', 'TRAIN', - kernel_subkernel0_data_train, kernel_subkernel0_alphabet) - sg('add_features', 'TEST', - kernel_subkernel0_data_test, kernel_subkernel0_alphabet) - } else { - sg('add_features', 'TRAIN', kernel_subkernel0_data_train) - sg('add_features', 'TEST', kernel_subkernel0_data_test) - } - - if (exists('kernel_subkernel1_alphabet')) { - sg('add_features', 'TRAIN', - kernel_subkernel1_data_train, kernel_subkernel1_alphabet) - sg('add_features', 'TEST', - kernel_subkernel1_data_test, kernel_subkernel1_alphabet) - } else { - sg('add_features', 'TRAIN', kernel_subkernel1_data_train) - sg('add_features', 'TEST', kernel_subkernel1_data_test) - } - - if (exists('kernel_subkernel2_alphabet')) { - sg('add_features', 'TRAIN', - kernel_subkernel2_data_train, kernel_subkernel2_alphabet) - sg('add_features', 'TEST', - kernel_subkernel2_data_test, kernel_subkernel2_alphabet) - } else { - sg('add_features', 'TRAIN', kernel_subkernel2_data_train) - sg('add_features', 'TEST', kernel_subkernel2_data_test) - } - } - - else if (exists('alphabet')) { - sg('set_features', 'TRAIN', data_train, alphabet) - sg('set_features', 'TEST', data_test, alphabet) - } - - else if (exists('data_train')) { - ftype <- eval(parse(text=paste(prefix, 'feature_type', sep=''))) - if (regexpr('Real', ftype)<0) { - print(paste('No support for feature type', ftype, 'in R!')) - return(FALSE) - } - - sg('set_features', 'TRAIN', data_train) - sg('set_features', 'TEST', data_test) - } - - else { - sg('set_features', 'TRAIN', kernel_data) - sg('set_features', 'TEST', kernel_data) - } - - convert_features_and_add_preproc(prefix) - - return(TRUE) -} diff --git a/tests/integration/r_static/util/set_kernel.R b/tests/integration/r_static/util/set_kernel.R deleted file mode 100644 index abe6065c037..00000000000 --- a/tests/integration/r_static/util/set_kernel.R +++ /dev/null @@ -1,164 +0,0 @@ -set_kernel <- function() { - source('util/tobool.R') - source('util/fix_kernel_name_inconsistency.R') - source('util/fix_normalizer_name_inconsistency.R') - - kname <- fix_kernel_name_inconsistency(kernel_name) - - if (regexpr(kname, 'AUC')>0 || regexpr(kname, 'CUSTOM')>0) { - print(paste('Kernel', kname, 'not supported yet!')) - return(FALSE) - } - - if (exists('kernel_arg0_size')) { - size_cache <- kernel_arg0_size - } - else if (exists('kernel_arg1_size')) { - size_cache <- kernel_arg1_size - } else { - size_cache <- 10 - } - - if (exists('kernel_feature_type')) { - ftype <- toupper(kernel_feature_type) - } - - if (regexpr('SIGMOID', kname)>0) { - sg('set_kernel', kname, ftype, size_cache, - kernel_arg1_gamma, kernel_arg2_coef0) - } - - else if (regexpr('CHI2', kname)>0) { - sg('set_kernel', kname, ftype, size_cache, kernel_arg1_width) - } - - else if (regexpr('CONST', kname)>0) { - sg('set_kernel', kname, ftype, size_cache, kernel_arg0_c) - } - - else if (regexpr('DIAG', kname)>0) { - sg('set_kernel', kname, ftype, size_cache, kernel_arg1_diag) - } - - else if (regexpr('GAUSSIANSHIFT', kname)>0) { - sg('set_kernel', kname, ftype, size_cache, - kernel_arg1_width, kernel_arg2_max_shift, kernel_arg3_shift_step) - } - - else if (regexpr('GAUSSIAN', kname)>0) { - if (exists('kernel_arg0_width')) { - width <- kernel_arg0_width - } else { - width <- kernel_arg1_width - } - sg('set_kernel', kname, ftype, size_cache, width) - sg('set_kernel', kname, ftype, size_cache, width) - } - - else if (regexpr('LINEAR', kname)>0) { - if (exists('kernel_arg0_scale')) { - sg('set_kernel', kname, ftype, size_cache, kernel_arg0_scale) - } else { - sg('set_kernel', kname, ftype, size_cache, -1) - } - } - - else if (regexpr('OLIGO', kname)>0) { - sg('set_kernel', kname, ftype, size_cache, - kernel_arg1_k, kernel_arg2_width) - } - - - else if (regexpr('POLYMATCH', kname)>0) { - sg('set_kernel', kname, ftype, size_cache, - kernel_arg1_degree, tobool(kernel_arg2_inhomogene)) - } - - else if (regexpr('POLY', kname)>0) { - sg('set_kernel', kname, ftype, size_cache, - kernel_arg1_degree, tobool(kernel_arg2_inhomogene)) - } - - else if (regexpr('MATCH', kname)>0) { - sg('set_kernel', kname, ftype, size_cache, kernel_arg1_degree) - } - - else if (regexpr('COMMSTRING', kname)>0) { # normal + WEIGHTED - if (exists('kernel_arg1_use_sign')) { - sg('set_kernel', kname, ftype, size_cache, - tobool(kernel_arg1_use_sign)) - } else { - sg('set_kernel', kname, ftype, size_cache) - } - } - - else if (regexpr('DEGREE', kname)>0) { # FIXED + WEIGHTED - if (exists('kernel_arg0_degree')) { - sg('set_kernel', kname, ftype, size_cache, kernel_arg0_degree) - } else { - sg('set_kernel', kname, ftype, size_cache, kernel_arg1_degree) - } - } - - else if (regexpr('HISTOGRAM', kname)>0 || regexpr('SALZBERG', kname)>0) { - pseudo_pos=1e-10 - pseudo_neg=1e-10 - sg('new_plugin_estimator', pseudo_pos, pseudo_neg) - sg('set_labels', 'TRAIN', classifier_labels) - sg('train_estimator') - - sg('set_kernel', kname, ftype, size_cache) - } - - else if (regexpr('DISTANCE', kname)>0) { - source('util/fix_distance_name_inconsistency.R') - dname <- fix_distance_name_inconsistency(kernel_arg2_distance) - # FIXME: REAL is cheating and will break in the future - sg('set_distance', dname, 'REAL') - sg('set_kernel', kname, size_cache, kernel_arg1_width) - } - - else if (regexpr('LOCALALIGNMENT', kname)>0) { - sg('set_kernel', kname, ftype, size_cache) - } - - else if (regexpr('LIK', kname)>0) { - sg('set_kernel', kname, ftype, size_cache, - kernel_arg1_length, kernel_arg2_inner_degree, - kernel_arg3_outer_degree) - } - - else if (regexpr('COMBINED', kname)>0) { - # this will break when test file is changed! - sg('set_kernel', 'COMBINED', size_cache) - - subkernel_name=fix_kernel_name_inconsistency(kernel_subkernel0_name) - sg('add_kernel', 1., subkernel_name, - toupper(kernel_subkernel0_feature_type), - kernel_subkernel0_arg0_size, - kernel_subkernel0_arg1_degree) - - subkernel_name=fix_kernel_name_inconsistency(kernel_subkernel1_name) - sg('add_kernel', 1., subkernel_name, - toupper(kernel_subkernel1_feature_type), - kernel_subkernel1_arg0_size, - kernel_subkernel1_arg1_degree, - tobool(kernel_subkernel1_arg2_inhomogene)) - - subkernel_name=fix_kernel_name_inconsistency(kernel_subkernel2_name) - sg('add_kernel', 1., subkernel_name, - toupper(kernel_subkernel2_feature_type), - kernel_subkernel2_arg0_size) - } - - else { - return(FALSE) - } - - if (exists('kernel_normalizer')) { - nname=fix_normalizer_name_inconsistency(kernel_normalizer) - sg('set_kernel_normalization', nname) - } - - return(TRUE) -} diff --git a/tests/integration/r_static/util/tobool.R b/tests/integration/r_static/util/tobool.R deleted file mode 100644 index 9e2ee7ac08d..00000000000 --- a/tests/integration/r_static/util/tobool.R +++ /dev/null @@ -1,9 +0,0 @@ -tobool <- function(value) { - evaluated=eval(parse(text=toupper(value))) - if (typeof(evaluated)=='logical') { - return(evaluated) - } else { - print(paste('Could not make a bool out of', value)) - return(FALSE) - } -} diff --git a/tests/integration/test_configure.py b/tests/integration/test_configure.py deleted file mode 100644 index d937a6406e3..00000000000 --- a/tests/integration/test_configure.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/python - -import os - -opts='' -#opts=' --prefix=/local/cong --libs=/local/cong/lib --includes=/local/cong/include' - -interfaces = ['elwms','cmdline', 'python_modular', 'python', 'r', 'r_modular', - 'octave', 'octave_modular', 'matlab','libshogun','libshogunui']; - -def powerset(s): - """Return the powerset of a list""" - d = dict(zip( - (1</dev/null 2>&1') - exit_status.append((conf,p)) - print conf,p - print exit_status - -if __name__ == '__main__': - test_configure()