diff --git a/.gitattributes b/.gitattributes index 8f9b335558fe..35256b054db0 100644 --- a/.gitattributes +++ b/.gitattributes @@ -35,6 +35,8 @@ Readme text *.psm1 text *.ps text *.i text +*.cmake text +*.patch text *.sh text eol=lf *.awk text eol=lf diff --git a/.gitignore b/.gitignore index 6c1b5e52b967..34801eccd13c 100644 --- a/.gitignore +++ b/.gitignore @@ -255,8 +255,9 @@ bindings/python/cntk/i_plus_i_0.mod bindings/csharp/Swig/cntk_cs_wrap.h bindings/csharp/Swig/cntk_cs_wrap.cxx bindings/csharp/Swig/cntk_cs_wrap.cxx -bindings/csharp/CNTKLibraryManagedDll/SwigProxyClasses/GeneratedCode/Debug/*.cs -bindings/csharp/CNTKLibraryManagedDll/SwigProxyClasses/GeneratedCode/Release/*.cs +bindings/csharp/Swig/Generated/* +bindings/csharp/CNTKLibraryManagedDll/build/Linux/Debug/CSharpBindings/CMakeFiles/CSharpBindings.dir/depend.make +bindings/csharp/CNTKLibraryManagedDll/build/Linux/Release/CSharpBindings/CMakeFiles/CSharpBindings.dir/depend.make # Java bindings bindings/java/Swig/cntk_java_wrap.h diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 000000000000..4f585e4319f7 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,11 @@ +cmake_minimum_required(VERSION 3.8) + +project(CNTK) + +include(Tools/cmake/options.cmake REQUIRED) +include(Tools/cmake/common.cmake REQUIRED) +include(Tools/cmake/cntk_common.cmake REQUIRED) + +add_subdirectory(bindings/csharp/swig) + +include(Tools/cmake/debug.cmake REQUIRED) \ No newline at end of file diff --git a/CNTK.sln b/CNTK.sln index ae4e6a462e83..84b46c245a7b 100644 --- a/CNTK.sln +++ b/CNTK.sln @@ -1609,7 +1609,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ImageRecognizerLib", "Tests {E5606ECE-48CA-4464-BB12-09D81D02B9EF} = {E5606ECE-48CA-4464-BB12-09D81D02B9EF} EndProjectSection EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CNTKLibraryCSTrainingTest", "Tests\EndToEndTests\CNTKv2CSharp\CNTKLibraryCSTrainingTest\CNTKLibraryCSTrainingTest.csproj", "{0DF2109B-BB85-4718-82DE-1C0536D4F2C3}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CNTKLibraryCSTrainingTest.standard", "Tests\EndToEndTests\CNTKv2CSharp\CNTKLibraryCSTrainingTest\CNTKLibraryCSTrainingTest.standard.csproj", "{0DF2109B-BB85-4718-82DE-1C0536D4F2C3}" ProjectSection(ProjectDependencies) = postProject {7B7A563D-AA8E-4660-A805-D50235A02120} = {7B7A563D-AA8E-4660-A805-D50235A02120} {91973E60-A7BE-4C86-8FDB-59C88A0B3715} = {91973E60-A7BE-4C86-8FDB-59C88A0B3715} @@ -1635,6 +1635,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CNTKLibraryManagedDll", "bi EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "V2LibraryCSTests", "Tests\UnitTests\V2LibraryCSTests\V2LibraryCSTests.csproj", "{16B2571F-CFD9-490F-AAC0-04089C6EF576}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "bindings", "bindings", "{AC472411-780E-40F2-93AF-B703E9B65264}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CNTKLibraryCSTrainingTest", "Tests\EndToEndTests\CNTKv2CSharp\CNTKLibraryCSTrainingTest\CNTKLibraryCSTrainingTest.csproj", "{4F2971CD-021F-443A-95F9-2AEAABDFD5F8}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug_CpuOnly|x64 = Debug_CpuOnly|x64 @@ -2280,7 +2284,7 @@ Global {0DF2109B-BB85-4718-82DE-1C0536D4F2C3}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 {0DF2109B-BB85-4718-82DE-1C0536D4F2C3}.Release_NoOpt|x64.ActiveCfg = Release_NoOpt|x64 {0DF2109B-BB85-4718-82DE-1C0536D4F2C3}.Release_NoOpt|x64.Build.0 = Release_NoOpt|x64 - {0DF2109B-BB85-4718-82DE-1C0536D4F2C3}.Release_UWP|x64.ActiveCfg = Release_NoOpt|x64 + {0DF2109B-BB85-4718-82DE-1C0536D4F2C3}.Release_UWP|x64.ActiveCfg = Release_CpuOnly|x64 {0DF2109B-BB85-4718-82DE-1C0536D4F2C3}.Release|x64.ActiveCfg = Release|x64 {0DF2109B-BB85-4718-82DE-1C0536D4F2C3}.Release|x64.Build.0 = Release|x64 {91EA9F28-B9B6-4FC7-A47D-9838F5915700}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 @@ -2347,6 +2351,18 @@ Global {16B2571F-CFD9-490F-AAC0-04089C6EF576}.Release_UWP|x64.ActiveCfg = Release_CpuOnly|x64 {16B2571F-CFD9-490F-AAC0-04089C6EF576}.Release|x64.ActiveCfg = Release|x64 {16B2571F-CFD9-490F-AAC0-04089C6EF576}.Release|x64.Build.0 = Release|x64 + {4F2971CD-021F-443A-95F9-2AEAABDFD5F8}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 + {4F2971CD-021F-443A-95F9-2AEAABDFD5F8}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 + {4F2971CD-021F-443A-95F9-2AEAABDFD5F8}.Debug_UWP|x64.ActiveCfg = Debug_CpuOnly|x64 + {4F2971CD-021F-443A-95F9-2AEAABDFD5F8}.Debug|x64.ActiveCfg = Debug|x64 + {4F2971CD-021F-443A-95F9-2AEAABDFD5F8}.Debug|x64.Build.0 = Debug|x64 + {4F2971CD-021F-443A-95F9-2AEAABDFD5F8}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64 + {4F2971CD-021F-443A-95F9-2AEAABDFD5F8}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 + {4F2971CD-021F-443A-95F9-2AEAABDFD5F8}.Release_NoOpt|x64.ActiveCfg = Release_NoOpt|x64 + {4F2971CD-021F-443A-95F9-2AEAABDFD5F8}.Release_NoOpt|x64.Build.0 = Release_NoOpt|x64 + {4F2971CD-021F-443A-95F9-2AEAABDFD5F8}.Release_UWP|x64.ActiveCfg = Release_CpuOnly|x64 + {4F2971CD-021F-443A-95F9-2AEAABDFD5F8}.Release|x64.ActiveCfg = Release|x64 + {4F2971CD-021F-443A-95F9-2AEAABDFD5F8}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -2494,7 +2510,7 @@ Global {80B83483-CCDC-4E59-AB54-B6495E967A11} = {77EAB92C-CDCB-40D2-BD19-6C8C6C876B51} {784A839C-762F-4A85-9EF1-A1E00546AD6C} = {D30B34AF-3618-4C55-900E-8F60A9F39E66} {6730F9BE-92AA-45F7-9F98-CD13E725CCA9} = {784A839C-762F-4A85-9EF1-A1E00546AD6C} - {1526F027-B007-472D-82E2-5A91340F3B62} = {DD043083-71A4-409A-AA91-F9C548DCF7EC} + {1526F027-B007-472D-82E2-5A91340F3B62} = {AC472411-780E-40F2-93AF-B703E9B65264} {2A95B23C-D91E-4DF9-B8F0-5E997608AB65} = {47755F2E-D674-4175-9E38-8EA053455072} {FB604F98-008F-45CD-B06E-42C30E121F13} = {2A95B23C-D91E-4DF9-B8F0-5E997608AB65} {5EDBCD1A-4F07-4618-84C9-FC6905A438B4} = {FB604F98-008F-45CD-B06E-42C30E121F13} @@ -2573,6 +2589,8 @@ Global {2ECE5AEB-F471-4A1D-9BAD-963D5C8A8A1D} = {DD043083-71A4-409A-AA91-F9C548DCF7EC} {3A09FFE0-7865-4268-8301-73ED64BB75DF} = {1526F027-B007-472D-82E2-5A91340F3B62} {16B2571F-CFD9-490F-AAC0-04089C6EF576} = {6F19321A-65E7-4829-B00C-3886CD6C6EDE} + {AC472411-780E-40F2-93AF-B703E9B65264} = {DD043083-71A4-409A-AA91-F9C548DCF7EC} + {4F2971CD-021F-443A-95F9-2AEAABDFD5F8} = {B3B46744-DBB5-42C2-BAD7-9151D9486045} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9FF5B559-FC1B-4694-963F-355905287887} diff --git a/Examples/TrainingCSharp/Common/TransferLearning.cs b/Examples/TrainingCSharp/Common/TransferLearning.cs index 6532234f2d56..2875deef5ce9 100644 --- a/Examples/TrainingCSharp/Common/TransferLearning.cs +++ b/Examples/TrainingCSharp/Common/TransferLearning.cs @@ -139,7 +139,7 @@ public static void TrainAndEvaluateWithAnimalData(DeviceDescriptor device, bool string predictionNodeName = "prediction"; Variable imageInput, labelInput; Function trainingLoss, predictionError; - Function transferLearningModel = CreateTransferLearningModel(BaseResnetModelFile, featureNodeName, predictionNodeName, + Function transferLearningModel = CreateTransferLearningModel(Path.Combine(ExampleImageFoler, BaseResnetModelFile), featureNodeName, predictionNodeName, lastHiddenNodeName, animalModelNumClasses, device, out imageInput, out labelInput, out trainingLoss, out predictionError); diff --git a/Makefile b/Makefile index d6c28bb4c00a..315e76fe1261 100644 --- a/Makefile +++ b/Makefile @@ -113,6 +113,7 @@ ALL:= ALL_LIBS:= PYTHON_LIBS:= JAVA_LIBS:= +CSHARP_LIBS:= LIBS_FULLPATH:= SRC:= @@ -333,6 +334,7 @@ PERF_PROFILER_LIB:= $(LIBDIR)/lib$(PERF_PROFILER).so ALL_LIBS += $(PERF_PROFILER_LIB) PYTHON_LIBS += $(PERF_PROFILER_LIB) JAVA_LIBS += $(PERF_PROFILER_LIB) +CSHARP_LIBS += $(PERF_PROFILER_LIB) SRC += $(PP_SRC) $(PERF_PROFILER_LIB): $(PP_OBJ) @@ -433,6 +435,7 @@ CNTKMATH_LIB:= $(LIBDIR)/lib$(CNTKMATH).so ALL_LIBS += $(CNTKMATH_LIB) PYTHON_LIBS += $(CNTKMATH_LIB) JAVA_LIBS += $(CNTKMATH_LIB) +CSHARP_LIBS += $(CNTKMATH_LIB) SRC+=$(MATH_SRC) $(CNTKMATH_LIB): $(MATH_OBJ) | $(PERF_PROFILER_LIB) @@ -568,6 +571,7 @@ CNTKLIBRARY_LIB:=$(LIBDIR)/lib$(CNTKLIBRARY).so ALL_LIBS+=$(CNTKLIBRARY_LIB) PYTHON_LIBS+=$(CNTKLIBRARY_LIB) JAVA_LIBS+=$(CNTKLIBRARY_LIB) +CSHARP_LIBS+=$(CNTKLIBRARY_LIB) SRC+=$(CNTKLIBRARY_SRC) $(CNTKLIBRARY_LIB): $(CNTKLIBRARY_OBJ) | $(CNTKMATH_LIB) @@ -1554,6 +1558,50 @@ ALL += java endif +######################################## +# C# Support +######################################## +ifeq ("$(CSHARP_SUPPORT)","true") + +# This is a short-term hack to shoehorn cmake into our build system. In the near future, we will fully migrate +# to a cmake-based system and this hack will no longer be necessary. + +ifeq ("$(BUILDTYPE)","debug") + CSHARP_BUILDTYPE:=Debug +endif +ifeq ("$(BUILDTYPE)","release") + CSHARP_BUILDTYPE:=Release +endif + +.PHONY: csharp +csharp: $(CSHARP_LIBS) + @echo $(SEPARATOR) + @echo creating $@ for $(ARCH) with build type $(CSHARP_BUILDTYPE) + mkdir -p bindings/csharp/Swig/build/Linux/$(CSHARP_BUILDTYPE) + cd bindings/csharp/Swig/build/Linux/$(CSHARP_BUILDTYPE) && \ + cmake ../../.. -DCNTK_VERSION=$(BUILD_VERSION) -DCMAKE_BUILD_TYPE=$(CSHARP_BUILDTYPE) && \ + make + mkdir -p bindings/csharp/CNTKLibraryManagedDll/build/Linux/$(CSHARP_BUILDTYPE) + cd bindings/csharp/CNTKLibraryManagedDll/build/Linux/$(CSHARP_BUILDTYPE) && \ + cmake ../../.. -DCNTK_VERSION=$(BUILD_VERSION) -DCMAKE_BUILD_TYPE=$(CSHARP_BUILDTYPE) && \ + make + cp --recursive bindings/csharp/CNTKLibraryManagedDll/build/Linux/$(CSHARP_BUILDTYPE)/AnyCPU/$(CSHARP_BUILDTYPE)/* $(LIBDIR) + +ALL += csharp + +# Note that CMakeLists.txt has not been created for this project yet. The paths created here are really ugly. +V2LibraryCSTests.dll: csharp + @echo $(SEPARATOR) + @echo creating $@ for $(ARCH) with build type $(CSHARP_BUILDTYPE) + cd Tests/UnitTests/V2LibraryCSTests && \ + mkdir -p build/Linux/$(CSHARP_BUILDTYPE) && \ + dotnet build /p:OutDirPrefix=build/Linux/$(CSHARP_BUILDTYPE) /p:PlatformName=Linux -c $(CSHARP_BUILDTYPE) + cp $(LIBDIR)/*.so Tests/UnitTests/V2LibraryCSTests/build/Linux/$(CSHARP_BUILDTYPE)/AnyCPU/$(CSHARP_BUILDTYPE) + +ALL += V2LibraryCSTests.dll + +endif + ######################################## # General compile and dependency rules ######################################## diff --git a/Tests/EndToEndTests/CNTKv2CSharp/CNTKLibraryCSTrainingTest/CNTKLibraryCSTrainingTest.csproj b/Tests/EndToEndTests/CNTKv2CSharp/CNTKLibraryCSTrainingTest/CNTKLibraryCSTrainingTest.csproj index 199392d844c8..83551cc4de06 100644 --- a/Tests/EndToEndTests/CNTKv2CSharp/CNTKLibraryCSTrainingTest/CNTKLibraryCSTrainingTest.csproj +++ b/Tests/EndToEndTests/CNTKv2CSharp/CNTKLibraryCSTrainingTest/CNTKLibraryCSTrainingTest.csproj @@ -1,97 +1,68 @@ - - - - + + + - Debug - x64 - {0DF2109B-BB85-4718-82DE-1C0536D4F2C3} - $(IntDir)obj\ - $(IntDir)obj\ - $(OutDir) - Exe + ..\..\..\.. + + + + netcoreapp2.0 + CNTK.CNTKLibraryCSTrainingTest + + false + + $(CntkVersion) + + Microsoft Corporation + + Copyright © $([System.DateTime]::Now.ToString(`yyyy`)) + + $(OutDirPrefix)\$(Platform)\$(Configuration) + + x64 + + Debug;Debug_CpuOnly;Release;Release_CpuOnly;Release_NoOpt x64 - 4 - false + portable + true - MinimumRecommendedRules.ruleset - prompt - Properties - CNTK.CNTKLibraryCSTrainingTest - CNTKLibraryCSTrainingTest - v4.6.1 - 512 - - true - - - true - full - false - - - pdbonly - false - - - - DEBUG;TRACE - - - TRACE;DEBUG;CPUONLY + + + Exe + - - true + + TRACE + false + true - - true - TRACE;CPUONLY + + + TRACE;DEBUG_CPUONLY;NETCOREAPP2_0;CPUONLY - - false - TRACE + + + TRACE;RELEASE_CPUONLY;NETCOREAPP2_0;CPUONLY + - - - - {3A09FFE0-7865-4268-8301-73ED64BB75DF} - CNTKLibraryManagedDll - - - - - CNTKImageProcessing.cs - - - CifarResNetClassifier.cs - - - LogisticRegression.cs - - - LSTMSequenceClassifier.cs - - - MNISTClassifier.cs - - - TestHelper.cs - - - TransferLearning.cs - - - - + + + + + + + + + + + - + + + - - - - - - \ No newline at end of file + + diff --git a/Tests/EndToEndTests/CNTKv2CSharp/CNTKLibraryCSTrainingTest/CNTKLibraryCSTrainingTest.standard.csproj b/Tests/EndToEndTests/CNTKv2CSharp/CNTKLibraryCSTrainingTest/CNTKLibraryCSTrainingTest.standard.csproj new file mode 100644 index 000000000000..8d899aa0cb85 --- /dev/null +++ b/Tests/EndToEndTests/CNTKv2CSharp/CNTKLibraryCSTrainingTest/CNTKLibraryCSTrainingTest.standard.csproj @@ -0,0 +1,97 @@ + + + + + + Debug + x64 + {0DF2109B-BB85-4718-82DE-1C0536D4F2C3} + $(IntDir)obj\ + $(IntDir)obj\ + $(OutDir) + Exe + x64 + 4 + false + true + MinimumRecommendedRules.ruleset + prompt + Properties + CNTK.CNTKLibraryCSTrainingTest + CNTKLibraryCSTrainingTest.standard + v4.6.1 + 512 + + true + + + true + full + false + + + pdbonly + false + + + + DEBUG;TRACE + + + TRACE;DEBUG;CPUONLY + + + true + TRACE + + + true + TRACE;CPUONLY + + + false + TRACE + + + + + + {3A09FFE0-7865-4268-8301-73ED64BB75DF} + CNTKLibraryManagedDll + + + + + CNTKImageProcessing.cs + + + CifarResNetClassifier.cs + + + LogisticRegression.cs + + + LSTMSequenceClassifier.cs + + + MNISTClassifier.cs + + + TestHelper.cs + + + TransferLearning.cs + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Tests/EndToEndTests/CNTKv2CSharp/CNTKLibraryCSTrainingTest/Program.cs b/Tests/EndToEndTests/CNTKv2CSharp/CNTKLibraryCSTrainingTest/Program.cs index cc298bcb27a2..6f0deef01369 100644 --- a/Tests/EndToEndTests/CNTKv2CSharp/CNTKLibraryCSTrainingTest/Program.cs +++ b/Tests/EndToEndTests/CNTKv2CSharp/CNTKLibraryCSTrainingTest/Program.cs @@ -22,7 +22,6 @@ static void Main(string[] args) #else Console.WriteLine("======== Train model using GPU build ========"); #endif - List devices = new List(); if (ShouldRunOnCpu()) { @@ -35,7 +34,6 @@ static void Main(string[] args) string runTest = args.Length == 0 ? string.Empty : args[0]; - foreach (var device in devices) { /// Data folders of example classes are set for non-CNTK test runs. @@ -47,34 +45,48 @@ static void Main(string[] args) LogisticRegression.TrainAndEvaluate(device); break; case "SimpleFeedForwardClassifierTest": - SimpleFeedForwardClassifierTest.DataFolder = "."; Console.WriteLine($"======== running SimpleFeedForwardClassifierTest.TrainSimpleFeedForwardClassifier using {device.Type} ========"); SimpleFeedForwardClassifierTest.TrainSimpleFeedForwardClassifier(device); break; case "CifarResNetClassifierTest": - CifarResNetClassifier.CifarDataFolder = "./cifar-10-batches-py"; Console.WriteLine($"======== running CifarResNet.TrainAndEvaluate using {device.Type} ========"); + + if (args.Length > 1) + { + Console.WriteLine($"-------- running with test data in {args[1]} --------"); + CifarResNetClassifier.CifarDataFolder = args[1]; + } + CifarResNetClassifier.TrainAndEvaluate(device, true); break; case "LSTMSequenceClassifierTest": - LSTMSequenceClassifier.DataFolder = "../../../Text/SequenceClassification/Data"; Console.WriteLine($"======== running LSTMSequenceClassifier.Train using {device.Type} ========"); LSTMSequenceClassifier.Train(device); break; case "MNISTClassifierTest": - MNISTClassifier.ImageDataFolder = "../../../Image/Data/"; Console.WriteLine($"======== running MNISTClassifier.TrainAndEvaluate with Convnet using {device.Type} ========"); MNISTClassifier.TrainAndEvaluate(device, true, true); break; case "TransferLearningTest": - TransferLearning.ExampleImageFoler = "."; TransferLearning.BaseResnetModelFile = "ResNet_18.model"; Console.WriteLine($"======== running TransferLearning.TrainAndEvaluate with animal data using {device.Type} ========"); + + if (args.Length > 1) + { + Console.WriteLine($"-------- running with test data in {args[1]} --------"); + TransferLearning.ExampleImageFoler = args[1]; + } + TransferLearning.TrainAndEvaluateWithAnimalData(device, true); break; - default: + + case "": RunAllExamples(device); break; + + default: + Console.WriteLine("'{0}' is not a valid test name.", runTest); + break; } } diff --git a/Tests/EndToEndTests/CNTKv2CSharp/ExampleTests/CifarResNetClassifierTest/run-test b/Tests/EndToEndTests/CNTKv2CSharp/ExampleTests/CifarResNetClassifierTest/run-test index e5d750860c8d..b9c4568a6603 100644 --- a/Tests/EndToEndTests/CNTKv2CSharp/ExampleTests/CifarResNetClassifierTest/run-test +++ b/Tests/EndToEndTests/CNTKv2CSharp/ExampleTests/CifarResNetClassifierTest/run-test @@ -16,22 +16,23 @@ if [ "$OS" == "Windows_NT" ]; then DataSourceDir=`cygpath -au $CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY`/Image/CIFAR/v0 cp -R $DataSourceDir/cifar-10-batches-py $TestDataDir || exit $? - + # Set CUDA_VISIBLE_DEVICES to exclude all gpu if running on cpu device [ "$TEST_DEVICE" == "cpu" ] && export CUDA_VISIBLE_DEVICES=-1 - pushd $TestDataDir + pushd $TestDataDir + + pushd $TEST_BIN_DIR > /dev/null + dotnet CNTKLibraryCSTrainingTest.dll CifarResNetClassifierTest `cygpath -aw $TestDataDir/cifar-10-batches-py` + popd > /dev/null + + # Delete the test data + popd + rm -rf $TestDataDir + + exit 0 - $TEST_BIN_DIR/CNTKLibraryCSTrainingTest.exe CifarResNetClassifierTest else echo Cannot run CNTKLibraryCSTrainingTest on Linux. exit 1 fi - -ExitCode=$? - -# Delete the test data -popd -rm -rf $TestDataDir - -exit $ExitCode diff --git a/Tests/EndToEndTests/CNTKv2CSharp/ExampleTests/LSTMSequenceClassifierTest/run-test b/Tests/EndToEndTests/CNTKv2CSharp/ExampleTests/LSTMSequenceClassifierTest/run-test index d1eaa6ff2e9f..8f954196c63a 100644 --- a/Tests/EndToEndTests/CNTKv2CSharp/ExampleTests/LSTMSequenceClassifierTest/run-test +++ b/Tests/EndToEndTests/CNTKv2CSharp/ExampleTests/LSTMSequenceClassifierTest/run-test @@ -3,7 +3,9 @@ . $TEST_ROOT_DIR/run-test-common if [ "$OS" == "Windows_NT" ]; then - $TEST_BIN_DIR/CNTKLibraryCSTrainingTest.exe LSTMSequenceClassifierTest + pushd $TEST_BIN_DIR > /dev/null + dotnet CNTKLibraryCSTrainingTest.dll LSTMSequenceClassifierTest + popd > /dev/null else echo Cannot run CNTKLibraryCSTrainingTest on Linux. exit 1 diff --git a/Tests/EndToEndTests/CNTKv2CSharp/ExampleTests/LogisticRegressionTest/run-test b/Tests/EndToEndTests/CNTKv2CSharp/ExampleTests/LogisticRegressionTest/run-test index 2f9809deb5df..80b19fe940c9 100644 --- a/Tests/EndToEndTests/CNTKv2CSharp/ExampleTests/LogisticRegressionTest/run-test +++ b/Tests/EndToEndTests/CNTKv2CSharp/ExampleTests/LogisticRegressionTest/run-test @@ -3,7 +3,9 @@ . $TEST_ROOT_DIR/run-test-common if [ "$OS" == "Windows_NT" ]; then - $TEST_BIN_DIR/CNTKLibraryCSTrainingTest.exe LogisticRegressionTest + pushd $TEST_BIN_DIR > /dev/null + dotnet CNTKLibraryCSTrainingTest.dll LogisticRegressionTest + popd > /dev/null else echo Cannot run CNTKLibraryCSTrainingTest on Linux. exit 1 diff --git a/Tests/EndToEndTests/CNTKv2CSharp/ExampleTests/MNISTClassifierTest/run-test b/Tests/EndToEndTests/CNTKv2CSharp/ExampleTests/MNISTClassifierTest/run-test index 5b40de43f434..53e09f4f70ad 100644 --- a/Tests/EndToEndTests/CNTKv2CSharp/ExampleTests/MNISTClassifierTest/run-test +++ b/Tests/EndToEndTests/CNTKv2CSharp/ExampleTests/MNISTClassifierTest/run-test @@ -3,7 +3,9 @@ . $TEST_ROOT_DIR/run-test-common if [ "$OS" == "Windows_NT" ]; then - $TEST_BIN_DIR/CNTKLibraryCSTrainingTest.exe MNISTClassifierTest + pushd $TEST_BIN_DIR > /dev/null + dotnet CNTKLibraryCSTrainingTest.dll MNISTClassifierTest + popd > /dev/null else echo Cannot run CNTKLibraryCSTrainingTest on Linux. exit 1 diff --git a/Tests/EndToEndTests/CNTKv2CSharp/ExampleTests/SimpleFeedForwardClassifierTest/run-test b/Tests/EndToEndTests/CNTKv2CSharp/ExampleTests/SimpleFeedForwardClassifierTest/run-test index 7a8697b3756b..f28378fdc28b 100644 --- a/Tests/EndToEndTests/CNTKv2CSharp/ExampleTests/SimpleFeedForwardClassifierTest/run-test +++ b/Tests/EndToEndTests/CNTKv2CSharp/ExampleTests/SimpleFeedForwardClassifierTest/run-test @@ -4,7 +4,10 @@ if [ "$OS" == "Windows_NT" ]; then cp $TEST_DIR/../../../../../Tests/EndToEndTests/Simple2d/Data/SimpleDataTrain_cntk_text.txt $TEST_DIR || exit $? - $TEST_BIN_DIR/CNTKLibraryCSTrainingTest.exe SimpleFeedForwardClassifierTest + + pushd $TEST_BIN_DIR > /dev/null + dotnet CNTKLibraryCSTrainingTest.dll SimpleFeedForwardClassifierTest + popd > /dev/null else echo Cannot run CNTKLibraryCSTrainingTest on Linux. exit 1 diff --git a/Tests/EndToEndTests/CNTKv2CSharp/ExampleTests/TransferLearningTest/run-test b/Tests/EndToEndTests/CNTKv2CSharp/ExampleTests/TransferLearningTest/run-test index 31cdf330133b..64d1c7daa0f5 100644 --- a/Tests/EndToEndTests/CNTKv2CSharp/ExampleTests/TransferLearningTest/run-test +++ b/Tests/EndToEndTests/CNTKv2CSharp/ExampleTests/TransferLearningTest/run-test @@ -11,7 +11,7 @@ if [ "$OS" == "Windows_NT" ]; then fi TestDataDir=$TEST_RUN_DIR/TestData - mkdir $TestDataDir + mkdir -p $TestDataDir # Copy and extract animal data to the test run directory DataSourceDir=`cygpath -au $CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY`/Image @@ -23,18 +23,19 @@ if [ "$OS" == "Windows_NT" ]; then ModelSource=`cygpath -au $CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY`/PreTrainedModels/ResNet/v1/ResNet_18.model cp $ModelSource $TestDataDir || exit $? - pushd $TestDataDir - - $TEST_BIN_DIR/CNTKLibraryCSTrainingTest.exe TransferLearningTest + pushd $TestDataDir + + pushd $TEST_BIN_DIR > /dev/null + dotnet CNTKLibraryCSTrainingTest.dll TransferLearningTest `cygpath -aw $TestDataDir` + popd > /dev/null + + # Delete the test data + popd + rm -rf $TestDataDir + + exit 0 + else echo Cannot run CNTKLibraryCSTrainingTest on Linux. exit 1 fi - -ExitCode=$? - -# Delete the test data -popd -rm -rf $TestDataDir - -exit $ExitCode diff --git a/Tests/UnitTests/V2LibraryCSTests/V2LibraryCSTests.csproj b/Tests/UnitTests/V2LibraryCSTests/V2LibraryCSTests.csproj index b64313b2ac61..c9989b99ac72 100644 --- a/Tests/UnitTests/V2LibraryCSTests/V2LibraryCSTests.csproj +++ b/Tests/UnitTests/V2LibraryCSTests/V2LibraryCSTests.csproj @@ -1,5 +1,9 @@ - + + + + ..\..\.. + netcoreapp2.0 @@ -13,12 +17,14 @@ Copyright © $([System.DateTime]::Now.ToString(`yyyy`)) - ..\..\..\x64\$(Configuration) - + $(OutDirPrefix)\$(Platform)\$(Configuration) + x64 Debug;Debug_CpuOnly;Release;Release_CpuOnly;Release_NoOpt x64 + portable + true @@ -26,7 +32,6 @@ TRACE false - pdbonly diff --git a/Tools/cmake/cntk_common.cmake b/Tools/cmake/cntk_common.cmake new file mode 100644 index 000000000000..90ad6a11f64e --- /dev/null +++ b/Tools/cmake/cntk_common.cmake @@ -0,0 +1,48 @@ +set(CNTK_VERSION 2.5.1) + +# ---------------------------------------------------------------------- +# UnsupportedComponentError +# Issues a fatal error when attempting to generate projects for use in unverified environments. +# +function(UnsupportedComponentError value component) + message(FATAL_ERROR + +"CNTK has not been verified with the version '${value}' of the ${component}.\n\ +\ +However, this does not mean that CNTK doesn't work with this component. If you are feeling adventureous, you can add '${value}' to the list of supported ${component} items, build and test CNTK, and report your findings to https://github.com/Microsoft/CNTK/issues (we welcome contributions from the community!). If you believe that this component should be supported, please create an issue at https://github.com/Microsoft/CNTK/issues. +" + ) +endfunction() + +# ---------------------------------------------------------------------- +if(MSVC) + set(CNTK_SUPPORTED_PLATFORMS + x64; + ) + + if(${CMAKE_VS_PLATFORM_NAME} IN_LIST CNTK_SUPPORTED_PLATFORMS) + # There doesn't seem to be a NOT operator, thus the wonky syntax + else() + UnsupportedComponentError(${CMAKE_VS_PLATFORM_NAME} "development platform") + endif() + + set(CNTK_SUPPORTED_MSVC_VERSIONS + 1911; # Visual Studio 2017 version 15.3 + ) + + if(${MSVC_VERSION} IN_LIST CNTK_SUPPORTED_MSVC_VERSIONS) + # There doesn't seem to be a NOT operator, thus the wonky syntax + else() + UnsupportedComponentError(${MSVC_VERSION} "Visual Studio Compiler") + endif() + + set(CNTK_SUPPORTED_WINDOWS_SKDS + 10.0.17134.0; # Windows 10 SDK for April 2018 Update, version 1803 + ) + + if(${CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION} IN_LIST CNTK_SUPPORTED_WINDOWS_SKDS) + # There doesn't seem to be a NOT operator, thus the wonky syntax + else() + UnsupportedComponentError(${CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION} "Windows SDK") + endif() +endif() \ No newline at end of file diff --git a/Tools/cmake/common.cmake b/Tools/cmake/common.cmake new file mode 100644 index 000000000000..fc86a18f0869 --- /dev/null +++ b/Tools/cmake/common.cmake @@ -0,0 +1,37 @@ +# ---------------------------------------------------------------------- +# EnsureProperties +# Validates that all properties have been defined. +# +function(EnsureProperties) + foreach(property ${ARGN}) + if(NOT DEFINED "${property}") + message(FATAL_ERROR + +"'${property}' is not defined.\n\ +\ +This property must be defined to correctly generate content. There is a configuration problem if you see this error when generating the core CNTK project. If you are generating this file in isolation, you can add the entry within the cmake GUI or specify it on the cmake command line with -Dvar=value. +" + ) + endif() + endforeach() +endfunction() + +# ---------------------------------------------------------------------- +# EnsureTools +# Validates that all tools are available. This is a macro to ensure that the property '_binary' is available within +# the namespace of calling scripts. +# +macro(EnsureTools) + foreach(tool ${ARGN}) + find_program(${tool}_binary ${tool}) + if(${tool}_binary STREQUAL ${tool}_binary-NOTFOUND) + message(FATAL_ERROR + +"The tool '${tool}' was not found.\n\ +\ +Make sure that this tool is available in a directory available via the 'PATH' environment variable and run cmake again. +" + ) + endif() + endforeach() +endmacro() diff --git a/Tools/cmake/cpp_common.cmake b/Tools/cmake/cpp_common.cmake new file mode 100644 index 000000000000..8dba734727ab --- /dev/null +++ b/Tools/cmake/cpp_common.cmake @@ -0,0 +1,197 @@ +# This file contains definitions for C++ settings that impact the Application Binary Interface (ABI) that are +# generally applicable across a variety of different projects. Project-specific customizations should be defined +# in a separate file. + +option( + CODE_COVERAGE + "Produce builds that can be used to extract code coverage information." + "OFF" +) + +option( + SUPPORT_AVX2 + "Produce builds that support Advanced Vector Extensions." + "OFF" +) + +set(CMAKE_CONFIGURATION_TYPES Debug;Release;Release_NoOpt) + +# If CMAKE_BUILD_TYPE is defined, we are looking at a single configuration and can extract information directly from this value. +# If it is not defined, then we need to use a dynamic generator. +if(DEFINED CMAKE_BUILD_TYPE AND NOT CMAKE_BUILD_TYPE STREQUAL "") + set(config "${CMAKE_BUILD_TYPE}") + string(COMPARE EQUAL "${config}" "Debug" is_debug) + string(COMPARE EQUAL "${config}" "Release_NoOpt" is_release_noopt) + +elseif(CMAKE_GENERATOR STREQUAL "Unix Makefiles") + # This code isn't quite right. What we actually want to do is say that CMAKE_BUILD_TYPE must be defined + # for any generator that doesn't support the use of CONFIG as a generator expression. I haven't found a way + # to detect that in the cmake logic. + message(FATAL_ERROR "'CMAKE_BUILD_TYPE' must be defined") + +else() + set(config $) + set(is_debug $) + set(is_release_noopt $) + +endif() + +if(MSVC) + # ---------------------------------------------------------------------- + # | + # | Microsoft Visual Studio + # | + # ---------------------------------------------------------------------- + + # ---------------------------------------------------------------------- + # | Preprocessor Definitions + add_definitions( + -DUNICODE; # Use UNICODE character set + -D_UNICODE; # Use UNICODE character set + ) + + # Debug-specific flags + foreach(definition + -DDEBUG # Debug mode + -D_DEBUG # Debug mode + ) + # Note that add_definitions isn't able to handle generator expressions, so we + # have to do the generation manually. + string(APPEND CMAKE_CXX_FLAGS_DEBUG " ${definition}") + endforeach() + + # ---------------------------------------------------------------------- + # | Compiler Flags + add_compile_options( # Option + # ------------------------------------ + /bigobj # Increase number of sections in object file + /fp:except- # Enable Floating Point Exceptions: No + /fp:fast # Floating Point Model: Fast + /GR # Run Time Type Information (RTTI) + /MP # Build with multiple processes + /openmp # OpenMP 2.0 Support + /sdl # Enable Additional Security Checks + /W4 # Warning level 4 + /WX # Warning as errors + ) + + # Option Debug Release Release_NoOpt + # ------------------------------------ ---------------------- ---------------------- ---------------------- + add_compile_options($<$:/Gy>) # Enable Function-Level Linking Yes Yes + add_compile_options($<$:/Oi>) # Enable Intrinsic Functions Yes Yes + add_compile_options($<$>:/Ot>) # Favor Size or Speed fast + add_compile_options($<$:/Qpar>) # Enable Parallel Code Generation Yes Yes + add_compile_options($) # Program Database Edit & Continue Standard Standard + + # ---------------------------------------------------------------------- + # | Linker Flags + if(${CODE_COVERAGE}) + foreach(linker_flag # Option + # ------------------------------------ + /PROFILE # Enable profiling + ) + string(APPEND CMAKE_EXE_LINKER_FLAGS " ${linker_flag}") + string(APPEND CMAKE_SHARED_LINKER_FLAGS " ${linker_flag}") + endforeach() + endif() + + # Release-specific linker flags + foreach(linker_flag + /DEBUG # Generate Debug Information + /OPT:ICF # Enable COMDAT Folding + /OPT:REF # References + ) + # Note that there isn't a cmake method called add_linker_options that is able to handle + # generator expressions, so we have to do the generation manually. + string(APPEND CMAKE_EXE_LINKER_FLAGS_RELEASE " ${linker_flag}") + string(APPEND CMAKE_SHARED_LINKER_FLAGS_RELEASE " ${linker_flag}") + endforeach() + +else() + # ---------------------------------------------------------------------- + # | + # | GCC(-like) + # | + # ---------------------------------------------------------------------- + + # ---------------------------------------------------------------------- + # | Preprocessor Definitions + add_definitions( + -D__USE_XOPEN2K + -D_POSIX_SOURCE + -D_XOPEN_SOURCE=600 + -DNO_SYNC + ) + + # Debug-specific flags + foreach(definition + -D_DEBUG # Debug mode + ) + # Note that add_definitions isn't able to handle generator expressions, so we + # have to do the generation manually. + string(APPEND CMAKE_CXX_FLAGS_DEBUG " ${definition}") + endforeach() + + # ---------------------------------------------------------------------- + # | Compiler Flags + add_compile_options( # Option + # ------------------------------------ + -fcheck-new # Check the return value of new in C++. + -fopenmp # Enable OpenMP + -fpermissive # Downgrade conformance errors to warnings. + -fPIC # Generate position-independent code if possible (large mode). + -msse4.1 # Support MMX, SSE, SSE2, SSE3, SSSE3 and SSE4.1 built-in functions and code generation. + -std=c++11 # Conform to the ISO 2011 C standard. + -Wall # Enable most warning messages. + -Wextra # Print extra (possibly unwanted) warnings. + # TODO: -Werror # Treat all warnings as errors. + ) + + if(${CODE_COVERAGE}) + add_compile_options( + -fprofile-arcs # Insert arc-based program profiling code. + -ftest-coverage # Create data files needed by "gcov". + ) + endif() + + if(${SUPPORT_AVX2}) + add_compile_options( + -mavx2 # Support MMX, SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2, AVX and AVX2 built-in functions and code generation. + ) + endif() + # Option Debug Release Release_NoOpt + # ------------------------------------ ---------------------- ---------------------- ---------------------- + add_compile_options($<$>:-O4>) # Set optimization level 4 + + # ---------------------------------------------------------------------- + # | Linker Flags + foreach(linker_flag # Option + # ------------------------------------ + # -rdynamic # ???? -dynamicbase? + ) + # Note that add_definitions isn't able to handle generator expressions, so we + # have to do the generation manually. + string(APPEND CMAKE_EXE_LINKER_FLAGS " ${linker_flag}") + string(APPEND CMAKE_SHARED_LINKER_FLAGS " ${linker_flag}") + endforeach() + + if(${CODE_COVERAGE}) + foreach(linker_flag + --coverage # ???? + -lgcov # Link with gcov libraries + + ) + # Note that add_definitions isn't able to handle generator expressions, so we + # have to do the generation manually. + string(APPEND CMAKE_EXE_LINKER_FLAGS " ${linker_flag}") + string(APPEND CMAKE_SHARED_LINKER_FLAGS " ${linker_flag}") + endforeach() + endif() + +endif() + +# Define the Release_NoOpt configuration in terms of Release. Differentiation between the configurations +# are handled in add_compile_options above. +set(CMAKE_CXX_FLAGS_RELEASE_NOOPT ${CMAKE_CXX_FLAGS_RELEASE}) +set(CMAKE_EXE_LINKER_FLAGS_RELEASE_NOOPT ${CMAKE_EXE_LINKER_FLAGS_RELEASE}) +set(CMAKE_SHARED_LINKER_FLAGS_RELEASE_NOOPT ${CMAKE_SHARED_LINKER_FLAGS_RELEASE}) diff --git a/Tools/cmake/csharp_common.cmake b/Tools/cmake/csharp_common.cmake new file mode 100644 index 000000000000..518e106fa56c --- /dev/null +++ b/Tools/cmake/csharp_common.cmake @@ -0,0 +1,35 @@ +# This file contains definitions for C# settings that impact code generation that are +# generically applicable across a variety of different projects. Project-specific customizations +# should be defined in a separate file. + +include(${CMAKE_CURRENT_LIST_DIR}/common.cmake REQUIRED) + +# Ensure that dotnet is available on Linux. Windows builds rely on MSBuild instead. +if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux") + EnsureTools(dotnet;) +elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows") + EnsureTools(dotnet;) +else() + message(FATAL_ERROR "The CMAKE_HOST_SYSTEM_NAME value of '${CMAKE_HOST_SYSTEM_NAME}' is not recognized.") +endif() + +set(CMAKE_CONFIGURATION_TYPES Debug;Release) + +# If CMAKE_BUILD_TYPE is defined, we are looking at a single configuration and can extract information directly from this value. +# If it is not defined, then we need to use a dynamic generator. +if(DEFINED CMAKE_BUILD_TYPE AND NOT CMAKE_BUILD_TYPE STREQUAL "") + set(config "${CMAKE_BUILD_TYPE}") + string(COMPARE EQUAL "${config}" "Debug" is_debug) + +elseif(CMAKE_GENERATOR STREQUAL "Unix Makefiles") + # This code isn't quite right. What we actually want to do is say that CMAKE_BUILD_TYPE must be defined + # for any generator that doesn't support the use of CONFIG as a generator expression. I haven't found a way + # to detect that in the cmake logic. + message(FATAL_ERROR "'CMAKE_BUILD_TYPE' must be defined") + +else() + set(config $) + set(is_debug $) + +endif() + diff --git a/Tools/cmake/debug.cmake b/Tools/cmake/debug.cmake new file mode 100644 index 000000000000..16ac404d18c7 --- /dev/null +++ b/Tools/cmake/debug.cmake @@ -0,0 +1,11 @@ +# Functionality to aid in the debugging of cmake and the cmake generation process. + +if(${CNTK_DEBUG_CMAKE}) + get_cmake_property(_variables VARIABLES) + list (SORT _variables) + foreach (_var ${_variables}) + message(STATUS "${_var}=${${_var}}") + endforeach() +endif() + +set(CMAKE_EXPORT_COMPILE_COMMANDS ${CNTK_EXPORT_COMPILE_COMMANDS}) \ No newline at end of file diff --git a/Tools/cmake/options.cmake b/Tools/cmake/options.cmake new file mode 100644 index 000000000000..90eb29fefa12 --- /dev/null +++ b/Tools/cmake/options.cmake @@ -0,0 +1,14 @@ +# Options that control cmake generation + +option( + CNTK_DEBUG_CMAKE + "Enable cmake-specific debug output" + "OFF" +) + +option( + CNTK_EXPORT_COMPILE_COMMANDS + "Enable the generation of compile commands to 'compile_commands.json'." + "OFF" +) + diff --git a/Tools/docker/CNTK-CPUOnly-Image/Dockerfile b/Tools/docker/CNTK-CPUOnly-Image/Dockerfile index 61f41e0d655d..dd6c657b001c 100644 --- a/Tools/docker/CNTK-CPUOnly-Image/Dockerfile +++ b/Tools/docker/CNTK-CPUOnly-Image/Dockerfile @@ -1,6 +1,9 @@ # CNTK Dockerfile # CPU only # No 1-bit SGD +# +# To build, run from the parent with the command line: +# docker build -t -f CNTK-CPUOnly-Image/Dockerfile . FROM ubuntu:16.04 @@ -39,6 +42,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ openjdk-8-jdk \ # For SWIG libpcre3-dev && \ + # .NET Core SDK + apt-transport-https && \ rm -rf /var/lib/apt/lists/* RUN OPENMPI_VERSION=1.10.3 && \ @@ -151,6 +156,20 @@ RUN cd /root && \ make -j $(nproc) && \ make install +COPY ./Patches /tmp/patches +RUN /tmp/patches/patch_swig.sh /usr/local/share/swig/3.0.10 && \ + rm -rfd /tmp/patches + +# .NET Core SDK +RUN cd /tmp && \ + (wget -q packages-microsoft-prod.deb https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb || true) && \ + dpkg -i packages-microsoft-prod.deb && \ + apt-get update && \ + apt-get --yes install dotnet-sdk-2.1.200 && \ + rm /tmp/packages-microsoft-prod.deb && \ + apt-get --yes autoremove && \ + rm -rf /var/lib/apt/lists/* + # Anaconda RUN wget -q https://repo.continuum.io/archive/Anaconda3-4.2.0-Linux-x86_64.sh && \ bash Anaconda3-4.2.0-Linux-x86_64.sh -b && \ diff --git a/Tools/docker/CNTK-GPU-Image/Dockerfile b/Tools/docker/CNTK-GPU-Image/Dockerfile index 49faec67345d..671138031c1e 100644 --- a/Tools/docker/CNTK-GPU-Image/Dockerfile +++ b/Tools/docker/CNTK-GPU-Image/Dockerfile @@ -5,6 +5,10 @@ # Label: com.nvidia.cuda.version: 9.0.176 # Label: com.nvidia.cudnn.version: 7.0.4.31 # Label: com.nvidia.nccl.version: 2.1.2 +# +# To build, run from the parent with the command line: +# docker build -t -f CNTK-GPU-Image/Dockerfile . + # Ubuntu 16.04.5 FROM nvidia/cuda@sha256:33add9c50ab76b8f3a92187c0418ed600d5bea27690fda40711122fdc28ce2f4 @@ -41,6 +45,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ openjdk-8-jdk \ # For SWIG libpcre3-dev && \ + # .NET Core SDK + apt-transport-https && \ rm -rf /var/lib/apt/lists/* RUN OPENMPI_VERSION=1.10.3 && \ @@ -155,6 +161,20 @@ RUN cd /root && \ make -j $(nproc) && \ make install +COPY ./Patches /tmp/patches +RUN /tmp/patches/patch_swig.sh /usr/local/share/swig/3.0.10 && \ + rm -rfd /tmp/patches + +# .NET Core SDK +RUN cd /tmp && \ + (wget -q packages-microsoft-prod.deb https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb || true) && \ + dpkg -i packages-microsoft-prod.deb && \ + apt-get update && \ + apt-get --yes install dotnet-sdk-2.1.200 && \ + rm /tmp/packages-microsoft-prod.deb && \ + apt-get --yes autoremove && \ + rm -rf /var/lib/apt/lists/* + # Anaconda RUN wget -q https://repo.continuum.io/archive/Anaconda3-4.2.0-Linux-x86_64.sh && \ bash Anaconda3-4.2.0-Linux-x86_64.sh -b && \ diff --git a/Tools/docker/Patches/Swig/std_wstring.i.patch b/Tools/docker/Patches/Swig/std_wstring.i.patch new file mode 100644 index 000000000000..48f0e439a635 --- /dev/null +++ b/Tools/docker/Patches/Swig/std_wstring.i.patch @@ -0,0 +1,40 @@ +--- std_wstring.i 2017-09-05 18:29:41.000000000 +0000 ++++ std_wstring.modified.i 2018-05-31 20:40:22.257638700 +0000 +@@ -13,6 +13,28 @@ + + %{ + #include ++ ++std::wstring UTF16ToWString(wchar_t const *str) ++{ ++ if(str == nullptr) ++ return std::wstring(); ++ ++ short const * pBegin(reinterpret_cast(str)); ++ short const * ptr(pBegin); ++ ++ while(*ptr != 0) ++ ++ptr; ++ ++ std::wstring result; ++ ++ result.reserve(ptr - pBegin); ++ ++ while(pBegin != ptr) ++ result.push_back(*pBegin++); ++ ++ return result; ++} ++ + %} + + namespace std { +@@ -71,7 +93,7 @@ + SWIG_CSharpSetPendingExceptionArgument(SWIG_CSharpArgumentNullException, "null wstring", 0); + return $null; + } +- std::wstring $1_str($input); ++ std::wstring $1_str(UTF16ToWString($input)); + $1 = &$1_str; %} + %typemap(out) const wstring & %{ $result = SWIG_csharp_wstring_callback($1->c_str()); %} + diff --git a/Tools/docker/Patches/Swig/wchar.i.patch b/Tools/docker/Patches/Swig/wchar.i.patch new file mode 100644 index 000000000000..03d256048d90 --- /dev/null +++ b/Tools/docker/Patches/Swig/wchar.i.patch @@ -0,0 +1,90 @@ +--- wchar.i 2017-09-05 18:29:41.000000000 +0000 ++++ wchar.modified.i 2018-05-31 20:36:25.778548300 +0000 +@@ -18,10 +18,36 @@ + %} + + %pragma(csharp) imclasscode=%{ +- protected class SWIGWStringHelper { ++ /* On Linux, C# strings are encoded as UTF8 rather than UTF16 as they are on Windows. In C++, wchar_t is 4 bytes on Linux and 2 bytes on Windows. On Windows, conversion between ++ a C++ wchar_t based-string and a C# string work because they are both the same size (UTF16 and sizeof(wchar_t) == 2). On Linux, we must convert the C++ string from wchar_t (4 bytes) ++ and UTF8 (multi-byte). ++ ++ Ideally, swig would peform this conversion for us. There is an issue opened on April 18, 2018 addressing this limitation: http://github.com/swig/swig/issues/1233. ++ */ ++ public class SWIGWStringHelperImpl { ++ public static string CreateUTF8String([global::System.Runtime.InteropServices.MarshalAs(global::System.Runtime.InteropServices.UnmanagedType.LPWStr)]global::System.IntPtr cString) { ++ int length = 0; ++ ++ while(global::System.Runtime.InteropServices.Marshal.ReadInt32(cString, length) != 0) ++ length += 4; ++ ++ if(length == 0) ++ return string.Empty; ++ ++ byte[] buffer = new byte[length]; ++ ++ global::System.Runtime.InteropServices.Marshal.Copy(cString, buffer, 0, buffer.Length); ++ ++ byte[] utf8buffer = global::System.Text.Encoding.Convert(global::System.Text.Encoding.UTF32, global::System.Text.Encoding.UTF8, buffer); ++ ++ return global::System.Text.Encoding.Default.GetString(utf8buffer); ++ } ++ }; ++ ++ protected class SWIGWStringHelper : SWIGWStringHelperImpl { + + public delegate string SWIGWStringDelegate(global::System.IntPtr message); +- static SWIGWStringDelegate wstringDelegate = new SWIGWStringDelegate(CreateWString); ++ static SWIGWStringDelegate wstringDelegate = new SWIGWStringDelegate(CreateUTF8String); + + [global::System.Runtime.InteropServices.DllImport("$dllimport", EntryPoint="SWIGRegisterWStringCallback_$module")] + public static extern void SWIGRegisterWStringCallback_$module(SWIGWStringDelegate wstringDelegate); +@@ -45,6 +71,25 @@ + SWIGEXPORT void SWIGSTDCALL SWIGRegisterWStringCallback_$module(SWIG_CSharpWStringHelperCallback callback) { + SWIG_csharp_wstring_callback = callback; + } ++ ++#include ++ ++// Converts from a UTF8 string into a std::wstring. ++std::wstring UTF8ToWString(const char *str) ++{ ++ if(str == nullptr) ++ return std::wstring(); ++ ++ // Note that this code ONLY WORKS for ASCII CHARS. Need to calculate the mblength for true UTF8. ++ size_t const length(strlen(str) + 1); ++ std::wstring buffer; ++ ++ buffer.resize(length); ++ ++ size_t const result(std::mbstowcs(const_cast(buffer.data()), str, buffer.size() * sizeof(std::wstring::value_type))); ++ return result != static_cast(-1) ? buffer : L""; ++} ++ + %} + #endif // SWIG_CSHARP_WSTRING_HELPER_ + #endif // SWIG_CSHARP_NO_WSTRING_HELPER +@@ -82,8 +127,8 @@ + + %typemap(csin) wchar_t * "$csinput" + %typemap(csout, excode=SWIGEXCODE) wchar_t * { +- string ret = global::System.Runtime.InteropServices.Marshal.PtrToStringUni($imcall);$excode +- return ret; ++ global::System.IntPtr cString = $imcall;$excode ++ return $imclassname.SWIGWStringHelperImpl.CreateUTF8String(cString); + } + %typemap(csvarin, excode=SWIGEXCODE2) wchar_t * %{ + set { +@@ -95,7 +140,10 @@ + return ret; + } %} + +-%typemap(in) wchar_t * %{ $1 = ($1_ltype)$input; %} ++%typemap(in) wchar_t * ++%{ std::wstring $1_buffer(UTF8ToWString(reinterpret_cast($input))); ++ $1 = const_cast($1_buffer.c_str()); %} ++ + %typemap(out) wchar_t * %{ $result = (wchar_t *)$1; %} + + %typemap(typecheck) wchar_t * = char *; diff --git a/Tools/docker/Patches/patch_swig.sh b/Tools/docker/Patches/patch_swig.sh new file mode 100644 index 000000000000..4b969631e8bd --- /dev/null +++ b/Tools/docker/Patches/patch_swig.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Swig doesn't handle C#/wchar_t conversions well on Linux; for more info, see http://github.com/swig/swig/issues/1233. This script will apply +# a minimal patch to Swig code for use with CNTK. These patches are a minimal set of changes and may not handle all cases. +# +# Patches were created with the command line: +# diff -u + +if [[ "$1" == "" ]] +then + echo "Usage: $0 " + exit -1 +fi + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +pushd $script_dir/Swig > /dev/null + +for patch_filename in $script_dir/Swig/*.patch; do + filename=$(basename -- "$patch_filename") + filename=${filename%.patch} + + patch --forward $1/csharp/$filename --input $patch_filename +done + + +popd > /dev/null diff --git a/bindings/csharp/CNTKLibraryManagedDll/CMakeLists.txt b/bindings/csharp/CNTKLibraryManagedDll/CMakeLists.txt new file mode 100644 index 000000000000..9bd9fba384a2 --- /dev/null +++ b/bindings/csharp/CNTKLibraryManagedDll/CMakeLists.txt @@ -0,0 +1,165 @@ +cmake_minimum_required (VERSION 3.8) + +include(../../../Tools/cmake/common.cmake REQUIRED) + +EnsureProperties(CNTK_VERSION;CMAKE_BUILD_TYPE) + +include(../../../Tools/cmake/csharp_common.cmake REQUIRED) + +ADD_SUBDIRECTORY(../Swig CSharpBindings) + +# ${source_filenames} (defined below) changes based on configuration. Unfortunately, some cmake generators don't support +# input based on generator expressions. In the name of supporting the least common denominator, restrict cmake generation +# to a single configuration. This is standard practice for makefiles but a bit unusual for Visual Studio. +if(${config} STREQUAL "Debug") + set(binary_name_debug_suffix d) +elseif(${config} STREQUAL "Release") + set(binary_name_debug_suffix "") +else() + message(FATAL_ERROR "'CMAKE_BUILD_TYPE' must be 'Debug' or 'Release' ('${CMAKE_BUILD_TYPE}' found)") +endif() + +set(CMAKE_CONFIGURATION_TYPES ${config}) + +set(binary_name_suffix -${CNTK_VERSION}${binary_name_debug_suffix}) +set(binary_name Cntk.Core.CSBinding${binary_name_suffix}) + +set(source_filenames + Helper.cs + Utils.cs + + ShimApiClasses/AxisShim.cs + ShimApiClasses/CNTKLibShim.cs + ShimApiClasses/ConstantShim.cs + ShimApiClasses/DeviceDescriptorShim.cs + ShimApiClasses/FunctionShim.cs + ShimApiClasses/LearnerShim.cs + ShimApiClasses/MinibatchSourceConfigShim.cs + ShimApiClasses/MinibatchSourceShim.cs + ShimApiClasses/NDArrayViewShim.cs + ShimApiClasses/NDMaskShim.cs + ShimApiClasses/NDShapeShim.cs + ShimApiClasses/ParameterShim.cs + ShimApiClasses/StreamConfigurationShim.cs + ShimApiClasses/TrainerShim.cs + ShimApiClasses/ValueShim.cs + ShimApiClasses/VariableShim.cs + + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/AdditionalLearningOptions.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/Axis.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/AxisVector.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/BoolVector.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/CharVector.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/CNTKDictionary.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/CNTKLib.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/CNTKLibPINVOKE.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/Constant.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/ConstantVector.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/DataType.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/DataUnit.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/DeviceDescriptor.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/DeviceDescriptorVector.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/DeviceKind.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/DictionaryValue.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/DictionaryVector.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/DoubleVector.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/DoubleVectorVector.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/Evaluator.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/float16.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/FloatVector.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/FloatVectorVector.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/Function.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/FunctionPtrVector.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/HTKFeatureConfiguration.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/HTKFeatureConfigurationVector.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/IntVector.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/Learner.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/LearnerVector.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/MaskKind.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/MinibatchData.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/MinibatchInfo.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/MinibatchSource.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/MinibatchSourceConfig.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/ModelFormat.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/NDArrayView.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/NDArrayViewPtrVector.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/NDMask.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/NDShape.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/PaddingMode.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/PairDoubleDouble.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/PairFloatFloat.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/PairIntInt.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/PairNDArrayViewPtrNDArrayViewPtr.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/PairSizeTDouble.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/PairSizeTInt.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/PairSizeTSizeT.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/Parameter.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/ParameterCloningMethod.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/ParameterVector.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/PoolingType.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/ProgressWriter.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/ProgressWriterVector.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/SizeTVector.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/SizeTVectorVector.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/StorageFormat.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/StreamConfiguration.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/StreamConfigurationVector.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/StreamInformation.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/StringVector.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/SWIGTYPE_p_int8_t.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/SWIGTYPE_p_void.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/TraceLevel.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/Trainer.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/TrainingParameterScheduleDouble.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/UnorderedMapParameterNDArrayViewPtr.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/UnorderedMapStreamInformationMinibatchData.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/UnorderedMapStreamInformationPairNDArrayViewPtrNDArrayViewPtr.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/UnorderedMapStringDictionaryValue.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/UnorderedMapVariableMinibatchData.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/UnorderedMapVariableValuePtr.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/UnorderedMapVariableVariable.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/UnsignedCharVector.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/Value.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/Variable.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/VariableKind.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/VariablePair.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/VariablePairVector.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/VariableVector.cs + ../Swig/Generated/${CMAKE_SYSTEM_NAME}/${config}/VectorPairSizeTDouble.cs +) + +if(CMAKE_GENERATOR STREQUAL "Unix Makefiles") + set(post_build_copy_statement + ${CMAKE_COMMAND} + -E + copy + ${CMAKE_CURRENT_BINARY_DIR}/CSharpBindings/Cntk.Core.CSBinding${binary_name_suffix}${CMAKE_SHARED_LIBRARY_SUFFIX} + ${CMAKE_CURRENT_BINARY_DIR}/AnyCPU/${config} + ) +elseif(CMAKE_GENERATOR MATCHES "Visual Studio.+") + set(post_build_copy_statement + ${CMAKE_COMMAND} + -E + copy_directory + ${CMAKE_CURRENT_BINARY_DIR}/CSharpBindings/x64/${config} + ${CMAKE_CURRENT_BINARY_DIR}/AnyCPU/${config} + ) +else() + message(FATAL_ERROR "The 'CMAKE_GENERATOR' value '${CMAKE_GENERATOR}' is not supported") +endif() + +add_custom_target( + CNTKLibraryManagedDll ALL + + DEPENDS + CSharpBindings + + SOURCES + ${source_filenames} + + COMMAND dotnet build ${CMAKE_CURRENT_SOURCE_DIR}/CNTKLibraryManagedDll.csproj /p:OutDirPrefix=${CMAKE_CURRENT_BINARY_DIR} /p:Configuration=${config} /p:PlatformName=${CMAKE_SYSTEM_NAME} /t:Restore + COMMAND dotnet build ${CMAKE_CURRENT_SOURCE_DIR}/CNTKLibraryManagedDll.csproj /p:OutDirPrefix=${CMAKE_CURRENT_BINARY_DIR} /p:Configuration=${config} /p:PlatformName=${CMAKE_SYSTEM_NAME} /t:Build + COMMAND ${post_build_copy_statement} +) + +add_dependencies(CNTKLibraryManagedDll CSharpBindings) diff --git a/bindings/csharp/CNTKLibraryManagedDll/CNTKLibraryManagedDll.csproj b/bindings/csharp/CNTKLibraryManagedDll/CNTKLibraryManagedDll.csproj index f5c1865079a8..d72199d76ba5 100644 --- a/bindings/csharp/CNTKLibraryManagedDll/CNTKLibraryManagedDll.csproj +++ b/bindings/csharp/CNTKLibraryManagedDll/CNTKLibraryManagedDll.csproj @@ -1,5 +1,32 @@ - - + + + + + ..\..\.. + + + Windows + netstandard2.0 @@ -11,7 +38,7 @@ false CNTK - ..\..\..\x64\$(Configuration) + $(OutDirPrefix)\$(Platform)\$(Configuration) false Copyright © 2016-$([System.DateTime]::Now.ToString(`yy`)) true @@ -19,26 +46,13 @@ Microsoft Corporation Cntk.Core.Managed $(CntkVersion) - + x64 + portable + Cntk.Core.Managed Cntk.Core.Managed-$(CntkComponentSuffix) - - - - - - - - - - - - - - - - + @@ -57,93 +71,92 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - diff --git a/bindings/csharp/CNTKLibraryManagedDll/SwigProxyClasses/GeneratedCode/Debug/README.md b/bindings/csharp/CNTKLibraryManagedDll/SwigProxyClasses/GeneratedCode/Debug/README.md deleted file mode 100644 index dc0bde50fb4f..000000000000 --- a/bindings/csharp/CNTKLibraryManagedDll/SwigProxyClasses/GeneratedCode/Debug/README.md +++ /dev/null @@ -1 +0,0 @@ -This directory is used by SWIG to put generated C# proxy classes. The generated files are not saved in the repository, but the directory is needed. \ No newline at end of file diff --git a/bindings/csharp/CNTKLibraryManagedDll/SwigProxyClasses/GeneratedCode/Release/README.md b/bindings/csharp/CNTKLibraryManagedDll/SwigProxyClasses/GeneratedCode/Release/README.md deleted file mode 100644 index dc0bde50fb4f..000000000000 --- a/bindings/csharp/CNTKLibraryManagedDll/SwigProxyClasses/GeneratedCode/Release/README.md +++ /dev/null @@ -1 +0,0 @@ -This directory is used by SWIG to put generated C# proxy classes. The generated files are not saved in the repository, but the directory is needed. \ No newline at end of file diff --git a/bindings/csharp/Swig/CMakeLists.txt b/bindings/csharp/Swig/CMakeLists.txt new file mode 100644 index 000000000000..f74134ebb1e1 --- /dev/null +++ b/bindings/csharp/Swig/CMakeLists.txt @@ -0,0 +1,212 @@ +cmake_minimum_required (VERSION 3.8) + +include(../../../Tools/cmake/common.cmake REQUIRED) + +EnsureProperties(CNTK_VERSION) +EnsureTools(swig;) + +include(../../../Tools/cmake/cpp_common.cmake REQUIRED) + +set(binary_name_suffix -${CNTK_VERSION}$<${is_debug}:d>) +set(binary_name Cntk.Core.CSBinding${binary_name_suffix}) + +# ---------------------------------------------------------------------- +# | SWIG +set(swig_configuration $) +set(swig_output_dir "${CMAKE_CURRENT_SOURCE_DIR}/Generated/${CMAKE_SYSTEM_NAME}/${swig_configuration}") + +set(native_swig_generated_files + ${CMAKE_CURRENT_SOURCE_DIR}/Generated/cntk_cs_wrap.h + ${CMAKE_CURRENT_SOURCE_DIR}/Generated/cntk_cs_wrap.cxx +) + +set(csharp_swig_generated_files + ${swig_output_dir}/AdditionalLearningOptions.cs; + ${swig_output_dir}/Axis.cs; + ${swig_output_dir}/AxisVector.cs; + ${swig_output_dir}/BoolVector.cs; + ${swig_output_dir}/CharVector.cs; + ${swig_output_dir}/CNTKDictionary.cs; + ${swig_output_dir}/CNTKLib.cs; + ${swig_output_dir}/CNTKLibPINVOKE.cs; + ${swig_output_dir}/Constant.cs; + ${swig_output_dir}/ConstantVector.cs; + ${swig_output_dir}/DataType.cs; + ${swig_output_dir}/DataUnit.cs; + ${swig_output_dir}/DeviceDescriptor.cs; + ${swig_output_dir}/DeviceDescriptorVector.cs; + ${swig_output_dir}/DeviceKind.cs; + ${swig_output_dir}/DictionaryValue.cs; + ${swig_output_dir}/DictionaryVector.cs; + ${swig_output_dir}/DoubleVector.cs; + ${swig_output_dir}/DoubleVectorVector.cs; + ${swig_output_dir}/Evaluator.cs; + ${swig_output_dir}/float16.cs; + ${swig_output_dir}/FloatVector.cs; + ${swig_output_dir}/FloatVectorVector.cs; + ${swig_output_dir}/Function.cs; + ${swig_output_dir}/FunctionPtrVector.cs; + ${swig_output_dir}/HTKFeatureConfiguration.cs; + ${swig_output_dir}/HTKFeatureConfigurationVector.cs; + ${swig_output_dir}/IntVector.cs; + ${swig_output_dir}/Learner.cs; + ${swig_output_dir}/LearnerVector.cs; + ${swig_output_dir}/MaskKind.cs; + ${swig_output_dir}/MinibatchData.cs; + ${swig_output_dir}/MinibatchInfo.cs; + ${swig_output_dir}/MinibatchSource.cs; + ${swig_output_dir}/MinibatchSourceConfig.cs; + ${swig_output_dir}/ModelFormat.cs; + ${swig_output_dir}/NDArrayView.cs; + ${swig_output_dir}/NDArrayViewPtrVector.cs; + ${swig_output_dir}/NDMask.cs; + ${swig_output_dir}/NDShape.cs; + ${swig_output_dir}/PaddingMode.cs; + ${swig_output_dir}/PairDoubleDouble.cs; + ${swig_output_dir}/PairFloatFloat.cs; + ${swig_output_dir}/PairIntInt.cs; + ${swig_output_dir}/PairNDArrayViewPtrNDArrayViewPtr.cs; + ${swig_output_dir}/PairSizeTDouble.cs; + ${swig_output_dir}/PairSizeTInt.cs; + ${swig_output_dir}/PairSizeTSizeT.cs; + ${swig_output_dir}/Parameter.cs; + ${swig_output_dir}/ParameterCloningMethod.cs; + ${swig_output_dir}/ParameterVector.cs; + ${swig_output_dir}/PoolingType.cs; + ${swig_output_dir}/ProgressWriter.cs; + ${swig_output_dir}/ProgressWriterVector.cs; + ${swig_output_dir}/SizeTVector.cs; + ${swig_output_dir}/SizeTVectorVector.cs; + ${swig_output_dir}/StorageFormat.cs; + ${swig_output_dir}/StreamConfiguration.cs; + ${swig_output_dir}/StreamConfigurationVector.cs; + ${swig_output_dir}/StreamInformation.cs; + ${swig_output_dir}/StringVector.cs; + ${swig_output_dir}/SWIGTYPE_p_int8_t.cs; + ${swig_output_dir}/SWIGTYPE_p_void.cs; + ${swig_output_dir}/TraceLevel.cs; + ${swig_output_dir}/Trainer.cs; + ${swig_output_dir}/TrainingParameterScheduleDouble.cs; + ${swig_output_dir}/UnorderedMapParameterNDArrayViewPtr.cs; + ${swig_output_dir}/UnorderedMapStreamInformationMinibatchData.cs; + ${swig_output_dir}/UnorderedMapStreamInformationPairNDArrayViewPtrNDArrayViewPtr.cs; + ${swig_output_dir}/UnorderedMapStringDictionaryValue.cs; + ${swig_output_dir}/UnorderedMapVariableMinibatchData.cs; + ${swig_output_dir}/UnorderedMapVariableValuePtr.cs; + ${swig_output_dir}/UnorderedMapVariableVariable.cs; + ${swig_output_dir}/UnsignedCharVector.cs; + ${swig_output_dir}/Value.cs; + ${swig_output_dir}/Variable.cs; + ${swig_output_dir}/VariableKind.cs; + ${swig_output_dir}/VariablePair.cs; + ${swig_output_dir}/VariablePairVector.cs; + ${swig_output_dir}/VariableVector.cs; + ${swig_output_dir}/VectorPairSizeTDouble.cs; +) + +set(swig_generated_files + ${native_swig_generated_files} + ${csharp_swig_generated_files} +) + +if(MSVC) + set(swig_params + -D_MSC_VER + -Werror + ) +else() + set(swig_params "") +endif() + +# Create the swig command line; it needs to be constructed manually as cmake won't convert path separators for the command if it +# is provided as a single value. +set(swig_command_line + ${swig_params} + -c++ + -csharp + -namespace CNTK + -dllimport + \"${binary_name}${CMAKE_SHARED_MODULE_SUFFIX}\" +) + +# There has got to be a better way to do this. When ${swig_output_dir} is converted to a native path, the +# embedded generator expression is de-generator-expression-ized and won't expand when used in the custom +# command below. So, remove the generator expression from the var, convert to the native path, and then +# reapply the generator expression. This only works because we know that the last path component is the +# generator expression. +get_filename_component(cmake_swig_output_dir_hack "${swig_output_dir}" DIRECTORY) + +file(TO_NATIVE_PATH "${cmake_swig_output_dir_hack}" native_swig_output_dir) +set(swig_command_line + ${swig_command_line} + -outdir + \"${native_swig_output_dir}/${swig_configuration}\" +) + +foreach(swig_include ${CMAKE_CURRENT_SOURCE_DIR}/../../../Source/CNTKv2LibraryDll/API;${CMAKE_CURRENT_SOURCE_DIR}/../../common) + file(TO_NATIVE_PATH "${swig_include}" swig_include) + set(swig_command_line + ${swig_command_line} + \"-I${swig_include}\" + ) +endforeach() + +file(TO_NATIVE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cntk_cs.i" swig_input) +set(swig_command_line + ${swig_command_line} + \"${swig_input}\" +) + +# Invoke the command line +add_custom_command( + COMMENT + "Generating SWIG Proxies for C++ and C#" + + DEPENDS + cntk_cs.i + std_unordered_map.i + ${CMAKE_CURRENT_SOURCE_DIR}/../../common/CNTKExceptionHandling.i + ${CMAKE_CURRENT_SOURCE_DIR}/../../common/CNTKManagedCommon.i + ${CMAKE_CURRENT_SOURCE_DIR}/../../common/CNTKValueExtend.i + ${CMAKE_CURRENT_SOURCE_DIR}/../../common/CNTKWarnFilters.i + + OUTPUT + # The most correct output is ${swig_generated_files}. However, some cmake generators don't support output based on generator + # expressions. In the name of supporting the least common denominator, specify the outputs that aren't based on generator expressions. + # + # ${swig_generated_files} + ${native_swig_generated_files} + + COMMAND ${CMAKE_COMMAND} -E make_directory "${swig_output_dir}" + COMMAND ${swig_binary} ${swig_command_line} + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/cntk_cs_wrap.h" "${CMAKE_CURRENT_SOURCE_DIR}/Generated" + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/cntk_cs_wrap.cxx" "${CMAKE_CURRENT_SOURCE_DIR}/Generated" + COMMAND ${CMAKE_COMMAND} -E remove -f "${CMAKE_CURRENT_SOURCE_DIR}/cntk_cs_wrap.h" + COMMAND ${CMAKE_COMMAND} -E remove -f "${CMAKE_CURRENT_SOURCE_DIR}/cntk_cs_wrap.cxx" +) + +# ---------------------------------------------------------------------- +# | CS Bindings +add_library(CSharpBindings SHARED ${native_swig_generated_files}) + +set_target_properties(CSharpBindings PROPERTIES RUNTIME_OUTPUT_DIRECTORY x64/) +set_target_properties(CSharpBindings PROPERTIES OUTPUT_NAME ${binary_name}) +set_target_properties(CSharpBindings PROPERTIES PREFIX "") + +target_include_directories(CSharpBindings + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../../Source/CNTKv2LibraryDll/API + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../../Source/Common/Include +) + +# TODO: Eventually, we should bind the this library rather than hard-coding the value +if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux") + string(TOLOWER "${config}" lower_config) + + target_link_libraries(CSharpBindings + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../../build/${lower_config}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}Cntk.Core${binary_name_suffix}${CMAKE_SHARED_LIBRARY_SUFFIX} + ) +else() + target_link_libraries(CSharpBindings + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../../x64/${config}/Cntk.Core${binary_name_suffix}${CMAKE_STATIC_LIBRARY_SUFFIX} + ) +endif() diff --git a/bindings/csharp/Swig/CNTKLibraryCSBinding.vcxproj b/bindings/csharp/Swig/CNTKLibraryCSBinding.vcxproj index 0a3557eef858..2be888f2afb2 100644 --- a/bindings/csharp/Swig/CNTKLibraryCSBinding.vcxproj +++ b/bindings/csharp/Swig/CNTKLibraryCSBinding.vcxproj @@ -107,16 +107,26 @@ $(SWIG_PATH)\swig.exe -c++ -csharp -D_MSC_VER -Werror -I$(SolutionDir)Source\CNTKv2LibraryDll\API -I$(SolutionDir)bindings\common -namespace CNTK -outdir $(SolutionDir)bindings\csharp\CNTKLibraryManagedDll\SwigProxyClasses\GeneratedCode\$(FlavorName) -dllimport Cntk.Core.CSBinding-$(CntkComponentVersion).dll cntk_cs.i cntk_cs_wrap.cxx; cntk_cs_warp.h $(SolutionDir)Source\CNTKv2LibraryDll\API\CNTKLibrary.h;$(SolutionDir)Source\CNTKv2LibraryDll\API\CNTKLibraryInternals.h;$(SolutionDir)Source\Common\Include\ExceptionWithCallStack.h;$(SolutionDir)bindings\common\CNTKValueExtend.i;$(SolutionDir)bindings\common\CNTKExceptionHandling.i;$(SolutionDir)bindings\common\CNTKManagedCommon.i;std_unordered_map.i + $(SWIG_PATH)\swig.exe -c++ -csharp -D_MSC_VER -Werror -I$(SolutionDir)Source\CNTKv2LibraryDll\API -I$(SolutionDir)bindings\common -namespace CNTK -outdir $(ProjectDir)\Generated\Windows\$(FlavorName) -dllimport Cntk.Core.CSBinding-$(CntkComponentVersion).dll cntk_cs.i && move /Y $(ProjectDir)\cntk_cs_wrap.* $(ProjectDir)\Generated + $(SWIG_PATH)\swig.exe -c++ -csharp -D_MSC_VER -Werror -I$(SolutionDir)Source\CNTKv2LibraryDll\API -I$(SolutionDir)bindings\common -namespace CNTK -outdir $(ProjectDir)\Generated\Windows\$(FlavorName) -dllimport Cntk.Core.CSBinding-$(CntkComponentVersion).dll cntk_cs.i && move /Y $(ProjectDir)\cntk_cs_wrap.* $(ProjectDir)\Generated + $(SWIG_PATH)\swig.exe -c++ -csharp -D_MSC_VER -Werror -I$(SolutionDir)Source\CNTKv2LibraryDll\API -I$(SolutionDir)bindings\common -namespace CNTK -outdir $(ProjectDir)\Generated\Windows\$(FlavorName) -dllimport Cntk.Core.CSBinding-$(CntkComponentVersion).dll cntk_cs.i && move /Y $(ProjectDir)\cntk_cs_wrap.* $(ProjectDir)\Generated + $(SWIG_PATH)\swig.exe -c++ -csharp -D_MSC_VER -Werror -I$(SolutionDir)Source\CNTKv2LibraryDll\API -I$(SolutionDir)bindings\common -namespace CNTK -outdir $(ProjectDir)\Generated\Windows\$(FlavorName) -dllimport Cntk.Core.CSBinding-$(CntkComponentVersion).dll cntk_cs.i && move /Y $(ProjectDir)\cntk_cs_wrap.* $(ProjectDir)\Generated + $(SWIG_PATH)\swig.exe -c++ -csharp -D_MSC_VER -Werror -I$(SolutionDir)Source\CNTKv2LibraryDll\API -I$(SolutionDir)bindings\common -namespace CNTK -outdir $(ProjectDir)\Generated\Windows\$(FlavorName) -dllimport Cntk.Core.CSBinding-$(CntkComponentVersion).dll cntk_cs.i && move /Y $(ProjectDir)\cntk_cs_wrap.* $(ProjectDir)\Generated + Generated\cntk_cs_wrap.cxx;Generated\cntk_cs_warp.h + Generated\cntk_cs_wrap.cxx;Generated\cntk_cs_warp.h + Generated\cntk_cs_wrap.cxx;Generated\cntk_cs_warp.h + Generated\cntk_cs_wrap.cxx;Generated\cntk_cs_warp.h + Generated\cntk_cs_wrap.cxx;Generated\cntk_cs_warp.h - + - + diff --git a/bindings/csharp/Swig/CNTKLibraryCSBinding.vcxproj.filters b/bindings/csharp/Swig/CNTKLibraryCSBinding.vcxproj.filters index db8de7490045..01a9bca1aa84 100644 --- a/bindings/csharp/Swig/CNTKLibraryCSBinding.vcxproj.filters +++ b/bindings/csharp/Swig/CNTKLibraryCSBinding.vcxproj.filters @@ -18,20 +18,20 @@ - + Source Files - - Source Files - Header Files Header Files + + Header Files + diff --git a/bindings/csharp/Swig/Generated/Linux/Debug/Readme.txt b/bindings/csharp/Swig/Generated/Linux/Debug/Readme.txt new file mode 100644 index 000000000000..fcaac79463e5 --- /dev/null +++ b/bindings/csharp/Swig/Generated/Linux/Debug/Readme.txt @@ -0,0 +1 @@ +This directory is used by SWIG when generating C# proxy classes. The generated files are not saved in the repository, but the directory is needed so that it exists when SWIG is invoked. \ No newline at end of file diff --git a/bindings/csharp/Swig/Generated/Linux/Release/Readme.txt b/bindings/csharp/Swig/Generated/Linux/Release/Readme.txt new file mode 100644 index 000000000000..fcaac79463e5 --- /dev/null +++ b/bindings/csharp/Swig/Generated/Linux/Release/Readme.txt @@ -0,0 +1 @@ +This directory is used by SWIG when generating C# proxy classes. The generated files are not saved in the repository, but the directory is needed so that it exists when SWIG is invoked. \ No newline at end of file diff --git a/bindings/csharp/Swig/Generated/Windows/Debug/Readme.txt b/bindings/csharp/Swig/Generated/Windows/Debug/Readme.txt new file mode 100644 index 000000000000..fcaac79463e5 --- /dev/null +++ b/bindings/csharp/Swig/Generated/Windows/Debug/Readme.txt @@ -0,0 +1 @@ +This directory is used by SWIG when generating C# proxy classes. The generated files are not saved in the repository, but the directory is needed so that it exists when SWIG is invoked. \ No newline at end of file diff --git a/bindings/csharp/Swig/Generated/Windows/Release/Readme.txt b/bindings/csharp/Swig/Generated/Windows/Release/Readme.txt new file mode 100644 index 000000000000..fcaac79463e5 --- /dev/null +++ b/bindings/csharp/Swig/Generated/Windows/Release/Readme.txt @@ -0,0 +1 @@ +This directory is used by SWIG when generating C# proxy classes. The generated files are not saved in the repository, but the directory is needed so that it exists when SWIG is invoked. \ No newline at end of file diff --git a/configure b/configure index 922a645fbc53..cb796d30f0dc 100755 --- a/configure +++ b/configure @@ -15,8 +15,8 @@ cuda_check=include/cuda.h enable_cuda= enable_python= - enable_java= +enable_csharp= # cntk build version # Modify both of the following during major release @@ -374,6 +374,17 @@ function default_use_java () } enable_java=$(default_use_java) +function default_use_csharp() +{ + if [[ `command -v dotnet` && `command -v swig` && `command -v cmake` ]] + then + echo yes + else + echo no + fi +} +enable_csharp=$(default_use_csharp) + function show_default () { if test x$1 = x @@ -390,13 +401,15 @@ function show_help () echo "Usage: configure [options]" echo "Options:" echo " -h|--help this help" + echo " --with-buildtype=(debug|release) $(show_default $default_buildtype)" echo " --with-build-top=directory build directory $(show_default $build_top)" echo " --add directory add directory to library search path" - echo " --asgd[=(yes|no)] use ASGD powered by Multiverso $(show_default $(default_use_asgd))" + echo " --asgd[=(yes|no)] use ASGD powered by Multiverso $(show_default ${default_use_asgd})" echo " --cuda[=(yes|no)] use cuda GPU $(show_default $(default_use_cuda))" echo " --python[=(yes|no)] with Python bindings $(show_default $(default_use_python))" echo " --java[=(yes|no)] with Java bindings $(show_default $(default_use_java))" echo " --with-jdk[=directory] $(show_default $(find_jdk))" + echo " --csharp[=(yes|no)] with C# bindings $(show_default $(default_use_csharp))" echo " --mpi[=(yes|no)] use MPI communication $(show_default ${default_use_mpi})" echo " --gdr[=(yes|no)] use GPUDirect RDMA $(show_default ${default_cuda_gdr})" echo " --with-cuda[=directory] $(show_default $(find_cuda))" @@ -408,7 +421,6 @@ function show_help () echo " --with-mkl[=directory] $(show_default $(find_mkl))" echo " --with-mkl-sequential[=directory] $(show_default $(find_mkl))" echo " --with-openblas[=directory] (experimental) $(show_default $(find_openblas))" - echo " --with-buildtype=(debug|release) $(show_default $default_buildtype)" echo " --with-kaldi[=directory] $(show_default $(find_kaldi))" echo " --with-opencv[=directory] $(show_default $(find_opencv))" echo " --with-libzip[=directory] $(show_default $(find_libzip))" @@ -607,6 +619,17 @@ do fi ;; + --csharp*) + if test x$optarg = xyes || test x$optarg = xno + then + enable_csharp=$optarg + else + echo "Invalid value for --csharp $optarg" + show_help + exit + fi + ;; + --mpi*) if test x$optarg = xyes || test x$optarg = xno then @@ -1077,12 +1100,12 @@ then fi fi -if ( test $enable_java = yes || test $enable_python = yes ) && test x$swig_path = x +if ( test $enable_java = yes || test $enable_python = yes || test $enable_csharp = yes ) && test x$swig_path = x then swig_path=$(find_swig) if test x$swig_path = x then - echo 'Cannot locate SWIG (>= 3.0.10), which is required for Python/Java support.' + echo 'Cannot locate SWIG (>= 3.0.10), which is required for Python/Java/C# support.' echo Please see https://docs.microsoft.com/en-us/cognitive-toolkit/Setup-CNTK-on-Linux#optional-swig echo for installation instructions. exit 1 @@ -1250,7 +1273,7 @@ if test $enable_cuda = yes ; then echo CUDNN_PATH=$cudnn_path >> $config [ -z "$nccl_path" ] || echo NCCL_PATH=$nccl_path >> $config fi -if test $enable_python = yes || test $enable_java = yes ; then +if test $enable_python = yes || test $enable_java = yes || test $enable_csharp = yes; then echo SWIG_PATH=$swig_path/bin >> $config fi if test $enable_python = yes ; then @@ -1277,6 +1300,9 @@ if test $enable_java = yes ; then echo JDK_PATH=$jdk_path >> $config fi fi +if test $enable_csharp = yes ; then + echo CSHARP_SUPPORT=true >> $config +fi if test x$kaldi_path != x ; then echo KALDI_PATH=$kaldi_path >> $config fi