Skip to content
Permalink
Browse files

Update submodule, fix issue with dlls

  • Loading branch information...
sdpython committed Nov 5, 2018
1 parent 9ed585f commit fe1bdac2899798b7240646db4413887017782933
@@ -285,3 +285,4 @@ _doc/sphinxdoc/source/components
cscode/packages
_doc/examples/model.zip
_doc/sphinxdoc/source/nbcov*
_unittests/ut_binaries/diabete.csv
@@ -144,6 +144,36 @@ def test_check_outout(self):
stdout = pipe.StdOut
self.assertIn('Training learner 1', stdout)

def test_diabetes_ols(self):
from sklearn.datasets import load_diabetes
diabetes = load_diabetes()
diabetes_X_train = diabetes.data[:-20]
diabetes_X_test = diabetes.data[-20:]
diabetes_y_train = diabetes.target[:-20]
# diabetes_y_test = diabetes.target[-20:]

conc_train = pandas.DataFrame(diabetes_X_train.astype(numpy.float32),
columns=["F%d" % i for i in range(0, diabetes_X_train.shape[1])])
concat = f"concat{{col=Feature:{','.join(conc_train.columns)}}}"
self.assertEqual(
concat, "concat{col=Feature:F0,F1,F2,F3,F4,F5,F6,F7,F8,F9}")
conc_train["Label"] = diabetes_y_train.astype(numpy.float32)
conc_train.to_csv("diabete.csv", index=False)

pipe = CSPipeline([concat], "ols")
pipe.fit(conc_train, feature="Feature", label="Label")

pred = pipe.predict(conc_train)
self.assertEqual(pred.shape, (422, 22))

conc_test = pandas.DataFrame(diabetes_X_test.astype(numpy.float32),
columns=["F%d" % i for i in range(0, diabetes_X_train.shape[1])])
conc_test["Label"] = numpy.zeros(
(conc_test.shape[0],), dtype=numpy.float32)

pred2 = pipe.predict(conc_test)
self.assertEqual(pred2.shape, (20, 22))


if __name__ == "__main__":
unittest.main()
@@ -76,6 +76,11 @@ if %errorlevel% neq 0 exit /b %errorlevel%
python -u setup.py copybin release
if %errorlevel% neq 0 exit /b %errorlevel%

:finalcopy:

copy cscode\bin\machinelearningext\Release\mkl*.dll cscode\bin\AnyCPU.Release\TestCSharPyMLExtension\netcoreapp2.1
copy cscode\bin\machinelearningext\Debug\mkl*.dll cscode\bin\AnyCPU.Debug\TestCSharPyMLExtension\netcoreapp2.1


:end:
@echo [build.cmd] Completed.
@@ -67,7 +67,6 @@
<Reference Include="Scikit.ML.ScikitAPI">
<HintPath>..\bin\machinelearningext\$(Configuration)\Scikit.ML.DocHelperMlExt.dll</HintPath>
</Reference>

<Reference Include="Scikit.ML.Clustering">
<HintPath>..\bin\machinelearningext\$(Configuration)\Scikit.ML.Clustering.dll</HintPath>
</Reference>
@@ -25,7 +25,7 @@ public static ScikitPipeline CreateScikitPipeline(Stream st, IHostEnvironment ho
{
if (host == null)
throw new ArgumentNullException("host cannot be null.");
return new ScikitPipeline(st, host);
return new ScikitPipeline(st, ScikitPipeline.SaveFormat.Zip, host);
}

public static ScikitPipeline CreateScikitPipeline(string[] transforms = null, string predictor = null, IHostEnvironment host = null)
@@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>netcoreapp2.0</TargetFramework>
<TargetFramework>netcoreapp2.1</TargetFramework>
<IsPackable>false</IsPackable>
</PropertyGroup>

@@ -63,6 +63,39 @@
<Reference Include="Scikit.ML.PipelineHelper">
<HintPath>..\bin\machinelearningext\$(Configuration)\Scikit.ML.PipelineHelper.dll</HintPath>
</Reference>
<Reference Include="Scikit.ML.Clustering">
<HintPath>..\bin\machinelearningext\$(Configuration)\Scikit.ML.Clustering.dll</HintPath>
</Reference>
<Reference Include="Scikit.ML.FeaturesTransforms">
<HintPath>..\bin\machinelearningext\$(Configuration)\Scikit.ML.FeaturesTransforms.dll</HintPath>
</Reference>
<Reference Include="Scikit.ML.ModelSelection">
<HintPath>..\bin\machinelearningext\$(Configuration)\Scikit.ML.ModelSelection.dll</HintPath>
</Reference>
<Reference Include="Scikit.ML.MultiClass">
<HintPath>..\bin\machinelearningext\$(Configuration)\Scikit.ML.MultiClass.dll</HintPath>
</Reference>
<Reference Include="Scikit.ML.NearestNeighbors">
<HintPath>..\bin\machinelearningext\$(Configuration)\Scikit.ML.NearestNeighbors.dll</HintPath>
</Reference>
<Reference Include="Scikit.ML.PipelineGraphTraining">
<HintPath>..\bin\machinelearningext\$(Configuration)\Scikit.ML.PipelineGraphTraining.dll</HintPath>
</Reference>
<Reference Include="Scikit.ML.PipelineGraphTransforms">
<HintPath>..\bin\machinelearningext\$(Configuration)\Scikit.ML.PipelineGraphTransforms.dll</HintPath>
</Reference>
<Reference Include="Scikit.ML.PipelineLambdaTransforms">
<HintPath>..\bin\machinelearningext\$(Configuration)\Scikit.ML.PipelineLambdaTransforms.dll</HintPath>
</Reference>
<Reference Include="Scikit.ML.ProductionPrediction">
<HintPath>..\bin\machinelearningext\$(Configuration)\Scikit.ML.ProductionPrediction.dll</HintPath>
</Reference>
<Reference Include="Scikit.ML.RandomTransforms">
<HintPath>..\bin\machinelearningext\$(Configuration)\Scikit.ML.RandomTransforms.dll</HintPath>
</Reference>
<Reference Include="Scikit.ML.TimeSeries">
<HintPath>..\bin\machinelearningext\$(Configuration)\Scikit.ML.TimeSeries.dll</HintPath>
</Reference>
</ItemGroup>

<Target Name="PostBuild" AfterTargets="PostBuildEvent">
@@ -3,6 +3,8 @@
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System;
using System.IO;
using System.Runtime.InteropServices;
using System.Linq;
using Microsoft.ML.Runtime.Data;
using Scikit.ML.DataManipulation;
using CSharPyMLExtension;
@@ -24,6 +26,48 @@ public void TestPipelineIris()
var sout = env3.Item2.ToString();
Assert.IsNotNull(sout);
}

internal static class Mkl
{
private const string DllName = "MklImports";

public enum Layout
{
RowMajor = 101,
ColMajor = 102
}

public enum UpLo : byte
{
Up = (byte)'U',
Lo = (byte)'L'
}

[DllImport(DllName, EntryPoint = "LAPACKE_dpptrf")]
public static extern int PptrfInternal(Layout layout, UpLo uplo, int n, Double[] ap);
}

[TestMethod]
public void TestScikitAPI_MKL()
{
Mkl.PptrfInternal(Mkl.Layout.ColMajor, Mkl.UpLo.Lo, 2, new double[] { 0.1, 0.3 });
}

[TestMethod]
public void TestPipelineDiabete()
{
var diab = FileHelper.GetTestFile("diabete.csv");
var cols = Enumerable.Range(0, 10).Select(c => NumberType.R4).ToArray();
var colsName = string.Join(',', Enumerable.Range(0, 10).Select(c => $"F{c}"));
var df = DataFrameIO.ReadCsv(diab, sep: ',', dtypes: cols);
var env3 = PyEnvHelper.CreateStoreEnvironment();
var pipe = PyPipelineHelper.CreateScikitPipeline(new string[] { $"Concat{{col=Features:{colsName}}}" }, "ols", env3.Item1);
pipe.Train(df, "Features", "Label");
var sout = env3.Item2.ToString();
Assert.IsNotNull(sout);
DataFrame pred = PyPipelineHelper.FastPredictOrTransform(pipe, df);
Assert.IsTrue(df.Shape[0] > 0);
}
}
}

@@ -0,0 +1,50 @@
F0,F1,F2,F3,F4,F5,F6,F7,F8,F9,Label
0.038075905,0.05068012,0.061696205,0.021872355,-0.0442235,-0.03482076,-0.043400846,-0.002592262,0.01990842,-0.017646125,151.0
-0.0018820165,-0.044641636,-0.05147406,-0.026327835,-0.008448724,-0.01916334,0.07441156,-0.039493382,-0.068329744,-0.09220405,75.0
0.0852989,0.05068012,0.044451214,-0.0056706103,-0.04559945,-0.034194466,-0.03235593,-0.002592262,0.0028637706,-0.02593034,141.0
-0.08906294,-0.044641636,-0.0115950145,-0.036656447,0.012190569,0.024990594,-0.03603757,0.03430886,0.022692023,-0.009361912,206.0
0.0053830603,-0.044641636,-0.03638469,0.021872355,0.0039348514,0.015596139,0.008142084,-0.002592262,-0.031991445,-0.046640873,135.0
-0.092695475,-0.044641636,-0.04069594,-0.019442093,-0.06899065,-0.07928784,0.041276824,-0.076394506,-0.041180383,-0.096346155,97.0
-0.045472477,0.05068012,-0.047162812,-0.015999222,-0.04009564,-0.024800012,0.000778808,-0.039493382,-0.06291295,-0.038356658,138.0
0.063503675,0.05068012,-0.0018947058,0.06662967,0.090619884,0.10891438,0.022868635,0.017703354,-0.03581673,0.0030644094,63.0
0.041708443,0.05068012,0.061696205,-0.04009932,-0.013952536,0.006201686,-0.028674295,-0.002592262,-0.014956475,0.011348623,110.0
-0.07090025,-0.044641636,0.039062154,-0.033213574,-0.012576583,-0.034507614,-0.024992656,-0.002592262,0.06773633,-0.013504018,310.0
-0.09632801,-0.044641636,-0.08380842,0.008100872,-0.10338947,-0.09056119,-0.013947743,-0.076394506,-0.06291295,-0.034214552,101.0
0.027178291,0.05068012,0.017505912,-0.033213574,-0.0070727714,0.04597154,-0.06549067,0.07120998,-0.09643322,-0.059067193,69.0
0.016280675,-0.044641636,-0.028840007,-0.009113481,-0.0043208655,-0.009768886,0.04495846,-0.039493382,-0.03075121,-0.042498767,179.0
0.0053830603,0.05068012,-0.0018947058,0.008100872,-0.0043208655,-0.015718706,-0.0029028298,-0.002592262,0.038393248,-0.013504018,185.0
0.045340985,-0.044641636,-0.02560657,-0.012556352,0.01769438,-6.128358e-05,0.08177484,-0.039493382,-0.031991445,-0.07563562,118.0
-0.052737556,0.05068012,-0.018061887,0.08040116,0.089243926,0.10766178,-0.03971921,0.1081111,0.03605579,-0.042498767,171.0
-0.005514555,-0.044641636,0.04229559,0.04941532,0.024574144,-0.023860566,0.07441156,-0.039493382,0.05228,0.027917052,166.0
0.07076875,0.05068012,0.012116851,0.05630106,0.034205813,0.049416173,-0.03971921,0.03430886,0.027367707,-0.0010776975,144.0
-0.0382074,-0.044641636,-0.010517202,-0.036656447,-0.037343733,-0.019476488,-0.028674295,-0.002592262,-0.018118268,-0.017646125,97.0
-0.027309787,-0.044641636,-0.018061887,-0.04009932,-0.0029449128,-0.011334628,0.037595186,-0.039493382,-0.008944019,-0.054925088,168.0
-0.049105015,-0.044641636,-0.05686312,-0.043542188,-0.04559945,-0.04327577,0.000778808,-0.039493382,-0.011900685,0.01549073,68.0
-0.0854304,0.05068012,-0.022373136,0.0012151308,-0.037343733,-0.026365755,0.0155053595,-0.039493382,-0.07212845,-0.017646125,49.0
-0.0854304,-0.044641636,-0.00405033,-0.009113481,-0.0029449128,0.007767428,0.022868635,-0.039493382,-0.061176594,-0.013504018,68.0
0.045340985,0.05068012,0.060618393,0.031053344,0.028702004,-0.0473467,-0.05444576,0.07120998,0.13359898,0.13561183,245.0
-0.06363517,-0.044641636,0.035828717,-0.022884965,-0.03046397,-0.01885019,-0.006584468,-0.002592262,-0.025952425,-0.054925088,184.0
-0.06726771,0.05068012,-0.012672827,-0.04009932,-0.015328488,0.0046359436,-0.058127396,0.03430886,0.019199032,-0.034214552,202.0
-0.107225634,-0.044641636,-0.07734155,-0.026327835,-0.08962994,-0.09619786,0.026550272,-0.076394506,-0.042572103,-0.0052198046,137.0
-0.023677247,-0.044641636,0.05954058,-0.04009932,-0.042847544,-0.04358892,0.011823721,-0.039493382,-0.015998268,0.04034337,85.0
0.05260606,-0.044641636,-0.021295324,-0.07452802,-0.04009564,-0.0376391,-0.006584468,-0.039493382,-0.0006092542,-0.054925088,131.0
0.06713621,0.05068012,-0.006205954,0.0631868,-0.042847544,-0.09588471,0.052321736,-0.076394506,0.0594238,0.05276969,283.0
-0.060002632,-0.044641636,0.044451214,-0.019442093,-0.009824677,-0.0075768465,0.022868635,-0.039493382,-0.027128646,-0.009361912,129.0
-0.023677247,-0.044641636,-0.06548562,-0.08141377,-0.038719688,-0.05360967,0.059685014,-0.076394506,-0.037128344,-0.042498767,59.0
0.034443367,0.05068012,0.12528712,0.028758096,-0.05385517,-0.01290037,-0.10230705,0.1081111,0.00027148574,0.027917052,341.0
0.03081083,-0.044641636,-0.05039625,-0.00222774,-0.0442235,-0.08993489,0.11859122,-0.076394506,-0.018118268,0.0030644094,87.0
0.016280675,-0.044641636,-0.063329995,-0.05731367,-0.057983026,-0.048912443,0.008142084,-0.039493382,-0.0594727,-0.06735141,65.0
0.048973523,0.05068012,-0.030995632,-0.049280304,0.049341295,-0.0041322135,0.13331777,-0.05351581,0.021310847,0.019632837,102.0
0.012648137,-0.044641636,0.022894971,0.052858192,0.00806271,-0.028557794,0.037595186,-0.039493382,0.054724004,-0.02593034,265.0
-0.009147094,-0.044641636,0.011039039,-0.05731367,-0.024960158,-0.042962622,0.03023191,-0.039493382,0.017037133,-0.0052198046,276.0
-0.0018820165,0.05068012,0.071396515,0.09761551,0.087867975,0.0754075,-0.021311019,0.07120998,0.07142403,0.023774944,252.0
-0.0018820165,0.05068012,0.014272476,-0.07452802,0.0025588986,0.006201686,-0.013947743,-0.002592262,0.019199032,0.0030644094,90.0
0.0053830603,0.05068012,-0.008361578,0.021872355,0.054845106,0.073215455,-0.024992656,0.03430886,0.012553153,0.09419076,100.0
-0.09996056,-0.044641636,-0.06764124,-0.10895673,-0.07449446,-0.07271173,0.0155053595,-0.039493382,-0.04986847,-0.009361912,55.0
-0.060002632,0.05068012,-0.010517202,-0.014851599,-0.04972731,-0.023547418,-0.058127396,0.015858298,-0.009918957,-0.034214552,61.0
0.019913213,-0.044641636,-0.023450946,-0.071085155,0.020446286,-0.010082034,0.11859122,-0.076394506,-0.042572103,0.073480226,92.0
0.045340985,0.05068012,0.06816308,0.008100872,-0.016704442,0.0046359436,-0.07653558,0.07120998,0.032433227,-0.017646125,259.0
0.027178291,0.05068012,-0.03530688,0.032200966,-0.01120063,0.0015044587,-0.010266106,-0.002592262,-0.014956475,-0.050782982,53.0
-0.056370094,-0.044641636,-0.0115950145,-0.033213574,-0.046975404,-0.04765985,0.0044604456,-0.039493382,-0.007979398,-0.08806194,190.0
-0.07816532,-0.044641636,-0.0730303,-0.05731367,-0.08412613,-0.07427747,-0.024992656,-0.039493382,-0.018118268,-0.08391984,142.0
0.06713621,0.05068012,-0.04177375,0.011543743,0.0025588986,0.005888537,0.041276824,-0.039493382,-0.0594727,-0.021788232,75.0
@@ -41,8 +41,8 @@

packages = find_packages('src', exclude='src')
package_dir = {k: "src/" + k.replace(".", "/") for k in packages}
package_data = {project_var_name
+ ".binaries.Release": ["*.dll", "*.so", "*.json", "*.xml"]}
package_data = {project_var_name +
".binaries.Release": ["*.dll", "*.so", "*.json", "*.xml"]}

############
# functions
@@ -417,14 +417,23 @@ def copy_assemblies(libdef=None, version="Release"):
"4.4.0", "lib", "netstandard2.0"),
os.path.join(rootpkg, "lightgbm",
"2.2.1.1", "runtimes", "win-x64", "native"),
os.path.join(rootpkg, "google.protobuf",
"3.5.1", "lib", "netstandard1.0"),
])

if sys.platform.startswith("win"):
sub = "win-x64"
else:
sub = "linux-x64"

# Additional dependencies.
folders.extend([
os.path.join(rootpkg, "system.threading.tasks.dataflow",
"4.8.0", "lib", "netstandard2.0"),
os.path.join(rootpkg, "system.drawing.common",
"4.5.0", "lib", "netstandard2.0"),
os.path.join(rootpkg, "mlnetmkldeps", "0.0.0.7",
"runtimes", sub, "native"),
])

dests = ['src/csharpyml/binaries/%s' % version]
@@ -113,7 +113,7 @@ def transform(self, data, conc=1):

def _predict_transform(self, data, fctname, conc=1):
"""
Transforms pr predicts with a trained pipeline, there is no predictor.
Transforms or predicts with a trained pipeline, there is no predictor.
@param data dataframe (*pandas* or *C#*)
@param fctname `'predict'` or `'transform'`
@@ -161,7 +161,8 @@ def clean_name(name):
logger.warning("[csml] empty kind {0}\n{1}".format(k, e))
continue
for comp in comps:
refs[comp.Name] = ":ref:`l-{0}`".format(comp.Name.lower().replace(".", "-"))
refs[comp.Name] = ":ref:`l-{0}`".format(
comp.Name.lower().replace(".", "-"))

# kinds and components
for v, k in sorted_kinds:
@@ -236,7 +237,7 @@ def write_components_pages(app, env, docnames):
content = f.read()
else:
content = None

if content != v:
with open(d, "w", encoding="utf-8") as f:
f.write(v)
@@ -248,8 +249,8 @@ def get_mlnet_assemblies(chdir=False):
"""
if chdir:
cur = os.getcwd()
os.chdir(dll)
res = MamlHelper.GetLoadedAssembliesLocation(True)
os.chdir(chdir)
res = MamlHelper.GetLoadedAssembliesLocation(True) # pylint: disable=E0602
if chdir:
os.chdir(cur)
dependencies = []
@@ -260,35 +261,36 @@ def get_mlnet_assemblies(chdir=False):
usings = ["System", "System.Linq", "System.Collections.Generic", "System.IO",
"System.Text"]
usings.extend([
"Microsoft.ML.Runtime",
"Microsoft.ML.Runtime.Api",
"Microsoft.ML.Runtime.Data",
"Microsoft.ML.Runtime.Learners",
"Microsoft.ML.Runtime.Ensemble",
"Microsoft.ML.Runtime.LightGBM",
"Microsoft.ML.Runtime.Model.Onnx",
"Microsoft.ML.Runtime.TimeSeriesProcessing",
"Microsoft.ML.Runtime.Tools",
"Microsoft.ML.Trainers",
"Microsoft.ML.Trainers.HalLearners",
"Microsoft.ML.Trainers.KMeans",
"Microsoft.ML.Trainers.FastTree",
"Microsoft.ML.Trainers.Online",
"Microsoft.ML.Trainers.PCA",
"Microsoft.ML.Transforms",
"Microsoft.ML.Transforms.Categorical",
"Microsoft.ML.Transforms.Normalizers",
"Microsoft.ML.Transforms.Projections",
"Microsoft.ML.Transforms.TensorFlow",
"Microsoft.ML.Transforms.Text",
"Microsoft.ML.Runtime.Sweeper",
])
res = MamlHelper.GetAssemblies()
usings.extend([a.FullName.split(',')[0] for a in res if "Scikit" in a.FullName])
"Microsoft.ML.Runtime",
"Microsoft.ML.Runtime.Api",
"Microsoft.ML.Runtime.Data",
"Microsoft.ML.Runtime.Learners",
"Microsoft.ML.Runtime.Ensemble",
"Microsoft.ML.Runtime.LightGBM",
"Microsoft.ML.Runtime.Model.Onnx",
"Microsoft.ML.Runtime.TimeSeriesProcessing",
"Microsoft.ML.Runtime.Tools",
"Microsoft.ML.Trainers",
"Microsoft.ML.Trainers.HalLearners",
"Microsoft.ML.Trainers.KMeans",
"Microsoft.ML.Trainers.FastTree",
"Microsoft.ML.Trainers.Online",
"Microsoft.ML.Trainers.PCA",
"Microsoft.ML.Transforms",
"Microsoft.ML.Transforms.Categorical",
"Microsoft.ML.Transforms.Normalizers",
"Microsoft.ML.Transforms.Projections",
"Microsoft.ML.Transforms.TensorFlow",
"Microsoft.ML.Transforms.Text",
"Microsoft.ML.Runtime.Sweeper",
])
res = MamlHelper.GetAssemblies() # pylint: disable=E0602
usings.extend([a.FullName.split(',')[0]
for a in res if "Scikit" in a.FullName])
return dependencies, usings


class RunCSharpMLDirective(RunCSharpDirective):

class RunCSharpMLDirective(RunCSharpDirective): # pylint: disable=E0602
"""
Implicits "and dependencies.
"""
@@ -298,8 +300,8 @@ def modify_script_before_running(self, script):
The methods modifies the script to *csharpy* to
run :epkg:`C#` from :epkg:`Python`.
"""
if not hasattr(RunCSharpDirective, 'deps_using'):
RunCSharpDirective.deps_using = get_mlnet_assemblies()
if not hasattr(RunCSharpDirective, 'deps_using'): # pylint: disable=E0602
RunCSharpDirective.deps_using = get_mlnet_assemblies() # pylint: disable=E0602
dependencies, usings = RunCSharpMLDirective.deps_using
return self._modify_script_before_running(script, usings, dependencies)

0 comments on commit fe1bdac

Please sign in to comment.
You can’t perform that action at this time.