Skip to content
Permalink
Browse files

Udate submodules

  • Loading branch information...
sdpython committed Nov 2, 2018
1 parent 973ad7a commit 665f392f258b245d1f79d4fedc6aef5d30689494
Submodule machinelearning updated 413 files
Submodule machinelearningext updated 89 files
+9 −0 .gitignore
+1 −1 README.md
+1 −1 build_doc.bat
+239 −0 docs/source/aonnx.rst
+122 −0 docs/source/apicsharpdoc.rst
+1 −1 docs/source/commandline.rst
+18 −1 docs/source/conf.py
+5 −0 docs/source/incompatibilities.rst
+5 −3 docs/source/index.rst
+2 −2 docs/source/machinelearning_docs.rst
+84 −8 docs/source/sphinx_mlext.py
+8 −4 docs/source/sphinx_mlext_templates.py
+1 −1 machinelearning
+4 −4 machinelearningext/DataManipulation/DataColumn.cs
+2 −2 machinelearningext/DataManipulation/DataContainer.cs
+1 −1 machinelearningext/DataManipulation/DataManipulation.csproj
+1 −1 machinelearningext/Dependencies.props
+2 −2 machinelearningext/Directory.Build.props
+3 −2 machinelearningext/DocHelperMlExt/DocHelperMlExt.csproj
+1 −1 machinelearningext/DocHelperMlExt/LinkHelper.cs
+87 −3 machinelearningext/DocHelperMlExt/MamlHelper.cs
+1 −1 machinelearningext/EntryPoints/EntryPoints.csproj
+1 −1 machinelearningext/FeaturesTransforms/FeaturesTransforms.csproj
+4 −5 machinelearningext/ModelSelection/SplitTrainTestTransform.cs
+1 −1 machinelearningext/MultiClass/MultiClass.csproj
+10 −4 machinelearningext/MultiClass/MultiClassConvertTransform.cs
+1 −1 machinelearningext/MultiClass/MultiToBinaryTransform.cs
+7 −7 machinelearningext/MultiClass/MultiToPredictorCommon.cs
+1 −0 machinelearningext/MultiClass/MultiToRankerTrainer.cs
+1 −0 machinelearningext/MultiClass/MultiToTrainerCommon.cs
+4 −4 machinelearningext/MultiClass/OptimizedOVAPredictor.cs
+12 −11 machinelearningext/MultiClass/OptimizedOVATrainer.cs
+14 −14 machinelearningext/NearestNeighbors/NearestNeighbors/NearestNeighborsValueMapper.cs
+186 −0 machinelearningext/OnnxHelper/Convert2Onnx.cs
+26 −0 machinelearningext/OnnxHelper/ConvertFromOnnx.cs
+38 −0 machinelearningext/OnnxHelper/OnnxHelper.csproj
+456 −0 machinelearningext/OnnxHelper/OnnxProtoBuf.cs
+353 −0 machinelearningext/OnnxHelper/ScikitOnnxContext.cs
+47 −0 machinelearningext/OnnxHelper/ScikitOnnxNode.cs
+51 −2 machinelearningext/PipelineGraphTransforms/Interfaces/ITaggedDataView.cs
+1 −0 machinelearningext/PipelineGraphTransforms/TagTrainOrScoreTransform.cs
+2 −2 machinelearningext/PipelineHelper/ExtendedSchema.cs
+2 −2 machinelearningext/PipelineHelper/Helpers/SchemaHelper.cs
+1 −1 machinelearningext/PipelineHelper/PipelineHelper.csproj
+14 −14 machinelearningext/PipelineHelper/TypedConverter.cs
+4 −4 machinelearningext/PipelineHelper/VBufferEqSort.cs
+1 −1 machinelearningext/PipelineLambdaTransforms/LambdaColumnPassThroughView.cs
+4 −4 machinelearningext/PipelineLambdaTransforms/PredictTransform.cs
+4 −4 machinelearningext/PipelineTraining/PrePostProcessPredictor.cs
+13 −1 machinelearningext/PipelineTransforms/DescribeTransform.cs
+17 −3 machinelearningext/PipelineTransforms/ExtendedCacheTransform.cs
+12 −1 machinelearningext/PipelineTransforms/PassThroughTransform.cs
+1 −1 machinelearningext/PipelineTransforms/PipelineTransforms.csproj
+2 −2 machinelearningext/PipelineTransforms/ULabelToR4LabelTransform.cs
+7 −7 machinelearningext/ProductionPrediction/InfiniteLoopViewCursor.cs
+0 −2 machinelearningext/ProductionPrediction/PassThroughEnvironment.cs
+1 −1 machinelearningext/ProductionPrediction/PredictionFunctionDataFrame.cs
+1 −1 machinelearningext/ProductionPrediction/TransformFromValueMapper.cs
+1 −2 machinelearningext/ProductionPrediction/ValueMapperDataFrameFromTransform.cs
+53 −0 machinelearningext/ProductionPrediction/ValueMapperDispose.cs
+34 −81 machinelearningext/ProductionPrediction/ValueMapperFromTransform.cs
+34 −58 machinelearningext/ProductionPrediction/ValueMapperFromTransformFloat.cs
+8 −13 machinelearningext/ProductionPrediction/ValueMapperPredictionEngine.cs
+8 −11 machinelearningext/ProductionPrediction/ValueMapperPredictionEngineFloat.cs
+1 −1 machinelearningext/RandomTransforms/RandomTransforms.csproj
+5 −5 machinelearningext/RandomTransforms/ShakeInputTransform.cs
+55 −27 machinelearningext/ScikitAPI/ComponentHelper.cs
+17 −0 machinelearningext/ScikitAPI/ScikitAPI.csproj
+74 −6 machinelearningext/ScikitAPI/ScikitPipeline.cs
+3 −3 machinelearningext/TestHelper/DummyInputs.cs
+4 −0 machinelearningext/TestHelper/EnvHelper.cs
+1 −1 machinelearningext/TestHelper/TestHelper.csproj
+1 −1 machinelearningext/TestHelper/TransformHelper.cs
+1 −1 machinelearningext/TestMachineLearningExt/TestDataManipulation.cs
+3 −2 machinelearningext/TestMachineLearningExt/TestMachineLearningExt.csproj
+51 −0 machinelearningext/TestMachineLearningExt/TestOnnx.cs
+0 −1 machinelearningext/TestMachineLearningExt/TestPipelineTraining.cs
+6 −6 machinelearningext/TestMachineLearningExt/TestPipelineTransforms.cs
+45 −52 machinelearningext/TestMachineLearningExt/TestProductionPrediction.cs
+6 −4 machinelearningext/TestMachineLearningExt/TestQuestion.cs
+3 −2 machinelearningext/TestMachineLearningExt/TestResampleTransforms.cs
+2 −1 machinelearningext/TestMachineLearningExt/TestScikitAPI.cs
+27 −42 machinelearningext/TestMachineLearningExt/Test_Benchmarks.cs
+13 −17 machinelearningext/TestProfileBenchmark/Benchmark_PredictionEngine.cs
+113 −0 machinelearningext/TestProfileBenchmark/Debug.cs
+26 −13 machinelearningext/TestProfileBenchmark/Program.cs
+1 −1 machinelearningext/TestProfileBenchmark/TestProfileBenchmark.csproj
+5 −5 machinelearningext/TimeSeries/DeTrendTransform.cs
+6 −0 machinelearningext/machinelearningext.sln
@@ -61,6 +61,8 @@ def mlnet_components_kinds():
"""
MamlHelper = get_maml_helper()
kinds = list(MamlHelper.GetAllKinds())
kinds += ["argument", "command"]
kinds = list(set(kinds))
titles = {
'anomalydetectortrainer': 'Anomaly Detection',
'binaryclassifiertrainer': 'Binary Classification',
@@ -75,7 +77,9 @@ def mlnet_components_kinds():
'ngramextractorfactory': 'N-Grams',
'rankertrainer': 'Ranking',
'regressortrainer': 'Regression',
'tokenizetransform': 'Tokenization'
'tokenizetransform': 'Tokenization',
'argument': 'Arguments',
'command': 'Commands',
}
return {k: titles[k] for k in kinds if k in titles}

@@ -107,6 +111,9 @@ def builds_components_pages(epkg):
def process_default(default_value):
if not default_value:
return ''
if "+" in default_value:
default_value = default_value.split(".")[-1].replace("+", ".")
return default_value
if len(default_value) > 28:
if len(default_value.split(".")) > 2:
default_value = default_value.replace(".", ". ")
@@ -124,6 +131,9 @@ def process_description(desc):
raise TypeError("desc must be a string not {0}".format(type(desc)))
return add_rst_links(desc, epkg)

def clean_name(name):
return name.replace(" ", "_").replace(".", "_").replace("(", "").replace(")", "").lower()

kinds = mlnet_components_kinds()
pages = {}

@@ -138,6 +148,21 @@ def process_description(desc):

MamlHelper = get_maml_helper()

# builds references
refs = {}
for v, k in sorted_kinds:
enumc = MamlHelper.EnumerateComponents(k)
try:
comps = list(enumc)
except Exception as e: # pylint: disable=W0703
logger.warning("[csml] issue with kind {0}\n{1}".format(k, e))
continue
if len(comps) == 0:
logger.warning("[csml] empty kind {0}\n{1}".format(k, e))
continue
for comp in comps:
refs[comp.Name] = ":ref:`l-{0}`".format(comp.Name.lower().replace(".", "-"))

# kinds and components
for v, k in sorted_kinds:
enumc = MamlHelper.EnumerateComponents(k)
@@ -147,26 +172,27 @@ def process_description(desc):
logger.warning("[csml] issue with kind {0}\n{1}".format(k, e))
continue
if len(comps) == 0:
logger.warning("[csml] empty kind {0}\n{1}".format(k, e))
continue

comp_names = list(sorted(c.Name.replace(" ", "_") for c in comps))
comp_names = list(sorted(clean_name(c.Name) for c in comps))
kind_name = v
kind_kind = k
pages[k] = kind_tpl.render(title=kind_name, fnames=comp_names, len=len)

for comp in comps:

if comp.Arguments is None:
if comp.Arguments is None and "version" not in comp.Name.lower():
logger.info(
"[csml] ---- SKIP ---- {}-{}-{}".format(k, comp.Name, comp.Description))
else:
assembly_name = comp.AssemblyName
args = {}
for arg in comp.Arguments:
args[arg.Name] = dict(Name=arg.Name, ShortName=arg.ShortName or '',
Default=process_default(
arg.DefaultValue),
Description=arg.Help)
if comp.Arguments is not None:
for arg in comp.Arguments:
dv = process_default(arg.DefaultValue)
args[arg.Name] = dict(Name=arg.Name, ShortName=arg.ShortName or '',
Default=refs.get(dv, dv), Description=arg.Help)
sorted_params = [v for k, v in sorted(args.items())]
aliases = ", ".join(comp.Aliases)

@@ -177,7 +203,7 @@ def process_description(desc):
else:
linkdocs = ""

comp_name = comp.Name.replace(" ", "_")
comp_name = clean_name(comp.Name)
pages[comp_name] = comp_tpl.render(title=comp.Name,
aliases=aliases,
summary=process_description(
@@ -205,8 +231,77 @@ def write_components_pages(app, env, docnames):
os.mkdir(dest)
for k, v in pages.items():
d = os.path.join(dest, k) + ".rst"
with open(d, "w", encoding="utf-8") as f:
f.write(v)
if os.path.exists(d):
with open(d, "r", encoding="utf-8") as f:
content = f.read()
else:
content = None

if content != v:
with open(d, "w", encoding="utf-8") as f:
f.write(v)


def get_mlnet_assemblies(chdir=False):
"""
Retrieves assemblies.
"""
if chdir:
cur = os.getcwd()
os.chdir(dll)
res = MamlHelper.GetLoadedAssembliesLocation(True)
if chdir:
os.chdir(cur)
dependencies = []
# addition = ["Core", "Data", "Maml", "Api"]
# root = os.path.dirname(res[0].Location)
# dependencies = [os.path.join(root, "Microsoft.ML.{0}.dll").format(a) for a in addition]
dependencies.extend([a for a in res if ".pyd" not in a and ".so" not in a])
usings = ["System", "System.Linq", "System.Collections.Generic", "System.IO",
"System.Text"]
usings.extend([
"Microsoft.ML.Runtime",
"Microsoft.ML.Runtime.Api",
"Microsoft.ML.Runtime.Data",
"Microsoft.ML.Runtime.Learners",
"Microsoft.ML.Runtime.Ensemble",
"Microsoft.ML.Runtime.LightGBM",
"Microsoft.ML.Runtime.Model.Onnx",
"Microsoft.ML.Runtime.TimeSeriesProcessing",
"Microsoft.ML.Runtime.Tools",
"Microsoft.ML.Trainers",
"Microsoft.ML.Trainers.HalLearners",
"Microsoft.ML.Trainers.KMeans",
"Microsoft.ML.Trainers.FastTree",
"Microsoft.ML.Trainers.Online",
"Microsoft.ML.Trainers.PCA",
"Microsoft.ML.Transforms",
"Microsoft.ML.Transforms.Categorical",
"Microsoft.ML.Transforms.Normalizers",
"Microsoft.ML.Transforms.Projections",
"Microsoft.ML.Transforms.TensorFlow",
"Microsoft.ML.Transforms.Text",
"Microsoft.ML.Runtime.Sweeper",
])
res = MamlHelper.GetAssemblies()
usings.extend([a.FullName.split(',')[0] for a in res if "Scikit" in a.FullName])
return dependencies, usings


class RunCSharpMLDirective(RunCSharpDirective):
"""
Implicits "and dependencies.
"""

def modify_script_before_running(self, script):
"""
The methods modifies the script to *csharpy* to
run :epkg:`C#` from :epkg:`Python`.
"""
if not hasattr(RunCSharpDirective, 'deps_using'):
RunCSharpDirective.deps_using = get_mlnet_assemblies()
dependencies, usings = RunCSharpMLDirective.deps_using
return self._modify_script_before_running(script, usings, dependencies)


def setup(app):
@@ -215,4 +310,5 @@ def setup(app):
"""
app.add_directive('mlcmd', MlCmdDirective)
app.connect("env-before-read-docs", write_components_pages)
app.add_directive('runcsharpml', RunCSharpMLDirective)
return {'version': sphinx.__display_version__, 'parallel_read_safe': True}
@@ -16,6 +16,9 @@
"""

mddocs_index_template_releases = """
.. _l-mlnet-releases-details:
ML.net releases details
=======================
@@ -26,8 +29,11 @@
"""

index_template = """
List of Machine Learning Components
===================================
.. _l-ml-net-components:
ML Components
=============
.. toctree::
:maxdepth: 1
@@ -48,7 +54,7 @@

component_template = """
.. _l-{{title.replace("(", "").replace(")", "").replace(" ", "-").lower()}}:
.. _l-{{title.replace("(", "").replace(")", "").replace(" ", "-").replace(".", "-").lower()}}:
{{title}}
{{"=" * len(title)}}

0 comments on commit 665f392

Please sign in to comment.
You can’t perform that action at this time.