Permalink
Browse files

Bi-annual PR (#1270)

* added trace_message listener to tunnel tracing through API

* fix floating point formatting

* fixed formatting

* exposing tracing information to C#

* fixed deduplicating command line arguments
towards --save_resume fix

* fixed azure trainer repro

* fix memory leak for seeded model
handling disagreeing arguments by starting a fresh model
commented

* localize best_constant tracking state (first_observed_label,...)
added documentation

* fix offline repro with save_resume and preserve_performance_counters
revert zeroing of perf counters on save

* adapted test case

* version bump

* included azure unit tests in VSO build

* Disable azure trainer on VSO as it can't run elevated.

* improve python setup error message
fix --bag when having duplicate options
improve json parsing failure handling

* nuget version bump

* restore nuget packages before build to overcome race condition

* optimized JSON direct - JSON string handling
added support for List<string> of JSON in JSON direct mode
reseting current pass in core VW if preserve_performance_counters is omitted

* added event id and timestamp to evaluation result
changed partitioning to match joined data partitioning

* added support for Dictionary<string, float[]> marshalling
added support for nested JSON objects, last property name in path will be used as namespace
fixed JSON null handling

* nuget version bump

* fixed embedded JSON in JSON direct mode
supporting JSON objects in arrays

* added --fixed_weight to marginal (supporting biased marginal feature computation)

* added missing exampel id & timestamp copy
version bump

* fix evaluation ouput
version bump

* fixed marginal bool options

* marginal +1 fix

* free reference to memory stream obj

* fix warning
added EventHub consumergroup support

* added solutiondir to csproj

* removed rapidjson submodule

* Updated reference paths.

* removed gtest submodule

* VS2015 + CNTK starting integration

* fixed release segfault w/ dup parameters

* fixed proj paths

* fixed proj paths

* removed framework

* fixed proj files

* fixed paths

* fixed paths

* fixing linux build

* fixed paths

* use T4 from VS2015

* switch T4 transform

* avoid barfing on null values

* improved exception reporting

* allow for split parsing of features and label for JSON

* validate sub reader parsing

* fixed arrays with null

* expose Label update post example create

* bound VW example memory consumption

* added DecisionService style JSON parsing

* fixed parsing of properties pass "c" context

* fixed JSON parsing (\0 termination missing)

* fixed null handling in array

* initialize member in ctor

* fixed marginal memory leak

* fix gzip decompression
remove cntk

* added missing zlib.cpp

* fixed gzip support in VS2015
switched back to zlib c-interface as boost added 30% overhead

* fixe json parsing (returned to DS state)

* removed vw_static project

* removed boost-iostream dependency
removed unused C# fil

* updated nuget.spec to skip zlib copy gymnastics
removed python3.5 & python 3.6 for now as boost-python is linked against python 2.7
  • Loading branch information...
eisber authored and JohnLangford committed Jul 15, 2017
1 parent bd42362 commit 14f585a6d4bb278b6619b8462f004542b3392896
Showing with 2,678 additions and 2,144 deletions.
  1. +1 −0 appveyor.yml
  2. +7 −7 c_test/c_test.vcxproj
  3. +8 −12 cluster/cluster.vcxproj
  4. +52 −0 cs/azure/Join/QueueDictionary.cs
  5. +2 −2 cs/azure/Properties/AssemblyInfo.cs
  6. +0 −118 cs/azure/System.Buffers/ArrayPool.cs
  7. +0 −78 cs/azure/System.Buffers/ArrayPoolEventSource.cs
  8. +0 −161 cs/azure/System.Buffers/DefaultArrayPool.cs
  9. +0 −115 cs/azure/System.Buffers/DefaultArrayPoolBucket.cs
  10. +0 −37 cs/azure/System.Buffers/Utilities.cs
  11. +2 −0 cs/azure/Trainer/Data/PipelineData.cs
  12. +6 −0 cs/azure/Trainer/Data/TrainerResult.cs
  13. +2 −1 cs/azure/Trainer/LearnEventProcessorFactory.cs
  14. +1 −1 cs/azure/Trainer/LearnEventProcessorHost.cs
  15. +6 −0 cs/azure/Trainer/OnlineTrainerSettingsInternal.cs
  16. +92 −38 cs/azure/Trainer/Operations/EvalOperation.cs
  17. +1 −1 cs/azure/Trainer/Operations/LatencyOperation.cs
  18. +6 −3 cs/azure/Trainer/Operations/LearnOperation.cs
  19. +29 −40 cs/azure/azure.csproj
  20. +4 −0 cs/azure_worker/OnlineTrainerSettingsWatcher.cs
  21. +2 −2 cs/azure_worker/Properties/AssemblyInfo.cs
  22. +2 −2 cs/cli/AssemblyInfo.cpp
  23. BIN cs/cli/Resource.rc
  24. +5 −5 cs/cli/cs_vld.vcxproj
  25. +66 −1 cs/cli/vowpalwabbit.cpp
  26. +12 −1 cs/cli/vowpalwabbit.h
  27. +5 −11 cs/cli/vw_base.cpp
  28. +0 −2 cs/cli/vw_base.h
  29. +103 −26 cs/cli/vw_clr.vcxproj
  30. +12 −1 cs/cli/vw_example.cpp
  31. +2 −1 cs/cli/vw_example.h
  32. +24 −7 cs/common/Bag.cs
  33. +36 −0 cs/common/VowpalWabbitDecisionServiceInteractionHeader.cs
  34. +5 −3 cs/common/vw_common.csproj
  35. +0 −4 cs/cs/Serializer/VowpalWabbitDefaultMarshaller.cs
  36. +61 −18 cs/cs/Serializer/VowpalWabbitDefaultMarshallerExt.tt
  37. +35 −14 cs/cs/Serializer/VowpalWabbitSingleExampleSerializerCompiler.cs
  38. +0 −10 cs/cs/VowpalWabbit.targets
  39. +39 −0 cs/cs/VowpalWabbitSerializationException.cs
  40. +10 −12 cs/cs/cs.csproj
  41. +0 −5 cs/cs/cs.nuspec
  42. +1 −0 cs/cs/packages.config
  43. +5 −5 cs/cs_console/App.config
  44. +2 −1 cs/cs_console/cs_console.csproj
  45. +26 −0 cs/cs_json/Serializer/IVowpalWabbitJsonConverter.cs
  46. +19 −6 cs/cs_json/Serializer/JsonRawStringConverter.cs
  47. +66 −0 cs/cs_json/Serializer/JsonRawStringListConverter.cs
  48. +19 −7 cs/cs_json/Serializer/JsonTypeInspector.cs
  49. +63 −24 cs/cs_json/Serializer/VowpalWabbitJsonBuilder.cs
  50. +65 −0 cs/cs_json/Serializer/VowpalWabbitJsonOptimizedSerializable.cs
  51. +4 −1 cs/cs_json/Serializer/{VowpalWabbitJsonConverter.cs → VowpalWabbitJsonSerializable.cs}
  52. +20 −2 cs/cs_json/Serializer/VowpalWabbitJsonSerializer.cs
  53. +14 −9 cs/cs_json/cs_json.csproj
  54. +2 −1 cs/cs_parallel/cs_parallel.csproj
  55. +3 −2 cs/leaktest/cs_leaktest.csproj
  56. +0 −1 cs/setup/Product.wxs
  57. +3 −2 cs/testcommon/cs_testcommon.csproj
  58. +0 −27 cs/transform_all.bat
  59. +174 −11 cs/unittest/TestJson.cs
  60. +101 −11 cs/unittest/TestJsonDirect.cs
  61. +123 −266 cs/unittest/TestMarshalNumeric.cs
  62. +3 −1 cs/unittest/TestMarshalNumeric.tt
  63. +77 −1 cs/unittest/VowpalWabbitExampleJsonValidator.cs
  64. +32 −1 cs/unittest/VowpalWabbitExampleValidator.cs
  65. +24 −26 cs/unittest/cs_unittest.csproj
  66. +1 −0 cs/unittest/packages.config
  67. +1 −1 cs/version.props
  68. +7 −7 deploy_vw/deploy_vw.vcxproj
  69. +16 −13 python/pylibvw.cc
  70. +2 −0 python/setup.py
  71. +41 −6 python/windows27/packages.config
  72. +92 −26 python/windows27/python27.vcxproj
  73. +0 −84 python/windows27/python27.vcxproj.filters
  74. +41 −6 python/windows35/packages.config
  75. +86 −20 python/windows35/python35.vcxproj
  76. +0 −84 python/windows35/python35.vcxproj.filters
  77. +46 −0 python/windows36/packages.config
  78. +185 −0 python/windows36/python36.vcxproj
  79. +25 −0 python/windows36/python36.vcxproj.filters
  80. +0 −3 rapidjson/.gitmodules
  81. +0 −1 rapidjson/thirdparty/gtest
  82. +1 −1 vowpalwabbit/accumulate.cc
  83. +8 −2 vowpalwabbit/crossplat_compat.h
  84. +1 −1 vowpalwabbit/ftrl.cc
  85. +1 −0 vowpalwabbit/gd.cc
  86. +1 −1 vowpalwabbit/gd_mf.cc
  87. +4 −5 vowpalwabbit/global_data.cc
  88. +3 −3 vowpalwabbit/lda_core.cc
  89. +93 −13 vowpalwabbit/libvw.vcxproj
  90. +6 −6 vowpalwabbit/main.cc
  91. +8 −5 vowpalwabbit/marginal.cc
  92. +42 −6 vowpalwabbit/packages.config
  93. +6 −4 vowpalwabbit/parse_args.cc
  94. +250 −33 vowpalwabbit/parse_example_json.h
  95. +1 −1 vowpalwabbit/search_multiclasstask.cc
  96. +1 −1 vowpalwabbit/svrg.cc
  97. +3 −43 vowpalwabbit/vw.sln
  98. +101 −30 vowpalwabbit/vw.vcxproj
  99. +96 −25 vowpalwabbit/vw_dynamic.vcxproj
  100. +0 −548 vowpalwabbit/vw_static.vcxproj
  101. +25 −21 vowpalwabbit/vwdll.cpp
@@ -11,6 +11,7 @@ build_script:
- vowpalwabbit\.nuget\nuget install -o vowpalwabbit\packages cs\cs\packages.config
- vowpalwabbit\.nuget\nuget install -o vowpalwabbit\packages cs\azure\packages.config
- vowpalwabbit\.nuget\nuget install -o vowpalwabbit\packages cs\azure_worker\packages.config
- vowpalwabbit\.nuget\nuget install -o vowpalwabbit\packages cs\setup_bundle\packages.config
- vowpalwabbit\.nuget\nuget install -o vowpalwabbit\packages cs\unittest\packages.config
- vowpalwabbit\.nuget\nuget install -o vowpalwabbit\packages python\windows27\packages.config
- vowpalwabbit\.nuget\nuget install -o vowpalwabbit\packages python\windows35\packages.config
@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="DebugLeakCheck|Win32">
<Configuration>DebugLeakCheck</Configuration>
@@ -36,39 +36,39 @@
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>MultiByte</CharacterSet>
<PlatformToolset>v120</PlatformToolset>
<PlatformToolset>v140</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='DebugLeakCheck|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>MultiByte</CharacterSet>
<PlatformToolset>v120</PlatformToolset>
<PlatformToolset>v140</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>MultiByte</CharacterSet>
<PlatformToolset>v120</PlatformToolset>
<PlatformToolset>v140</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='DebugLeakCheck|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>MultiByte</CharacterSet>
<PlatformToolset>v120</PlatformToolset>
<PlatformToolset>v140</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
<PlatformToolset>v120</PlatformToolset>
<PlatformToolset>v140</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
<PlatformToolset>v120</PlatformToolset>
<PlatformToolset>v140</PlatformToolset>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="DebugLeakCheck|Win32">
<Configuration>DebugLeakCheck</Configuration>
@@ -32,45 +32,46 @@
<RootNamespace>cluster</RootNamespace>
<ProjectName>spanning_tree</ProjectName>
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\vowpalwabbit\</SolutionDir>
<WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v120</PlatformToolset>
<PlatformToolset>v140</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='DebugLeakCheck|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v120</PlatformToolset>
<PlatformToolset>v140</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v120</PlatformToolset>
<PlatformToolset>v140</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='DebugLeakCheck|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v120</PlatformToolset>
<PlatformToolset>v140</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v120</PlatformToolset>
<PlatformToolset>v140</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v120</PlatformToolset>
<PlatformToolset>v140</PlatformToolset>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<Import Project="$(SolutionDir)Build.props" />
@@ -252,11 +253,6 @@
<ClCompile Include="..\vowpalwabbit\vw_exception.cc" />
<ClCompile Include="spanning_tree_main.cc" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\vowpalwabbit\vw_static.vcxproj">
<Project>{1e205806-7f80-47dd-a38d-fc08083f3592}</Project>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\vowpalwabbit\spanning_tree.h" />
<ClInclude Include="..\vowpalwabbit\vw_exception.h" />
@@ -0,0 +1,52 @@
using System;
using System.Collections.Generic;

namespace VowpalWabbit.Azure.Join
{
public class QueueDictionary<TKey, TValue>
{
public class Item
{
public TKey Key { get; set; }

public TValue Value { get; set; }
}

private readonly LinkedList<Item> queue = new LinkedList<Item>();

private readonly Dictionary<TKey, LinkedListNode<Item>> index = new Dictionary<TKey, LinkedListNode<Item>>();

public IEnumerable<Item> DequeueIf(Predicate<TValue> pred)
{
while (queue.First != null && pred(queue.First.Value.Value))
{
var item = queue.First.Value;

queue.RemoveFirst();
index.Remove(item.Key);

yield return item;
}
}

public TValue Remove(TKey key)
{
LinkedListNode<Item> node;
if (!index.TryGetValue(key, out node))
return default(TValue);

var value = node.Value.Value;

queue.Remove(node);
index.Remove(key);

return value;
}

public void Enqueue(TKey key, TValue value)
{
var node = queue.AddLast(new Item { Key = key, Value = value });
index.Add(key, node);
}
}
}
@@ -17,7 +17,7 @@
[assembly: System.Runtime.InteropServices.ComVisible(false)]
[assembly: System.CLSCompliant(false)]
[assembly: System.Runtime.InteropServices.Guid("6a577997-af00-4ca0-8453-fdc8bbdf2a57")]
[assembly: System.Reflection.AssemblyVersion("8.2.0.17")]
[assembly: System.Reflection.AssemblyFileVersion("8.2.0.17")]
[assembly: System.Reflection.AssemblyVersion("8.3.0.9")]
[assembly: System.Reflection.AssemblyFileVersion("8.3.0.9")]


This file was deleted.

Oops, something went wrong.

This file was deleted.

Oops, something went wrong.
Oops, something went wrong.

0 comments on commit 14f585a

Please sign in to comment.