removed Azure credentials

added python27 & 35 build without build env dependency
VowpalWabbit · Dec 19, 2016 · ca0e069 · ca0e069
1 parent fbfff76
commit ca0e069
Show file tree

Hide file tree

Showing 11 changed files with 332 additions and 30 deletions.
diff --git a/.gitignore b/.gitignore
@@ -120,3 +120,4 @@ vowpalwabbit/vw.VC.VC.opendb
 vowpalwabbit/vw.VC.db
 /cs/cs/Serializer/VowpalWabbitDefaultMarshallerExt.cs
 /python/pylibvw.pyd
+/python/pylibvw.cp35-win_amd64.pyd
diff --git a/cs/unittest/TestAzure.cs b/cs/unittest/TestAzure.cs
@@ -92,12 +92,44 @@ public byte[] JSONAsBytes
             public ActionFeatures[] Actions { get; set; }
         }
 
+        private static string GetConfiguration(string name)
+        {
+            var value = Environment.GetEnvironmentVariable(name);
+            if (!string.IsNullOrEmpty(value))
+                return value.Trim();
+
+            var path = Directory.GetCurrentDirectory();
+            do
+            {
+                var filename = Path.Combine(path, "vw_azure.config");
+                if (File.Exists(filename))
+                {
+                    var q = from line in File.ReadAllLines(filename)
+                            let m = Regex.Match(line, @"^(\S+)\s*=(.*)$")
+                            where m.Success
+                            where m.Groups[1].Value == name
+                            select m.Groups[2].Value;
+
+                    value = q.FirstOrDefault();
+                    if (!string.IsNullOrEmpty(value))
+                        return value.Trim();
+                }
+
+                var di = Directory.GetParent(path);
+                if (di == null)
+                    Assert.Fail($"Configuration variable '{name}' not found. Search for environment variable or vw_azure.config");
+
+                path = di.FullName;
+            }
+            while (true);
+        }
+
         [TestMethod]
         public async Task TestAzureTrainer()
         {
-            // TODO: move to config object, find from env or get from config file... 
-            var storageConnectionString = "DefaultEndpointsProtocol=https;AccountName=vwunit2storage;AccountKey=NLCjXurkixobS/vmnqk5e9D3y6GyJQmeWS2ZgYTQgTFCMH8m66Ii21tgTv9YbiKRRfMzxqOWKI332qs20ycXaw==";
-            var inputEventHubConnectionString = "Endpoint=sb://vwunit2sb.servicebus.windows.net/;SharedAccessKeyName=Manage;SharedAccessKey=1bH0yXVwsXMtRduwX7GWmDjUPOrW+wOi5bfLpzhIFSw=;EntityPath=vwunit2input";
+            var storageConnectionString = GetConfiguration("storageConnectionString");
+            var inputEventHubConnectionString = GetConfiguration("inputEventHubConnectionString");
+            var evalEventHubConnectionString = GetConfiguration("evalEventHubConnectionString");
 
             var trainArguments = "--cb_explore_adf --epsilon 0.2 -q ab";
 
@@ -117,7 +149,7 @@ public async Task TestAzureTrainer()
                 {
                     CheckpointPolicy = new CountingCheckpointPolicy(data.Count),
                     JoinedEventHubConnectionString = inputEventHubConnectionString,
-                    EvalEventHubConnectionString = "Endpoint=sb://vwunit2sb.servicebus.windows.net/;SharedAccessKeyName=Manage;SharedAccessKey=uGUK/DX35pRy77gGCT8RLiRKJUYoIII/VjIt2d1ukyI=;EntityPath=vwunit2eval",
+                    EvalEventHubConnectionString = evalEventHubConnectionString,
                     StorageConnectionString = storageConnectionString,
                     Metadata = new OnlineTrainerSettings
                     {

diff --git a/python/setup.py b/python/setup.py
@@ -100,8 +100,12 @@ def build_extension(self, ext):
         if not path.isdir(target_dir):
             makedirs(target_dir)
         if system == 'Windows':
-            copy(path.join(here, 'bin', 'pyvw.dll'), self.get_ext_fullpath(ext.name))
-            copy(path.join(here, 'bin', 'zlib.dll'), path.join(here, 'zlib.dll'))
+            if sys.version_info[0] == 2 and sys.version_info[1] == 7:
+               copy(path.join(here, 'bin', 'pyvw27.dll'), self.get_ext_fullpath(ext.name))
+            elif sys.version_info[0] == 3 and sys.version_info[1] == 5:
+               copy(path.join(here, 'bin', 'pyvw35.dll'), self.get_ext_fullpath(ext.name))
+            else:
+               raise Exception('Pre-built vw/python library for Windows is not supported')
         else:
             env = environ
             env['PYTHON_VERSION'] = '{v[0]}.{v[1]}'.format(v=sys.version_info)

diff --git a/python/packages.config → python/windows27/packages.config b/python/packages.config → python/windows27/packages.config
@@ -3,6 +3,7 @@
   <package id="boost" version="1.58.0.0" targetFramework="native" />
   <package id="boost_program_options-vc120" version="1.58.0.0" targetFramework="native" />
   <package id="boost_python" version="1.58.0.0" targetFramework="native" />
+  <package id="python2" version="2.7.12" targetFramework="native" />
   <package id="zlib" version="1.2.8.7" targetFramework="native" />
   <package id="zlib.redist" version="1.2.8.7" targetFramework="native" />
 </packages>
diff --git a/python/python.vcxproj → python/windows27/python27.vcxproj b/python/python.vcxproj → python/windows27/python27.vcxproj
@@ -14,6 +14,7 @@
     <ProjectGuid>{7D381808-CC76-4A1A-A8E6-3F71278858BB}</ProjectGuid>
     <RootNamespace>python</RootNamespace>
     <Keyword>Win32Proj</Keyword>
+    <ProjectName>python27</ProjectName>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
@@ -42,24 +43,24 @@
   </ImportGroup>
   <PropertyGroup Label="UserMacros" />
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
-    <TargetName>pyvw</TargetName>
+    <TargetName>pyvw27</TargetName>
     <OutDir>$(SolutionDir)..\python\bin\</OutDir>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
-    <TargetName>pyvw</TargetName>
+    <TargetName>pyvw27</TargetName>
     <OutDir>$(SolutionDir)..\python\bin\</OutDir>
   </PropertyGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
       <Optimization>Disabled</Optimization>
       <SDLCheck>true</SDLCheck>
-      <AdditionalIncludeDirectories>..\rapidjson\include;c:\python34\include</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(SolutionDir)..\rapidjson\include;$(SolutionDir)packages\python2.2.7.12\tools\include</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>_WINDLL;%(PreprocessorDefinitions);_WIN32</PreprocessorDefinitions>
     </ClCompile>
     <Link>
-      <AdditionalLibraryDirectories>c:\Python34\libs</AdditionalLibraryDirectories>
-      <AdditionalDependencies>kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;ws2_32.lib;%(AdditionalDependencies);python34.lib</AdditionalDependencies>
+      <AdditionalLibraryDirectories>$(SolutionDir)packages\python2.2.7.12\tools\libs</AdditionalLibraryDirectories>
+      <AdditionalDependencies>kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;ws2_32.lib;%(AdditionalDependencies);python27.lib</AdditionalDependencies>
     </Link>
     <PostBuildEvent>
       <Command>xcopy /y $(SolutionDir)$(Platform)\$(Configuration)\zlib.dll $(SolutionDir)..\python\bin\</Command>
@@ -72,46 +73,46 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>true</SDLCheck>
-      <AdditionalIncludeDirectories>..\rapidjson\include;c:\python34\include</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(SolutionDir)..\rapidjson\include;$(SolutionDir)packages\python2.2.7.12\tools\include</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>_WINDLL;%(PreprocessorDefinitions);_WIN32</PreprocessorDefinitions>
     </ClCompile>
     <Link>
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
       <OptimizeReferences>true</OptimizeReferences>
-      <AdditionalLibraryDirectories>c:\Python34\libs</AdditionalLibraryDirectories>
-      <AdditionalDependencies>kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;ws2_32.lib;%(AdditionalDependencies);python34.lib</AdditionalDependencies>
+      <AdditionalLibraryDirectories>$(SolutionDir)packages\python2.2.7.12\tools\libs</AdditionalLibraryDirectories>
+      <AdditionalDependencies>kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;ws2_32.lib;%(AdditionalDependencies);python27.lib</AdditionalDependencies>
     </Link>
     <PostBuildEvent>
       <Command>xcopy /y $(SolutionDir)$(Platform)\$(Configuration)\zlib.dll $(SolutionDir)..\python\bin\</Command>
     </PostBuildEvent>
   </ItemDefinitionGroup>
   <ItemGroup>
-    <ClCompile Include="pylibvw.cc" />
+    <ClCompile Include="..\pylibvw.cc" />
   </ItemGroup>
   <ItemGroup>
     <None Include="packages.config" />
   </ItemGroup>
   <ItemGroup>
-    <ProjectReference Include="..\vowpalwabbit\vw_dynamic.vcxproj">
+    <ProjectReference Include="$(SolutionDir)\vw_dynamic.vcxproj">
       <Project>{1e205806-7f80-47dd-a38d-fc08083f3593}</Project>
     </ProjectReference>
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
-    <Import Project="..\vowpalwabbit\packages\boost.1.58.0.0\build\native\boost.targets" Condition="Exists('..\vowpalwabbit\packages\boost.1.58.0.0\build\native\boost.targets')" />
-    <Import Project="..\vowpalwabbit\packages\boost_python.1.58.0.0\build\native\boost_python.targets" Condition="Exists('..\vowpalwabbit\packages\boost_python.1.58.0.0\build\native\boost_python.targets')" />
-    <Import Project="..\vowpalwabbit\packages\boost_program_options-vc120.1.58.0.0\build\native\boost_program_options-vc120.targets" Condition="Exists('..\vowpalwabbit\packages\boost_program_options-vc120.1.58.0.0\build\native\boost_program_options-vc120.targets')" />
-    <Import Project="..\vowpalwabbit\packages\zlib.redist.1.2.8.7\build\native\zlib.redist.targets" Condition="Exists('..\vowpalwabbit\packages\zlib.redist.1.2.8.7\build\native\zlib.redist.targets')" />
-    <Import Project="..\vowpalwabbit\packages\zlib.1.2.8.7\build\native\zlib.targets" Condition="Exists('..\vowpalwabbit\packages\zlib.1.2.8.7\build\native\zlib.targets')" />
+    <Import Project="$(SolutionDir)\packages\boost.1.58.0.0\build\native\boost.targets" Condition="Exists('$(SolutionDir)\packages\boost.1.58.0.0\build\native\boost.targets')" />
+    <Import Project="$(SolutionDir)\packages\boost_python.1.58.0.0\build\native\boost_python.targets" Condition="Exists('$(SolutionDir)\packages\boost_python.1.58.0.0\build\native\boost_python.targets')" />
+    <Import Project="$(SolutionDir)\packages\boost_program_options-vc120.1.58.0.0\build\native\boost_program_options-vc120.targets" Condition="Exists('$(SolutionDir)\packages\boost_program_options-vc120.1.58.0.0\build\native\boost_program_options-vc120.targets')" />
+    <Import Project="$(SolutionDir)\packages\zlib.redist.1.2.8.7\build\native\zlib.redist.targets" Condition="Exists('$(SolutionDir)\packages\zlib.redist.1.2.8.7\build\native\zlib.redist.targets')" />
+    <Import Project="$(SolutionDir)\packages\zlib.1.2.8.7\build\native\zlib.targets" Condition="Exists('$(SolutionDir)\packages\zlib.1.2.8.7\build\native\zlib.targets')" />
   </ImportGroup>
   <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
     <PropertyGroup>
       <ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
     </PropertyGroup>
-    <Error Condition="!Exists('..\vowpalwabbit\packages\boost.1.58.0.0\build\native\boost.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\vowpalwabbit\packages\boost.1.58.0.0\build\native\boost.targets'))" />
-    <Error Condition="!Exists('..\vowpalwabbit\packages\boost_python.1.58.0.0\build\native\boost_python.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\vowpalwabbit\packages\boost_python.1.58.0.0\build\native\boost_python.targets'))" />
-    <Error Condition="!Exists('..\vowpalwabbit\packages\boost_program_options-vc120.1.58.0.0\build\native\boost_program_options-vc120.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\vowpalwabbit\packages\boost_program_options-vc120.1.58.0.0\build\native\boost_program_options-vc120.targets'))" />
-    <Error Condition="!Exists('..\vowpalwabbit\packages\zlib.redist.1.2.8.7\build\native\zlib.redist.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\vowpalwabbit\packages\zlib.redist.1.2.8.7\build\native\zlib.redist.targets'))" />
-    <Error Condition="!Exists('..\vowpalwabbit\packages\zlib.1.2.8.7\build\native\zlib.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\vowpalwabbit\packages\zlib.1.2.8.7\build\native\zlib.targets'))" />
+    <Error Condition="!Exists('$(SolutionDir)\packages\boost.1.58.0.0\build\native\boost.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\packages\boost.1.58.0.0\build\native\boost.targets'))" />
+    <Error Condition="!Exists('$(SolutionDir)\packages\boost_python.1.58.0.0\build\native\boost_python.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\packages\boost_python.1.58.0.0\build\native\boost_python.targets'))" />
+    <Error Condition="!Exists('$(SolutionDir)\packages\boost_program_options-vc120.1.58.0.0\build\native\boost_program_options-vc120.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\packages\boost_program_options-vc120.1.58.0.0\build\native\boost_program_options-vc120.targets'))" />
+    <Error Condition="!Exists('$(SolutionDir)\packages\zlib.redist.1.2.8.7\build\native\zlib.redist.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\packages\zlib.redist.1.2.8.7\build\native\zlib.redist.targets'))" />
+    <Error Condition="!Exists('$(SolutionDir)\packages\zlib.1.2.8.7\build\native\zlib.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\packages\zlib.1.2.8.7\build\native\zlib.targets'))" />
   </Target>
 </Project>
diff --git a/python/python.vcxproj.filters → python/windows27/python27.vcxproj.filters b/python/python.vcxproj.filters → python/windows27/python27.vcxproj.filters
@@ -15,9 +15,6 @@
     </Filter>
   </ItemGroup>
   <ItemGroup>
-    <ClCompile Include="pylibvw.cc">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="$(MSBuildThisFileDirectory)..\..\lib\native\src\converter\boost_python.converter.arg_to_python_base.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
@@ -102,6 +99,9 @@
     <ClCompile Include="$(MSBuildThisFileDirectory)..\..\lib\native\src\boost_python.wrapper.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\pylibvw.cc">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <None Include="packages.config" />

diff --git a/python/windows35/packages.config b/python/windows35/packages.config
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<packages>
+  <package id="boost" version="1.58.0.0" targetFramework="native" />
+  <package id="boost_program_options-vc120" version="1.58.0.0" targetFramework="native" />
+  <package id="boost_python" version="1.58.0.0" targetFramework="native" />
+  <package id="python" version="3.5.2.2" targetFramework="native" />
+  <package id="zlib" version="1.2.8.7" targetFramework="native" />
+  <package id="zlib.redist" version="1.2.8.7" targetFramework="native" />
+</packages>