Skip to content

Commit

Permalink
The -test command for windows. Nuget packages (#5464)
Browse files Browse the repository at this point in the history
* working on testing

* testing updates

* tests almost working

* build changes

* all tests should be working

* changes from PR comments

* fixes for .net 3.1

* Fixed extension check. Removed <PackageId> where not needed

* Removed pkg folder and updated paths.
  • Loading branch information
michaelgsharp committed Nov 18, 2020
1 parent 3d0a760 commit acd873f
Show file tree
Hide file tree
Showing 106 changed files with 591 additions and 732 deletions.
13 changes: 6 additions & 7 deletions Directory.Build.props
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,8 @@

<PropertyGroup>
<Copyright>$(CopyrightNetFoundation)</Copyright>
<PackageLicenseExpression>MIT</PackageLicenseExpression>
<TreatWarningsAsErrors>True</TreatWarningsAsErrors>
<DebugType>embedded</DebugType>
<DebugType>portable</DebugType>
<DebugSymbols>true</DebugSymbols>
<LangVersion>latest</LangVersion>
</PropertyGroup>
Expand All @@ -31,7 +30,7 @@
</PropertyGroup>

<PropertyGroup>
<PackageAssetsPath>$(ArtifactsDir)/pkgassets/</PackageAssetsPath>
<PackageAssetsPath>$(ArtifactsDir)pkgassets/</PackageAssetsPath>
<PkgDir>$(RepoRoot)pkg/</PkgDir>
</PropertyGroup>

Expand All @@ -49,17 +48,17 @@
<PropertyGroup>
<!-- Optional: Publish the repository URL in the built .nupkg (in the NuSpec <Repository> element) -->
<PublishRepositoryUrl>true</PublishRepositoryUrl>

<!-- Optional: Embed source files that are not tracked by the source control manager in the PDB -->
<EmbedUntrackedSources>true</EmbedUntrackedSources>

<!-- Optional: Build symbol package (.snupkg) to distribute the PDB containing Source Link -->
<IncludeSymbols>true</IncludeSymbols>
<SymbolPackageFormat>snupkg</SymbolPackageFormat>
</PropertyGroup>

<PropertyGroup Condition="'$(DotNetBuildFromSource)' == 'true'">
<!--
<!--
When building using source-build the process is:
- Newtonsoft.Json versions 9.0.1 and 12.0.2 are built by source-build
- Version 12.0.2 is written to Version.props
Expand Down
35 changes: 35 additions & 0 deletions Directory.Build.targets
Original file line number Diff line number Diff line change
@@ -1,4 +1,39 @@
<?xml version="1.0" encoding="utf-8"?>
<Project>
<Import Project="Sdk.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />

<Target Name="CopyNativeAssembiles" AfterTargets="CopyFilesToOutputDirectory">
<PropertyGroup>
<LibPrefix Condition="'$(OS)' != 'Windows_NT'">lib</LibPrefix>
<LibExtension Condition="'$(OS)' == 'Windows_NT'">.dll</LibExtension>
<LibExtension Condition="'$(OS)' != 'Windows_NT'">.so</LibExtension>
<LibExtension Condition="$([MSBuild]::IsOSPlatform('osx'))">.dylib</LibExtension>

<TargetArchitecture Condition="'$(Platform)' == ''">x64</TargetArchitecture>
<NativeTargetArchitecture Condition="'$(NativeTargetArchitecture)' == ''">$(TargetArchitecture)</NativeTargetArchitecture>
<BinDir Condition="'$(BinDir)'==''">$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin'))</BinDir>
<NativeOutputPath>$(BinDir)Native\$(NativeTargetArchitecture).$(Configuration)\</NativeOutputPath>

<Platform Condition="'$(Platform)'==''">AnyCPU</Platform>
<PlatformConfig>$(Platform).$(Configuration)</PlatformConfig>
<OutputPath Condition="'$(OutputPath)'==''">$(BinDir)$(MSBuildProjectName)\Debug</OutputPath>
</PropertyGroup>

<ItemGroup>
<NativeAssemblyReference>
<FullAssemblyPath>$(NativeOutputPath)$(LibPrefix)%(NativeAssemblyReference.Identity)$(LibExtension)</FullAssemblyPath>
</NativeAssemblyReference>
</ItemGroup>

<Copy SourceFiles = "@(NativeAssemblyReference->'%(FullAssemblyPath)')"
DestinationFolder="$(OutDir)"
OverwriteReadOnlyFiles="$(OverwriteReadOnlyFiles)"
Retries="$(CopyRetryCount)"
RetryDelayMilliseconds="$(CopyRetryDelayMilliseconds)"
UseHardlinksIfPossible="$(CreateHardLinksForPublishFilesIfPossible)"
UseSymboliclinksIfPossible="$(CreateSymbolicLinksForPublishFilesIfPossible)">
<Output TaskParameter="DestinationFiles" ItemName="FileWrites"/>
</Copy>
</Target>

</Project>
128 changes: 84 additions & 44 deletions Microsoft.ML.sln

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
done
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"

"$DIR/eng/common/build.sh" --restore --build --pack --warnAsError false
"$DIR/eng/common/build.sh" --restore --build --pack --warnAsError false "$@"
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,14 @@

<ItemGroup>
<ProjectReference Include="..\..\..\src\Microsoft.ML.AutoML\Microsoft.ML.AutoML.csproj" />
<ProjectReference Include="..\..\..\src\Microsoft.ML.Core\Microsoft.ML.Core.csproj" >
<PrivateAssets>all</PrivateAssets>
</ProjectReference>

<ProjectReference Include="..\..\..\src\Microsoft.ML.Transforms\Microsoft.ML.Transforms.csproj" >
<PrivateAssets>all</PrivateAssets>
</ProjectReference>

<NativeAssemblyReference Include="MatrixFactorizationNative" />
<NativeAssemblyReference Include="FastTreeNative" />
</ItemGroup>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,13 @@

<ItemGroup>
<ProjectReference Include="..\..\..\src\Microsoft.ML.Vision\Microsoft.ML.Vision.csproj" />
<ProjectReference Include="..\..\..\src\Microsoft.ML.ImageAnalytics\Microsoft.ML.ImageAnalytics.csproj" />
<ProjectReference Include="..\..\..\src\Microsoft.ML.Data\Microsoft.ML.Data.csproj" />
<ProjectReference Include="..\..\..\src\Microsoft.ML.LightGbm\Microsoft.ML.LightGbm.csproj" />
<ProjectReference Include="..\..\..\src\Microsoft.ML.TensorFlow\Microsoft.ML.TensorFlow.csproj" />
<ProjectReference Include="..\..\..\src\Microsoft.ML.SamplesUtils\Microsoft.ML.SamplesUtils.csproj" />
<ProjectReference Include="..\..\..\src\Microsoft.ML.StandardTrainers\Microsoft.ML.StandardTrainers.csproj" />
<ProjectReference Include="..\..\..\src\Microsoft.ML.Transforms\Microsoft.ML.Transforms.csproj" />

<NativeAssemblyReference Include="CpuMathNative" />
<NativeAssemblyReference Include="FastTreeNative" />
Expand Down
3 changes: 3 additions & 0 deletions docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@
<ProjectReference Include="..\..\..\src\Microsoft.ML.DnnImageFeaturizer.ResNet18\Microsoft.ML.DnnImageFeaturizer.ResNet18.csproj" />
<ProjectReference Include="..\..\..\src\Microsoft.ML.Transforms\Microsoft.ML.Transforms.csproj" />
<ProjectReference Include="..\..\..\src\Microsoft.ML.OnnxConverter\Microsoft.ML.OnnxConverter.csproj" />
<ProjectReference Include="..\..\..\src\Microsoft.ML.ImageAnalytics\Microsoft.ML.ImageAnalytics.csproj" />
<ProjectReference Include="..\..\..\src\Microsoft.ML.Core\Microsoft.ML.Core.csproj" />
<ProjectReference Include="..\..\..\src\Microsoft.ML.Data\Microsoft.ML.Data.csproj" />

<NativeAssemblyReference Include="CpuMathNative" />
<NativeAssemblyReference Include="FastTreeNative" />
Expand Down
9 changes: 9 additions & 0 deletions eng/Build.props
Original file line number Diff line number Diff line change
@@ -1,8 +1,17 @@
<?xml version="1.0" encoding="utf-8"?>
<Project>

<ItemDefinitionGroup>
<ProjectToBuild>
<RestoreInParallel>true</RestoreInParallel>
<BuildInParallel>false</BuildInParallel>
</ProjectToBuild>
</ItemDefinitionGroup>

<ItemGroup>
<ProjectToBuild Include="$(RepoRoot)src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj" />
<ProjectToBuild Include="$(RepoRoot)src/Native/Native.proj" />
<ProjectToBuild Include="$(RepoRoot)Microsoft.ML.sln" />
</ItemGroup>

</Project>
11 changes: 6 additions & 5 deletions eng/Versions.props
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
<Project>
<PropertyGroup>
<!-- This repo version -->
<DotNetUseShippingVersions>true</DotNetUseShippingVersions>
<DotNetUseShippingVersions>false</DotNetUseShippingVersions>
<VersionPrefix>1.5.3</VersionPrefix>
<PreReleaseVersionLabel></PreReleaseVersionLabel>
<PreReleaseVersionLabel>dev</PreReleaseVersionLabel>

<!--ML.NET Core dependencies-->
<NewtonsoftJsonPackageVersion>10.0.3</NewtonsoftJsonPackageVersion>
Expand Down Expand Up @@ -37,7 +37,7 @@
<ReportGeneratorVersion>4.3.6</ReportGeneratorVersion>
<MicrosoftDotNetApiCompatPackageVersion>1.0.0-beta.19225.5</MicrosoftDotNetApiCompatPackageVersion>
<MicrosoftSourceLinkVersion>1.1.0-beta-20206-02</MicrosoftSourceLinkVersion>

<!-- Test-only Dependencies -->
<BenchmarkDotNetVersion>0.12.0</BenchmarkDotNetVersion>
<MicrosoftCodeAnalysisTestingVersion>1.0.1-beta1.20080.1</MicrosoftCodeAnalysisTestingVersion>
Expand Down Expand Up @@ -77,6 +77,7 @@
<MicrosoftExtensionsDependencyModelVersion>2.1.0</MicrosoftExtensionsDependencyModelVersion>
<MicrosoftExtensionsFileSystemGlobbingVersion>2.0.0</MicrosoftExtensionsFileSystemGlobbingVersion>
<MicrosoftNETCorePlatformsVersion>2.1.0</MicrosoftNETCorePlatformsVersion>
<MicrosoftNETCore3PlatformsVersion>3.1.0</MicrosoftNETCore3PlatformsVersion>
<MicrosoftNetCompilersToolsetVersion>3.8.0-3.20460.2</MicrosoftNetCompilersToolsetVersion>
<MoqVersion>4.8.3</MoqVersion>
<MonoOptionsVersion>5.3.0.1</MonoOptionsVersion>
Expand All @@ -103,9 +104,9 @@
<SystemThreadingTasksExtensionVersion>4.5.2</SystemThreadingTasksExtensionVersion>
<SystemValueTupleVersion>4.4.0</SystemValueTupleVersion>
<WindowsAzureStorageVersion>8.5.0</WindowsAzureStorageVersion>
<XUnitVersion>2.4.1</XUnitVersion>
<XUnitVersion>2.4.0</XUnitVersion>
<XUnitAbstractionsVersion>2.0.3</XUnitAbstractionsVersion>
<XUnitVSRunnerVersion>2.4.1</XUnitVSRunnerVersion>
<XUnitVSRunnerVersion>2.4.0</XUnitVSRunnerVersion>
<MicrosoftDotNetBuildTasksFeedVersion>5.0.0-beta.20461.7</MicrosoftDotNetBuildTasksFeedVersion>
<MicrosoftDotNetSignToolVersion>5.0.0-beta.20461.7</MicrosoftDotNetSignToolVersion>
<MicrosoftAzureDocumentDBVersion>1.22.0</MicrosoftAzureDocumentDBVersion>
Expand Down
File renamed without changes.
File renamed without changes.
78 changes: 78 additions & 0 deletions eng/pkg/Pack.props
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
<Project>

<PropertyGroup>
<PackageAssetsPath>$(ArtifactsDir)pkgassets/</PackageAssetsPath>
<IncludeSymbols>true</IncludeSymbols>
<SymbolPackageFormat>snupkg</SymbolPackageFormat>
<IsSymbolsPackage Condition="$(MSBuildProjectName.Contains('.snupkg'))">true</IsSymbolsPackage>
<PackageIdFolderName>$(MSBuildProjectName.Replace('.symbols', ''))</PackageIdFolderName>
<IncludeBuildOutput>true</IncludeBuildOutput>
<IsPackable>true</IsPackable>

</PropertyGroup>

<!-- nuspec properties -->
<PropertyGroup>
<Authors>Microsoft</Authors>
<PackageLicenseFile>LICENSE</PackageLicenseFile>
<PackageProjectUrl>https://dot.net/ml</PackageProjectUrl>
<PackageIcon>mlnetlogo.png</PackageIcon>
<PackageReleaseNotes>https://aka.ms/mlnetreleasenotes</PackageReleaseNotes>
<!-- space separated -->
<PackageTags>ML.NET ML Machine Learning</PackageTags>

<LibExtension Condition="'$(OS)' == 'Windows_NT'">.dll</LibExtension>
<LibExtension Condition="'$(OS)' != 'Windows_NT'">.so</LibExtension>
<LibExtension Condition="$([MSBuild]::IsOSPlatform('osx'))">.dylib</LibExtension>
</PropertyGroup>

<ItemGroup>

<!-- Create symbol packages correctly by copying the library and xml to the nuget. Then the pdb is auto put in the symbols package-->
<Content Include="$(PackageAssetsPath)$(PackageIdFolderName)\lib\**\*" PackagePath="lib" />
<Content Include="$(PackageAssetsPath)$(PackageIdFolderName)\analyzers\**\*" PackagePath="analyzers" />
<Content Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\**\*" PackagePath="runtimes" />
<Content Include="$(PackageAssetsPath)$(PackageIdFolderName)\tools\**\*" PackagePath="tools" />

<Content Include="$(RepoRoot)eng\pkg\mlnetlogo.png" Pack="true" PackagePath="" />
</ItemGroup>

<ItemGroup Condition="'$(IncludeMLNetNotices)' != 'false'">
<Content Include="$(RepoRoot)\THIRD-PARTY-NOTICES.TXT" Pack="true" PackagePath="" />
<Content Include="$(RepoRoot)\LICENSE" Pack="true" PackagePath=""/>
</ItemGroup>

<!-- This check does not work corectly anymore and also removes the PDB from the symbols packages.
Need to figure out the correct way to do this, but since the normal packages are building correctly
I am leaving this as is for now. -->
<ItemGroup Condition="'$(IsSymbolsPackage)' != 'true'">
<Content Remove="$(PackageAssetsPath)$(PackageIdFolderName)\**\*.pdb" />
<Content Remove="$(PackageAssetsPath)$(PackageIdFolderName)\**\*.dwarf" />
<Content Remove="$(PackageAssetsPath)$(PackageIdFolderName)\**\*.dbg" />
</ItemGroup>

<ItemGroup Condition="Exists('packages.config') OR
Exists('$(MSBuildProjectName).packages.config') OR
Exists('packages.$(MSBuildProjectName).config')">
<Content Include="$(MSBuildThisFileDirectory)\..\..\runtimes\win-x64\native\*.dll"
Condition="'$(PlatformTarget)' == 'x64'">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
<Visible>false</Visible>
<Link>%(Filename)%(Extension)</Link>
</Content>
<Content Include="$(MSBuildThisFileDirectory)\..\..\runtimes\win-x86\native\*.dll"
Condition="'$(PlatformTarget)' == 'x86'">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
<Visible>false</Visible>
<Link>%(Filename)%(Extension)</Link>
</Content>
</ItemGroup>

<!-- Work around https://github.com/NuGet/Home/issues/6091 -->
<ItemDefinitionGroup>
<PackageReference>
<PrivateAssets>None</PrivateAssets>
</PackageReference>
</ItemDefinitionGroup>

</Project>
File renamed without changes.
File renamed without changes
23 changes: 13 additions & 10 deletions global.json
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
{
"tools": {
"dotnet": "5.0.100-rc.1.20452.10"
},
"msbuild-sdks": {
"Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20461.7",
"Microsoft.DotNet.Helix.Sdk": "5.0.0-beta.20461.7",
"Microsoft.Build.Traversal": "2.1.1",
"Microsoft.SourceLink.GitHub": "1.1.0-beta-20206-02",
"Microsoft.SourceLink.Common": "1.1.0-beta-20206-02"
"tools": {
"dotnet": "5.0.100-rc.1.20452.10",
"runtimes": {
"dotnet/x64": ["$(MicrosoftNETCorePlatformsVersion)", "$(MicrosoftNETCore3PlatformsVersion)" ]
}
},
"msbuild-sdks": {
"Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20461.7",
"Microsoft.DotNet.Helix.Sdk": "5.0.0-beta.20461.7",
"Microsoft.Build.Traversal": "2.1.1",
"Microsoft.SourceLink.GitHub": "1.1.0-beta-20206-02",
"Microsoft.SourceLink.Common": "1.1.0-beta-20206-02"
}
}
}
59 changes: 0 additions & 59 deletions pkg/Directory.Build.props

This file was deleted.

16 changes: 0 additions & 16 deletions pkg/Microsoft.Extensions.ML/Microsoft.Extensions.ML.nupkgproj

This file was deleted.

This file was deleted.

17 changes: 0 additions & 17 deletions pkg/Microsoft.ML.AutoML/Microsoft.ML.AutoML.nupkgproj

This file was deleted.

3 changes: 0 additions & 3 deletions pkg/Microsoft.ML.AutoML/Microsoft.ML.AutoML.symbols.nupkgproj

This file was deleted.

0 comments on commit acd873f

Please sign in to comment.