Feature/net abstractions 5x

.gitignore

@@ -41,7 +41,6 @@ packages/*
 paket.exe
 paket-files/*.cached
 
-
 build/*
 !build/tools
 !build/keys

.paket/Paket.Restore.targets

@@ -60,6 +60,9 @@
       <Output TaskParameter="ConsoleOutput" PropertyName="PaketRestoreLockFileHash" />
     </Exec>
 
+    <!-- Debug whats going on -->
+    <Message Importance="low" Text="calling paket restore with targetframework=$(TargetFramework) targetframeworks=$(TargetFrameworks)" />
+
     <PropertyGroup Condition="Exists('$(PaketRestoreCacheFile)') ">
       <!-- if no hash has been done yet fall back to just reading in the files and comparing them -->
       <PaketRestoreCachedHash Condition=" '$(PaketRestoreCachedHash)' == '' ">$([System.IO.File]::ReadAllText('$(PaketRestoreCacheFile)'))</PaketRestoreCachedHash>
@@ -76,6 +79,13 @@
     <Exec Command='$(PaketCommand) restore --target-framework "$(TargetFramework)"' Condition=" '$(PaketRestoreRequired)' == 'true' AND '$(TargetFramework)' != '' " ContinueOnError="false" />
 
     <!-- Step 2 Detect project specific changes -->
+    <ItemGroup>
+      <MyTargetFrameworks Condition="'$(TargetFramework)' != '' " Include="$(TargetFramework)"></MyTargetFrameworks>
+      <!-- Don't include all frameworks when msbuild explicitly asks for a single one -->
+      <MyTargetFrameworks Condition="'$(TargetFrameworks)' != '' AND '$(TargetFramework)' == '' " Include="$(TargetFrameworks)"></MyTargetFrameworks>
+      <PaketResolvedFilePaths Include="@(MyTargetFrameworks -> '$(MSBuildProjectDirectory)\obj\$(MSBuildProjectFile).%(Identity).paket.resolved')"></PaketResolvedFilePaths>
+    </ItemGroup>
+    <Message Importance="low" Text="MyTargetFrameworks=@(MyTargetFrameworks) PaketResolvedFilePaths=@(PaketResolvedFilePaths)" />
     <PropertyGroup>
       <PaketReferencesCachedFilePath>$(MSBuildProjectDirectory)\obj\$(MSBuildProjectFile).paket.references.cached</PaketReferencesCachedFilePath>
       <!-- MyProject.fsproj.paket.references has the highest precedence -->
@@ -84,7 +94,9 @@
       <PaketOriginalReferencesFilePath Condition=" !Exists('$(PaketOriginalReferencesFilePath)')">$(MSBuildProjectDirectory)\$(MSBuildProjectName).paket.references</PaketOriginalReferencesFilePath>
       <!-- paket.references -->
       <PaketOriginalReferencesFilePath Condition=" !Exists('$(PaketOriginalReferencesFilePath)')">$(MSBuildProjectDirectory)\paket.references</PaketOriginalReferencesFilePath>
-      <PaketResolvedFilePath>$(MSBuildProjectDirectory)\obj\$(MSBuildProjectFile).$(TargetFramework).paket.resolved</PaketResolvedFilePath>
+
+      <DoAllResolvedFilesExist>false</DoAllResolvedFilesExist>
+      <DoAllResolvedFilesExist Condition="Exists(%(PaketResolvedFilePaths.Identity))">true</DoAllResolvedFilesExist>
       <PaketRestoreRequired>true</PaketRestoreRequired>
       <PaketRestoreRequiredReason>references-file-or-cache-not-found</PaketRestoreRequiredReason>
     </PropertyGroup>
@@ -103,33 +115,39 @@
     </PropertyGroup>
 
     <!-- Step 2 b detect relevant changes in project file (new targetframework) -->
-    <PropertyGroup Condition=" !Exists('$(PaketResolvedFilePath)') AND '$(TargetFramework)' != '' ">
+    <PropertyGroup Condition=" '$(DoAllResolvedFilesExist)' != 'true' ">
       <PaketRestoreRequired>true</PaketRestoreRequired>
-      <PaketRestoreRequiredReason>target-framework '$(TargetFramework)'</PaketRestoreRequiredReason>
+      <PaketRestoreRequiredReason>target-framework '$(TargetFramework)' or '$(TargetFrameworks)' files @(PaketResolvedFilePaths)</PaketRestoreRequiredReason>
     </PropertyGroup>
 
     <!-- Step 3 Restore project specific stuff if required -->
     <Message Condition=" '$(PaketRestoreRequired)' == 'true' " Importance="low" Text="Detected a change ('$(PaketRestoreRequiredReason)') in the project file '$(MSBuildProjectFullPath)', calling paket restore" />
-    <Exec Command='$(PaketCommand) restore --project "$(MSBuildProjectFullPath)"' Condition=" '$(PaketRestoreRequired)' == 'true' " ContinueOnError="false" />
+    <Exec Command='$(PaketCommand) restore --project "$(MSBuildProjectFullPath)" --target-framework "$(TargetFrameworks)"' Condition=" '$(PaketRestoreRequired)' == 'true' AND '$(TargetFramework)' == '' " ContinueOnError="false" />
+    <Exec Command='$(PaketCommand) restore --project "$(MSBuildProjectFullPath)" --target-framework "$(TargetFramework)"' Condition=" '$(PaketRestoreRequired)' == 'true' AND '$(TargetFramework)' != '' " ContinueOnError="false" />
 
     <!-- This shouldn't actually happen, but just to be sure. -->
-    <Error Condition=" !Exists('$(PaketResolvedFilePath)') AND '$(TargetFramework)' != '' AND '$(ResolveNuGetPackages)' != 'False' " Text="Paket file '$(PaketResolvedFilePath)' is missing while restoring $(MSBuildProjectFile). Please delete 'paket-files/paket.restore.cached' and call 'paket restore'." />
+    <PropertyGroup>
+      <DoAllResolvedFilesExist>false</DoAllResolvedFilesExist>
+      <DoAllResolvedFilesExist Condition="Exists(%(PaketResolvedFilePaths.Identity))">true</DoAllResolvedFilesExist>
+    </PropertyGroup>
+    <Error Condition=" '$(DoAllResolvedFilesExist)' != 'true' AND '$(ResolveNuGetPackages)' != 'False' " Text="One Paket file '@(PaketResolvedFilePaths)' is missing while restoring $(MSBuildProjectFile). Please delete 'paket-files/paket.restore.cached' and call 'paket restore'." />
 
     <!-- Step 4 forward all msbuild properties (PackageReference, DotNetCliToolReference) to msbuild -->
-    <ReadLinesFromFile Condition="Exists('$(PaketResolvedFilePath)')" File="$(PaketResolvedFilePath)" >
+    <ReadLinesFromFile Condition="'@(PaketResolvedFilePaths)' != ''" File="%(PaketResolvedFilePaths.Identity)" ><!--Condition="Exists('%(PaketResolvedFilePaths.Identity)')"-->
       <Output TaskParameter="Lines" ItemName="PaketReferencesFileLines"/>
     </ReadLinesFromFile>
 
-    <ItemGroup Condition=" Exists('$(PaketResolvedFilePath)') AND '@(PaketReferencesFileLines)' != '' " >
+    <ItemGroup Condition=" '@(PaketReferencesFileLines)' != '' " >
       <PaketReferencesFileLinesInfo Include="@(PaketReferencesFileLines)" >
         <PackageName>$([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[0])</PackageName>
         <PackageVersion>$([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[1])</PackageVersion>
         <AllPrivateAssets>$([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[4])</AllPrivateAssets>
       </PaketReferencesFileLinesInfo>
       <PackageReference Include="%(PaketReferencesFileLinesInfo.PackageName)">
         <Version>%(PaketReferencesFileLinesInfo.PackageVersion)</Version>
-        <PrivateAssets Condition="%(PaketReferencesFileLinesInfo.AllPrivateAssets) == 'true'">All</PrivateAssets>
+        <PrivateAssets Condition=" ('%(PaketReferencesFileLinesInfo.AllPrivateAssets)' == 'true') Or ('$(PackAsTool)' == 'true') ">All</PrivateAssets>
         <ExcludeAssets Condition="%(PaketReferencesFileLinesInfo.AllPrivateAssets) == 'exclude'">runtime</ExcludeAssets>
+        <Publish Condition=" '$(PackAsTool)' == 'true' ">true</Publish>
       </PackageReference>
     </ItemGroup>
 

NuGet.config

@@ -1,8 +1,8 @@
 <?xml version="1.0" encoding="utf-8"?>
 <configuration>
-  <packageSources>
-    <add key="Temp DiffPlex Feed" value="https://www.myget.org/F/diffplex-temp/api/v3/index.json" />
-    <add key="Xunit myget" value="https://www.myget.org/F/xunit/api/v3/index.json" />
-    <add key="nuget.org" value="https://www.nuget.org/api/v2/" />
-  </packageSources>
+	<packageSources>
+		<add key="Temp DiffPlex Feed" value="https://www.myget.org/F/diffplex-temp/api/v3/index.json" />
+		<add key="nuget.org" value="https://www.nuget.org/api/v2/" />
+		<add key="Elastic abstractions" value="https://www.myget.org/F/elasticsearch-net-abstractions-temp/api/v3/index.json" />
+	</packageSources>
 </configuration>

appveyor.yml

@@ -1,7 +1,7 @@
 version: 1.0.{build} 
 image: Visual Studio 2017 
 build_script: 
-  - cmd: build.bat 
+  - cmd: build.bat canary
 test: off
 environment:
   DOTNET_CLI_TELEMETRY_OPTOUT: true

build.sh

@@ -3,6 +3,6 @@ FAKE="packages/build/FAKE/tools/FAKE.exe"
 BUILDSCRIPT="build/scripts/Targets.fsx"
 
 mono .paket/paket.bootstrapper.exe
-if [[ -f .paket.lock ]]; then mono .paket/paket.exe restore; fi
-if [[ ! -f .paket.lock ]]; then mono .paket/paket.exe install; fi
+if [[ -f paket.lock ]]; then mono .paket/paket.exe restore; fi
+if [[ ! -f paket.lock ]]; then mono .paket/paket.exe install; fi
 mono $FAKE $BUILDSCRIPT "cmdline=$*" --fsiargs -d:MONO 

build/Clients.Common.targets

@@ -2,17 +2,10 @@
 <Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup>
     <!-- Default Version numbers -->
-    <CurrentVersion>0.0.0-bad</CurrentVersion>
-    <CurrentAssemblyVersion>0.0.0</CurrentAssemblyVersion>
-    <CurrentAssemblyFileVersion>0.0.0.0</CurrentAssemblyFileVersion>
-    <!-- 'dotnet xunit' does a build but has no way to pass properties or prevent build so we snoop for TRAVIS here
-    Ideally we handle this in the FAKE script (which we do for 'dotnet build')
-    TODO too lazy to write and test a <CHOOSE>
-    -->
-    <DotNetCoreOnly Condition="'$(TRAVIS)'=='true'">1</DotNetCoreOnly>
-    <DotNetCoreOnly Condition="'$(TRAVIS)'==''"></DotNetCoreOnly>
-    <DoSourceLink></DoSourceLink>
-
+    <CurrentVersion>5.0.0</CurrentVersion>
+    <CurrentAssemblyVersion>5.0.0</CurrentAssemblyVersion>
+    <CurrentAssemblyFileVersion>5.0.0</CurrentAssemblyFileVersion>
+
     <!-- Version and Informational reflect actual version -->
     <Version>$(CurrentVersion)</Version>
     <InformationalVersion>$(CurrentVersion)</InformationalVersion>
@@ -21,15 +14,19 @@
     <!-- File version reflects actual version number without prelease since that not allowed in its struct -->
     <FileVersion>$(CurrentAssemblyFileVersion)</FileVersion>
 
-    <SignAssembly Condition="'$(DotNetCoreOnly)'==''">true</SignAssembly>
-    <AssemblyOriginatorKeyFile Condition="'$(DotNetCoreOnly)'==''">..\..\build\keys\keypair.snk</AssemblyOriginatorKeyFile>
-    <GenerateDocumentationFile Condition="'$(DotNetCoreOnly)'==''">true</GenerateDocumentationFile>
+    <DefineConstants Condition="'$(TargetFramework)'=='netstandard1.3'">$(DefineConstants);DOTNETCORE</DefineConstants>
+
+    <SignAssembly Condition="'$(OS)' == 'Windows_NT'">true</SignAssembly>
+    <AssemblyOriginatorKeyFile>$(MSBuildThisFileDirectory)\keys\keypair.snk</AssemblyOriginatorKeyFile>
+    <GenerateDocumentationFile>true</GenerateDocumentationFile>
     <NoWarn>1591,1572,1571,1573,1587,1570</NoWarn>
     <Prefer32Bit>false</Prefer32Bit>
-    <DefineConstants Condition="'$(TargetFramework)'=='netstandard1.3' OR '$(DotNetCoreOnly)'=='1'">$(DefineConstants);DOTNETCORE</DefineConstants>
-    <DebugType Condition="'$(DotNetCoreOnly)'==''">embedded</DebugType>
-    <DebugType Condition="'$(Configuration)'=='Debug'">full</DebugType>
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>portable</DebugType>
+
+    <DoSourceLink></DoSourceLink>
     <SourceLink Condition="'$(DoSourceLink)'!=''">$(BaseIntermediateOutputPath)\sl-$(MsBuildProjectName)-$(TargetFramework).json</SourceLink>
+
     <RepoUri>https://raw.githubusercontent.com/elastic/elasticsearch-net</RepoUri>
   </PropertyGroup>
 

build/Elasticsearch.Net.nuspec

@@ -30,9 +30,6 @@
 		</dependencies>
 	</metadata>
 	<files>
-		<file src="output\Elasticsearch.Net\net45\Elasticsearch.Net.dll" target="lib\net45"/>
-		<file src="output\Elasticsearch.Net\net45\Elasticsearch.Net.XML" target="lib\net45"/>
-
 		<file src="output\Elasticsearch.Net\net46\Elasticsearch.Net.dll" target="lib\net46"/>
 		<file src="output\Elasticsearch.Net\net46\Elasticsearch.Net.XML" target="lib\net46"/>
 

build/NEST.nuspec

@@ -35,9 +35,6 @@
 		</dependencies>
 	</metadata>
 	<files>
-		<file src="output\Nest\net45\Nest.dll" target="lib\net45"/>
-		<file src="output\Nest\net45\Nest.XML" target="lib\net45"/>
-
 		<file src="output\Nest\net46\Nest.dll" target="lib\net46"/>
 		<file src="output\Nest\net46\Nest.XML" target="lib\net46"/>
 

build/scripts/Benchmarking.fsx

@@ -1,7 +1,7 @@
-#r "../../packages/build/NEST/lib/net45/Nest.dll"
-#r "../../packages/build/Elasticsearch.Net/lib/net45/Elasticsearch.Net.dll"
+#r "../../packages/build/NEST/lib/net46/Nest.dll"
+#r "../../packages/build/Elasticsearch.Net/lib/net46/Elasticsearch.Net.dll"
 #r "../../packages/build/Newtonsoft.Json/lib/net45/Newtonsoft.Json.dll"
-#r "../../packages/build/FSharp.Data/lib/net40/FSharp.Data.dll"
+#r "../../packages/build/FSharp.Data/lib/net45/FSharp.Data.dll"
 #I @"../../packages/build/FAKE/tools"
 #r @"FakeLib.dll"
 #nowarn "0044" //TODO sort out FAKE 5

build/scripts/Building.fsx

@@ -1,16 +1,21 @@
 #I @"../../packages/build/FAKE/tools"
-#I @"../../packages/build/FSharp.Data/lib/net40"
+#I @"../../packages/build/FSharp.Data/lib/net45"
+#I @"../../packages/build/Mono.Cecil/lib/net40"
 #r @"FakeLib.dll"
+#r @"Mono.Cecil.dll"
 #r @"FSharp.Data.dll"
+#nowarn "0044" //TODO sort out FAKE 5
 
 #load @"Paths.fsx"
 #load @"Tooling.fsx"
 #load @"Versioning.fsx"
 
 open System 
 open System.IO
+open System.Reflection
 open Fake 
 open FSharp.Data 
+open Mono.Cecil
 
 open Paths
 open Projects
@@ -20,20 +25,20 @@ open Versioning
 module Build =
 
     let private runningRelease = hasBuildParam "version" || hasBuildParam "apikey" || getBuildParam "target" = "canary" || getBuildParam "target" = "release"
-    let private quickBuild = not (getBuildParam "target" = "release" || getBuildParam "target" = "canary")
 
     type private GlobalJson = JsonProvider<"../../global.json">
     let private pinnedSdkVersion = GlobalJson.GetSample().Sdk.Version
     if isMono then setProcessEnvironVar "TRAVIS" "true"
+
     let private buildingOnTravis = getEnvironmentVarAsBool "TRAVIS" 
 
     let private sln = sprintf "src/Elasticsearch%s.sln" (if buildingOnTravis then ".DotNetCoreOnly" else "")
-
+    
     let private compileCore incremental =
         if not (DotNetCli.isInstalled()) then failwith  "You need to install the dotnet command line SDK to build for .NET Core"
         let runningSdkVersion = DotNetCli.getVersion()
         if (runningSdkVersion <> pinnedSdkVersion) then failwithf "Attempting to run with dotnet.exe with %s but global.json mandates %s" runningSdkVersion pinnedSdkVersion
-        let incrementalFramework = DotNetFramework.Net45
+        let incrementalFramework = DotNetFramework.Net46
         let sourceLink = if not incremental && not isMono && runningRelease then "1" else ""
         let props = 
             [ 
@@ -75,4 +80,102 @@ module Build =
         CleanDir Paths.BuildOutput
         DotNetCli.RunCommand (fun p -> { p with TimeOut = TimeSpan.FromMinutes(3.) }) "clean src/Elasticsearch.sln -c Release" |> ignore
         DotNetProject.All |> Seq.iter(fun p -> CleanDir(Paths.BinFolder p.Name))
-
+
+    type CustomResolver(folder) = 
+        inherit DefaultAssemblyResolver()
+        member this.Folder = folder;
+
+        override this.Resolve name = 
+            try
+                base.Resolve name
+            with
+            | ex -> 
+                AssemblyDefinition.ReadAssembly(Path.Combine(folder, "Elasticsearch.Net.dll"));
+
+    let private rewriteNamespace nest f = 
+        trace "Rewriting namespaces"
+        let folder = Paths.ProjectOutputFolder nest f
+        let nestDll = sprintf "%s/%s.dll" folder nest.Name
+        let nestRewrittenDll = sprintf "%s/%s-rewriten.dll" folder nest.Name
+        use resolver = new CustomResolver(folder)
+        let readerParams = ReaderParameters( AssemblyResolver = resolver, ReadWrite = true );
+        use nestAssembly = AssemblyDefinition.ReadAssembly(nestDll, readerParams);
+
+        for item in nestAssembly.MainModule.Types do
+            if item.Namespace.StartsWith("Newtonsoft.Json") then
+                item.Namespace <- item.Namespace.Replace("Newtonsoft.Json", "Nest.Json")
+
+        // Touch custom attribute arguments 
+        // Cecil does not update the types referenced within these attributes automatically,
+        // so enumerate them to ensure namespace renaming is reflected in these references.
+        let touchAttributes (attributes:Mono.Collections.Generic.Collection<CustomAttribute>) = 
+            for attr in attributes do
+                if attr.HasConstructorArguments then
+                    for constArg in attr.ConstructorArguments do
+                        if constArg.Type.Name = "Type" then ignore()    
+
+        // rewrite explicitly implemented interface definitions defined
+        // in Newtonsoft.Json
+        let rewriteName (method:IMemberDefinition) =
+            if method.Name.Contains("Newtonsoft.Json") then
+                method.Name <- method.Name.Replace("Newtonsoft.Json", "Nest.Json")
+
+        // recurse through all types and nested types   
+        let rec rewriteTypes (types:Mono.Collections.Generic.Collection<TypeDefinition>) =
+            for t in types do
+                touchAttributes t.CustomAttributes
+                for prop in t.Properties do 
+                    touchAttributes prop.CustomAttributes
+                    rewriteName prop
+                    if prop.GetMethod <> null then rewriteName prop.GetMethod
+                    if prop.SetMethod <> null then rewriteName prop.SetMethod
+                for method in t.Methods do 
+                    touchAttributes method.CustomAttributes
+                    rewriteName method
+                    for over in method.Overrides do rewriteName method
+                for field in t.Fields do touchAttributes field.CustomAttributes
+                for interf in t.Interfaces do touchAttributes interf.CustomAttributes
+                for event in t.Events do touchAttributes event.CustomAttributes
+                if t.HasNestedTypes then rewriteTypes t.NestedTypes
+
+        nestAssembly.MainModule.Types |> rewriteTypes
+
+        let resources = nestAssembly.MainModule.Resources
+        for i = resources.Count-1 downto 0 do
+            let resource = resources.[i]
+            // remove the Newtonsoft signing key
+            if resource.Name = "Newtonsoft.Json.Dynamic.snk" then resources.Remove(resource) |> ignore
+
+        let key = File.ReadAllBytes(Paths.Keys("keypair.snk"))
+        let kp = StrongNameKeyPair(key)
+        let wp = WriterParameters ( StrongNameKeyPair = kp);
+        nestAssembly.Write(wp) |> ignore;
+        trace "Finished rewriting namespaces"
+
+    let private ilRepackInternal() =
+        let fw = if isMono then [DotNetFramework.NetStandard1_3] else DotNetFramework.All
+        for f in fw do 
+            let nest = Project Project.Nest
+            let folder = Paths.ProjectOutputFolder nest f
+            let nestDll = sprintf "%s/%s.dll" folder nest.Name
+            let nestMergedDll = sprintf "%s/%s-merged.dll" folder nest.Name
+            let jsonDll = sprintf "%s/Newtonsoft.Json.dll" folder
+            let keyFile = Paths.Keys("keypair.snk");
+            let options = 
+                [ 
+                    "/keyfile:", keyFile;
+                    "/internalize", "";
+                    "/lib:", folder;
+                    "/out:", nestDll;
+                ] 
+                |> List.map (fun (p,v) -> sprintf "%s%s" p v)
+
+            let args = [nestDll; jsonDll;] |> List.append options;
+
+            Tooling.ILRepack.Exec args |> ignore
+            rewriteNamespace nest f |> ignore
+
+    let ILRepack() = 
+        //nothing to IL merge in the 5.x branch
+        ignore()
+