Feature/versioned packages

.editorconfig

@@ -12,7 +12,7 @@ indent_size = 4
 indent_style = tab
 indent_size = 4
 
-[*.{fs,fsx}]
+[*.{fs,fsx,yml}]
 indent_style = space
 indent_size = 4
 

appveyor.yml

@@ -1,8 +1,20 @@
 version: 1.0.{build} 
 image: Visual Studio 2017 
-build_script: 
-  - cmd: build.bat canary skipdocs
 test: off
 environment:
-  DOTNET_CLI_TELEMETRY_OPTOUT: true
-  DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
+    DOTNET_CLI_TELEMETRY_OPTOUT: true
+    DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
+build_script:
+    - cmd: build.bat canary skipdocs
+
+nuget:
+    disable_publish_on_pr: true
+
+for:
+
+- branches:
+      only:
+          - /^(master|\d+\.x)$/
+  artifacts:
+      - path: .\build\output\_packages\*.nupkg
+        name: NuGet

build/NEST.nuspec

@@ -4,23 +4,24 @@
 		<id>NEST</id>
 		<version>$version$</version>
 		<title>NEST - Elasticsearch .NET high level client</title>
+
 		<authors>Elastic and contributors</authors>
 		<owners>Elastic</owners>
+		<copyright>2014-$year$ Elasticsearch BV</copyright>
+
+		<requireLicenseAcceptance>false</requireLicenseAcceptance>
 		<licenseUrl>https://github.com/elastic/elasticsearch-net/blob/master/license.txt</licenseUrl>
 		<projectUrl>https://github.com/elastic/elasticsearch-net</projectUrl>
 		<iconUrl>https://raw.githubusercontent.com/elastic/elasticsearch-net/master/build/nuget-icon.png</iconUrl>
-		<requireLicenseAcceptance>false</requireLicenseAcceptance>
+
 		<description>Strongly typed interface to Elasticsearch. Fluent and classic object initializer mappings of requests and responses. Uses and exposes Elasticsearch.Net</description>
 		<summary>Strongly typed interface to Elasticsearch. Fluent and classic object initializer mappings of requests and responses. Uses and exposes Elasticsearch.Net</summary>
-		<releaseNotes>See https://github.com/elastic/elasticsearch-net/releases/tag/$version$</releaseNotes>
-		<copyright>2014-$year$ Elasticsearch BV</copyright>
+		<releaseNotes>https://github.com/elastic/elasticsearch-net/releases/tag/$version$</releaseNotes>
 		<tags>elasticsearch,elastic,search,lucene,nest</tags>
+
 		<dependencies>
 			<group targetFramework=".NETStandard2.0">
 				<dependency id="Elasticsearch.Net" version="[$version$, $nextMajorVersion$)" />
-				<!--<dependency id="System.Collections.Specialized" version="[4.3.0, )" />-->
-				<!--<dependency id="System.Reflection.TypeExtensions" version="[4.3.0, )" />-->
-				<!--<dependency id="System.Linq.Queryable" version="[4.0.1, )" />-->
 
 				<!-- since we no longer reference JSON.NET (internalized) we do need to ship with its System references -->
 				<dependency id="System.ComponentModel.TypeConverter" version="[4.3.0, )"/>

build/scripts/Building.fsx

@@ -12,10 +12,8 @@
 
 open System 
 open System.IO
-open System.Reflection
 open Fake 
 open FSharp.Data 
-open Mono.Cecil
 
 open Paths
 open Projects
@@ -78,100 +76,47 @@ module Build =
         CleanDir Paths.BuildOutput
         DotNetCli.RunCommand (fun p -> { p with TimeOut = TimeSpan.FromMinutes(5.) }) "clean src/Elasticsearch.sln -c Release" |> ignore
         DotNetProject.All |> Seq.iter(fun p -> CleanDir(Paths.BinFolder p.Name))
-
-    type CustomResolver(folder) = 
-        inherit DefaultAssemblyResolver()
-        member this.Folder = folder;
-
-        override this.Resolve name = 
-            try
-                base.Resolve name
-            with
-            | ex -> 
-                AssemblyDefinition.ReadAssembly(Path.Combine(folder, "Elasticsearch.Net.dll"));
-
-    let private rewriteNamespace nest f = 
-        trace "Rewriting namespaces"
-        let folder = Paths.ProjectOutputFolder nest f
-        let nestDll = sprintf "%s/%s.dll" folder nest.Name
-        let nestRewrittenDll = sprintf "%s/%s-rewriten.dll" folder nest.Name
-        use resolver = new CustomResolver(folder)
-        let readerParams = ReaderParameters( AssemblyResolver = resolver, ReadWrite = true );
-        use nestAssembly = AssemblyDefinition.ReadAssembly(nestDll, readerParams);
 
-        for item in nestAssembly.MainModule.Types do
-            if item.Namespace.StartsWith("Newtonsoft.Json") then
-                item.Namespace <- item.Namespace.Replace("Newtonsoft.Json", "Nest.Json")
-
-        // Touch custom attribute arguments 
-        // Cecil does not update the types referenced within these attributes automatically,
-        // so enumerate them to ensure namespace renaming is reflected in these references.
-        let touchAttributes (attributes:Mono.Collections.Generic.Collection<CustomAttribute>) = 
-            for attr in attributes do
-                if attr.HasConstructorArguments then
-                    for constArg in attr.ConstructorArguments do
-                        if constArg.Type.Name = "Type" then ignore()    
-
-        // rewrite explicitly implemented interface definitions defined
-        // in Newtonsoft.Json
-        let rewriteName (method:IMemberDefinition) =
-            if method.Name.Contains("Newtonsoft.Json") then
-                method.Name <- method.Name.Replace("Newtonsoft.Json", "Nest.Json")
-
-        // recurse through all types and nested types   
-        let rec rewriteTypes (types:Mono.Collections.Generic.Collection<TypeDefinition>) =
-            for t in types do
-                touchAttributes t.CustomAttributes
-                for prop in t.Properties do 
-                    touchAttributes prop.CustomAttributes
-                    rewriteName prop
-                    if prop.GetMethod <> null then rewriteName prop.GetMethod
-                    if prop.SetMethod <> null then rewriteName prop.SetMethod
-                for method in t.Methods do 
-                    touchAttributes method.CustomAttributes
-                    rewriteName method
-                    for over in method.Overrides do rewriteName method
-                for field in t.Fields do touchAttributes field.CustomAttributes
-                for interf in t.Interfaces do touchAttributes interf.CustomAttributes
-                for event in t.Events do touchAttributes event.CustomAttributes
-                if t.HasNestedTypes then rewriteTypes t.NestedTypes
-
-        nestAssembly.MainModule.Types |> rewriteTypes
+    let private assemblyRewriter = Paths.PaketDotNetGlobalTool "assembly-rewriter" @"tools\netcoreapp2.1\any\assembly-rewriter.dll"
+    let private keyFile = Paths.Keys "keypair.snk"
+    let Rewrite version framework projects =
+        let project = projects |> Seq.head
+        let folder = Paths.ProjectOutputFolder project framework
 
-        let resources = nestAssembly.MainModule.Resources
-        for i = resources.Count-1 downto 0 do
-            let resource = resources.[i]
-            // remove the Newtonsoft signing key
-            if resource.Name = "Newtonsoft.Json.Dynamic.snk" then resources.Remove(resource) |> ignore
-
-        let key = File.ReadAllBytes(Paths.Keys("keypair.snk"))
-        let kp = StrongNameKeyPair(key)
-        let wp = WriterParameters ( StrongNameKeyPair = kp);
-        nestAssembly.Write(wp) |> ignore;
-        trace "Finished rewriting namespaces"
+        let dllFullPath name = sprintf "%s/%s.dll" folder name
+        let outputName (p: DotNetProject) = match p.Name = project.Name with | true -> p.Name | _ -> p.InternalName 
+        let fullOutput (p: DotNetProject) = dllFullPath (p.Versioned (outputName p) version)
+        let dlls =
+            projects
+            |> Seq.map (fun p -> sprintf @"-i ""%s"" -o ""%s"" "  (dllFullPath p.Name) (fullOutput p))
+            |> Seq.fold (+) " "
+
+        let mergeCommand = sprintf @"%s %s" assemblyRewriter dlls
+        DotNetCli.RunCommand (fun p -> { p with TimeOut = TimeSpan.FromMinutes(3.) }) mergeCommand |> ignore
+
+        let mergedOutFile = fullOutput project
+        let ilMergeArgs = [
+            "/internalize"; 
+            (sprintf "/lib:%s" folder); 
+            (sprintf "/keyfile:%s" keyFile); 
+            (sprintf "/out:%s" mergedOutFile)
+        ]
+        let mergeDlls = 
+            projects 
+            |> Seq.filter (fun p -> p.Name = project.Name || not <| (DotNetProject.AllPublishable |> Seq.contains p)) 
+            |> Seq.map fullOutput
+        match project.NeedsMerge with 
+        | true -> Tooling.ILRepack.Exec (ilMergeArgs |> Seq.append mergeDlls) |> ignore
+        | _ -> Tooling.ILRepack.Exec (ilMergeArgs |> Seq.append [mergeDlls |> Seq.head]) |> ignore
 
     let private ilRepackInternal() =
         let fw = DotNetFramework.All
-        for f in fw do 
-            let nest = Project Project.Nest
-            let folder = Paths.ProjectOutputFolder nest f
-            let nestDll = sprintf "%s/%s.dll" folder nest.Name
-            let nestMergedDll = sprintf "%s/%s-merged.dll" folder nest.Name
-            let jsonDll = sprintf "%s/Newtonsoft.Json.dll" folder
-            let keyFile = Paths.Keys("keypair.snk");
-            let options = 
-                [ 
-                    "/keyfile:", keyFile;
-                    "/internalize", "";
-                    "/lib:", folder;
-                    "/out:", nestDll;
-                ] 
-                |> List.map (fun (p,v) -> sprintf "%s%s" p v)
-
-            let args = [nestDll; jsonDll;] |> List.append options;
-
-            Tooling.ILRepack.Exec args |> ignore
-            rewriteNamespace nest f |> ignore
+        let projects = DotNetProject.AllPublishable
+        let currentMajor = sprintf "%i" <| Versioning.CurrentVersion.Major
+        for f in fw do
+            for p in projects do
+                if p.VersionedMergeDependencies <> [] then Rewrite (Some currentMajor) f p.VersionedMergeDependencies
+                if p.MergeDependencies <> [] then Rewrite None f p.MergeDependencies
 
     let ILRepack() = 
         //ilrepack on mono crashes pretty hard on my machine

build/scripts/Commandline.fsx

@@ -25,9 +25,8 @@ Targets:
 * integrate <elasticsearch_versions> [clustername] [testfilter] 
   - run integration tests for <elasticsearch_versions> which is a semicolon separated list of
     elasticsearch versions to test or `latest`. Can filter tests by <clustername> and <testfilter>
-* canary [apikey] [feed]
-  - create a canary nuget package based on the current version if [feed] and [apikey] are provided
-    also pushes to upstream (myget)
+* canary 
+  - create a canary nuget package based on the current version.
 * diff <github|nuget|dir|assembly> <version|path 1> <version|path 2> [format]
 * cluster <cluster-name> [version]
   - Start a cluster defined in Tests.Core or Tests from the command line and leaves it running
@@ -220,12 +219,6 @@ module Commandline =
         | ["touch"; ] -> ignore()
         | ["temp"; ] -> ignore()
         | ["canary"; ] -> ignore()
-        | ["canary"; apiKey ] ->
-            setBuildParam "apiKey" apiKey
-            setBuildParam "feed" "elasticsearch-net"
-        | ["canary"; apiKey; feed ] ->
-            setBuildParam "apiKey" apiKey
-            setBuildParam "feed" feed
         | _ ->
             traceError usage
             exit 2

build/scripts/Differ.fsx

@@ -11,11 +11,7 @@ open System
 open System.IO
 
 open Fake 
-open System
-open System.IO
 open System.Linq
-open System.Net
-open System.Text
 open System.Text.RegularExpressions
 open System.Xml
 open System.Xml.Linq

build/scripts/Paths.fsx

@@ -27,6 +27,7 @@ module Paths =
     let SourceFolder = "src"
 
     let CheckedInTool(tool) = sprintf "%s/%s" CheckedInToolsFolder tool
+    let PaketDotNetGlobalTool tool subPath = sprintf "%s/%s" (Tool tool) subPath
     let Keys(keyFile) = sprintf "%s/%s" KeysFolder keyFile
     let Output(folder) = sprintf "%s/%s" BuildOutput folder
     let Source(folder) = sprintf "%s/%s" SourceFolder folder
@@ -43,7 +44,7 @@ module Paths =
             match p with
             | Tests -> sprintf "%s/%s/%s.csproj" SourceFolder project.Name project.Name
             | DocGenerator -> sprintf "%s/CodeGeneration/%s/%s.csproj" SourceFolder project.Name project.Name
-            
+        | _ -> null
 
     let BinFolder(folder) = 
         let f = replace @"\" "/" folder

build/scripts/Projects.fsx

@@ -26,14 +26,18 @@ module Projects =
         | Nest
         | ElasticsearchNet
         | NestJsonNetSerializer
-
+        
     type PrivateProject =
         | Tests
         | DocGenerator
+
+    type DependencyProject = 
+        | JsonNet 
 
     type DotNetProject = 
         | Project of Project
         | PrivateProject of PrivateProject
+        | DepencyProject of DependencyProject
 
         static member All = 
             seq [
@@ -49,20 +53,44 @@ module Projects =
                 Project Project.Nest; 
                 Project Project.NestJsonNetSerializer;
             ] 
-        static member Tests = seq [PrivateProject PrivateProject.Tests;] 
+        static member Tests = seq [PrivateProject PrivateProject.Tests]
+
+        member this.MergeDependencies=
+            match this with 
+            | Project Nest -> [Project Project.Nest; DepencyProject DependencyProject.JsonNet]
+            | _ -> []
+
+        member this.VersionedMergeDependencies =
+            match this with 
+            | Project Nest -> [Project Project.Nest; Project Project.ElasticsearchNet; DepencyProject DependencyProject.JsonNet]
+            | Project NestJsonNetSerializer -> [Project NestJsonNetSerializer; Project Project.Nest; Project Project.ElasticsearchNet ]
+            | Project ElasticsearchNet -> [Project ElasticsearchNet]
+            | _ -> []
 
         member this.Name =
             match this with
-            | Project p ->
-                match p with
-                | Nest -> "Nest"
-                | ElasticsearchNet -> "Elasticsearch.Net"
-                | NestJsonNetSerializer -> "Nest.JsonNetSerializer"
-            | PrivateProject p ->
-                match p with
-                | Tests -> "Tests"
-                | DocGenerator -> "DocGenerator"
-
+            | Project Nest -> "Nest"
+            | Project ElasticsearchNet -> "Elasticsearch.Net"
+            | Project NestJsonNetSerializer -> "Nest.JsonNetSerializer"
+            | PrivateProject Tests -> "Tests"
+            | PrivateProject DocGenerator -> "DocGenerator"
+            | DepencyProject JsonNet -> "Newtonsoft.Json"
+
+        member this.NugetId = match this with | Project Nest -> "NEST" | _ -> this.Name
+
+        member this.NeedsMerge = match this with | Project NestJsonNetSerializer -> false | _ -> true
+
+        member this.Versioned name version =
+            match version with
+            | Some s -> sprintf "%s%s" name s
+            | None -> name
+
+        member this.InternalName =
+            match this with
+            | Project p -> this.Name 
+            | PrivateProject p -> sprintf "Elastic.Internal.%s" this.Name
+            | DepencyProject JsonNet -> "Elastic.Internal.JsonNet"
+
         static member TryFindName (name: string) =
             DotNetProject.All
             |> Seq.map(fun p -> p.Name)