Feature/netcoreapp21 5x

.paket/Paket.Restore.targets

@@ -75,8 +75,7 @@
 
     <!-- Do a global restore if required -->
     <Exec Command='$(PaketBootStrapperCommand)' Condition="Exists('$(PaketBootStrapperExePath)') AND !(Exists('$(PaketExePath)'))" ContinueOnError="false" />
-    <Exec Command='$(PaketCommand) restore --target-framework "$(TargetFrameworks)"' Condition=" '$(PaketRestoreRequired)' == 'true' AND '$(TargetFramework)' == '' " ContinueOnError="false" />
-    <Exec Command='$(PaketCommand) restore --target-framework "$(TargetFramework)"' Condition=" '$(PaketRestoreRequired)' == 'true' AND '$(TargetFramework)' != '' " ContinueOnError="false" />
+    <Exec Command='$(PaketCommand) restore' Condition=" '$(PaketRestoreRequired)' == 'true' " ContinueOnError="false" />
 
     <!-- Step 2 Detect project specific changes -->
     <ItemGroup>

.travis.yml

@@ -1,9 +1,9 @@
 language: csharp
 solution: src/Elasticsearch.sln
-script: ./build.sh test
+script: ./build.sh test-one skipdocs
 dist: trusty
 mono: 4.6.2
-dotnet: 2.0.3
+dotnet: 2.1.3
 env:
   global:     
     - DOTNET_CLI_TELEMETRY_OPTOUT: 1

build/Clients.Common.targets

@@ -5,7 +5,7 @@
     <CurrentVersion>5.0.0</CurrentVersion>
     <CurrentAssemblyVersion>5.0.0</CurrentAssemblyVersion>
     <CurrentAssemblyFileVersion>5.0.0</CurrentAssemblyFileVersion>
-    
+
     <!-- Version and Informational reflect actual version -->
     <Version>$(CurrentVersion)</Version>
     <InformationalVersion>$(CurrentVersion)</InformationalVersion>

build/scripts/Benchmarking.fsx

@@ -143,12 +143,12 @@ module Benchmarker =
                 DotNetCli.RunCommand(fun p ->
                     { p with
                         WorkingDir = testsProjectDirectory
-                    }) "run -f net46 -c Release Benchmark"
+                    }) "run -f netcoreapp2.1 -c Release Benchmark"
              else
                 DotNetCli.RunCommand(fun p ->
                     { p with
                         WorkingDir = testsProjectDirectory
-                    }) "run -f net46 -c Release Benchmark non-interactive"
+                    }) "run -f netcoreapp2.1 -c Release Benchmark non-interactive"
         finally
             // running benchmarks can timeout so clean up any generated benchmark files
             let benchmarkOutputFiles =

build/scripts/Building.fsx

@@ -26,27 +26,26 @@ module Build =
 
     let private runningRelease = hasBuildParam "version" || hasBuildParam "apikey" || getBuildParam "target" = "canary" || getBuildParam "target" = "release"
 
-    type private GlobalJson = JsonProvider<"../../global.json">
+    type private GlobalJson = JsonProvider<"../../global.json", InferTypesFromValues=false>
     let private pinnedSdkVersion = GlobalJson.GetSample().Sdk.Version
     if isMono then setProcessEnvironVar "TRAVIS" "true"
-
     let private buildingOnTravis = getEnvironmentVarAsBool "TRAVIS" 
 
-    let private sln = sprintf "src/Elasticsearch%s.sln" (if buildingOnTravis then ".DotNetCoreOnly" else "")
+    let private sln = "src/Elasticsearch.sln"
 
     let private compileCore incremental =
         if not (DotNetCli.isInstalled()) then failwith  "You need to install the dotnet command line SDK to build for .NET Core"
         let runningSdkVersion = DotNetCli.getVersion()
         if (runningSdkVersion <> pinnedSdkVersion) then failwithf "Attempting to run with dotnet.exe with %s but global.json mandates %s" runningSdkVersion pinnedSdkVersion
-        let incrementalFramework = DotNetFramework.Net46
+        let incrementalFramework = DotNetFramework.NetStandard1_3
         let sourceLink = if not incremental && not isMono && runningRelease then "1" else ""
         let props = 
             [ 
                 "CurrentVersion", (Versioning.CurrentVersion.ToString());
                 "CurrentAssemblyVersion", (Versioning.CurrentAssemblyVersion.ToString());
                 "CurrentAssemblyFileVersion", (Versioning.CurrentAssemblyFileVersion.ToString());
                 "DoSourceLink", sourceLink;
-                "DotNetCoreOnly", if buildingOnTravis then "1" else "";
+                "FakeBuild", "1";
                 "OutputPathBaseDir", Path.GetFullPath Paths.BuildOutput;
             ] 
             |> List.map (fun (p,v) -> sprintf "%s=%s" p v)
@@ -153,7 +152,7 @@ module Build =
         trace "Finished rewriting namespaces"
 
     let private ilRepackInternal() =
-        let fw = if isMono then [DotNetFramework.NetStandard1_3] else DotNetFramework.All
+        let fw = DotNetFramework.All
         for f in fw do 
             let nest = Project Project.Nest
             let folder = Paths.ProjectOutputFolder nest f

build/scripts/Commandline.fsx

@@ -28,14 +28,13 @@ Targets:
 * canary [apikey] [feed]
   - create a canary nuget package based on the current version if [feed] and [apikey] are provided
     also pushes to upstream (myget)
-* diff <github|nuget|directories|assemblies> <version|path 1> <version|path 2> [format]
+* diff <github|nuget|dir|assembly> <version|path 1> <version|path 2> [format]
 
 NOTE: both the `test` and `integrate` targets can be suffixed with `-all` to force the tests against all suported TFM's
 
 Execution hints can be provided anywhere on the command line
 - skiptests : skip running tests as part of the target chain
 - skipdocs : skip generating documentation
-- source_serialization : force tests to use a client with custom source serialization
 - seed:<N> : provide a seed to run the tests with.
 - random:<K><:B> : sets random K to bool B if if B is omitted will default to true
   K can be: sourceserializer, typedkeys or oldconnection (only valid on windows)
@@ -58,21 +57,30 @@ module Commandline =
         |> List.filter (fun x -> (x.StartsWith("random:")))
         |> List.map (fun x -> (x.Replace("random:", "")))
 
+    let docsBranch = 
+        match args |> List.tryFind (fun x -> x.StartsWith("docs:")) with
+        | Some t -> t.Replace("docs:", "")
+        | _ -> ""
+
     let private filteredArgs = 
         args 
         |> List.filter (
             fun x -> 
-                x <> "skiptests" && x <> "skipdocs" && x <> "source_serialization" && not (x.StartsWith("seed:")) && not (x.StartsWith("random:"))
+                x <> "skiptests" && 
+                x <> "skipdocs" && 
+                not (x.StartsWith("seed:")) && 
+                not (x.StartsWith("random:")) && 
+                not (x.StartsWith("docs:"))
         )
 
     let multiTarget =
         match (filteredArgs |> List.tryHead) with
-        | Some t when t.EndsWith("-all") -> MultiTarget.All
-        | _ -> MultiTarget.One
+        | Some t when t.EndsWith("-one") -> MultiTarget.One
+        | _ -> MultiTarget.All
 
     let target =
         match (filteredArgs |> List.tryHead) with
-        | Some t -> t.Replace("-all", "")
+        | Some t -> t.Replace("-one", "")
         | _ -> "build"
 
     let validMonoTarget =
@@ -206,6 +214,7 @@ module Commandline =
             setBuildParam "first" firstVersionOrPath
             setBuildParam "second" secondVersionOrPath         
 
+        | ["touch"; ] -> ignore()
         | ["temp"; ] -> ignore()
         | ["canary"; ] -> ignore()
         | ["canary"; apiKey ] ->

build/scripts/Differ.fsx

@@ -7,18 +7,20 @@
 #load @"Tooling.fsx"
 #load @"Projects.fsx"
 
-open Microsoft.FSharp.Reflection
 open System
-open System.Collections.Generic
+open System.IO
+
+open Fake 
+open System
 open System.IO
 open System.Linq
 open System.Net
 open System.Text
 open System.Text.RegularExpressions
 open System.Xml
 open System.Xml.Linq
-open Fake 
 open Fake.Git.CommandHelper
+
 open Paths
 open Projects
 open Tooling
@@ -262,63 +264,42 @@ module Differ =
         | :? XmlException -> ignore()
 
     let private convertToAsciidoc path first second = 
-        let createDiffDescription description (writer:TextWriter) =
-            let m = Regex.Match(description, "(.*?) changed from (.*?) to (.*).")
-            if m.Success then
-                let memberType = m.Groups.[1].Value
-                let o = m.Groups.[2].Value
-                let n = m.Groups.[3].Value
-                writer.WriteLine("+")
-                writer.WriteLine(sprintf "%s change" memberType)
-                writer.WriteLine("+")
-                writer.WriteLine("[source,csharp]")
-                writer.WriteLine("----")
-                writer.WriteLine(sprintf "// before in %s" first)
-                writer.WriteLine(sprintf "%s" o)
-                writer.WriteLine(sprintf "// now in %s" second)
-                writer.WriteLine(sprintf "%s" n)
-                writer.WriteLine("----")
-            else
-                writer.WriteLine(sprintf "%s" description)
-
         let name = path |> Path.GetFileNameWithoutExtension
         try
             let doc = XDocument.Load path
             let output = Path.ChangeExtension(path, "asciidoc")
             DeleteFile output
             use file = File.OpenWrite <| output
             use writer = new StreamWriter(file)                
-            writer.WriteLine(name |> replace "." "-" |> toLower |> sprintf "[[%s-breaking-changes]]")
+            writer.WriteLine(name |> replace "." "-" |> sprintf "[[%s-breaking-changes]]")
             writer.WriteLine(sprintf "== Breaking changes for %s between %s and %s" name first second)
             writer.WriteLine()
 
             for element in (doc |> descendents "Type") do
                 let typeName = element |> attributeValue "Name" |> replace (sprintf "%s." name) ""
                 let diffType  = element |> attributeValue "DiffType" |> convertDiffType
-
                 match diffType with
-                | Deleted -> writer.WriteLine(sprintf "`%s`:: deleted" typeName)
-                | New -> writer.WriteLine(sprintf "`%s`:: added" typeName)
+                | Deleted -> writer.WriteLine(sprintf "[float]%s=== `%s` is deleted" Environment.NewLine typeName)
+                | New -> writer.WriteLine(sprintf "[float]%s=== `%s` is added" Environment.NewLine typeName)
                 | Modified -> 
                     let members = Seq.append (element |> elements "Method") (element |> elements "Property")        
                     if Seq.isEmpty members |> not then          
-                        writer.WriteLine(sprintf "`%s`::" typeName)
+                        writer.WriteLine(sprintf "[float]%s=== `%s`" Environment.NewLine typeName)
                         for m in members do
                             let memberName = m |> attributeValue "Name"
                             if isNotNullOrEmpty memberName then
                                 let diffType  = m |> attributeValue "DiffType"                         
                                 if isNotNullOrEmpty diffType then                    
                                     match convertDiffType diffType with
-                                            | Deleted -> writer.WriteLine(sprintf "  * `%s` deleted" memberName)
-                                            | New -> writer.WriteLine(sprintf "  * `%s` added" memberName)
+                                            | Deleted -> writer.WriteLine(sprintf "[float]%s==== `%s` is deleted" Environment.NewLine memberName)
+                                            | New -> writer.WriteLine(sprintf "[float]%s==== `%s` is added" Environment.NewLine memberName)
                                             | Modified -> 
                                                 match (m.Descendants(XName.op_Implicit "DiffItem") |> Seq.tryHead) with
                                                 | Some diffItem ->
-                                                    writer.WriteLine(sprintf "  * `%s`" memberName)                                                                                      
+                                                    writer.WriteLine(sprintf "[float]%s==== `%s`" Environment.NewLine memberName)                                                                                      
                                                     let diffDescription = diffItem.Value
-                                                    writer |> createDiffDescription diffDescription                                    
+                                                    writer.WriteLine(Regex.Replace(diffDescription, "changed from (.*?) to (.*).", "changed from `$1` to `$2`."))                                         
                                                 | None -> ()
-                writer.WriteLine()
         with
         | :? XmlException -> ignore()
 

build/scripts/Documentation.fsx

@@ -3,6 +3,7 @@
 #nowarn "0044" //TODO sort out FAKE 5
 
 #load @"Paths.fsx"
+#load @"Commandline.fsx"
 
 open System
 open System.IO
@@ -11,17 +12,19 @@ open Fake
 
 open Paths
 open Projects
+open Commandline
 
 module Documentation = 
 
     let Generate() = 
         let docGenerator = PrivateProject(DocGenerator)
-        let path = Paths.ProjectOutputFolder docGenerator DotNetFramework.Net46
-        let generator = sprintf "%s/%s.exe" path docGenerator.Name
-        ExecProcess (fun p ->
-            p.WorkingDirectory <- Paths.Source("CodeGeneration") @@ docGenerator.Name
-            p.FileName <- generator
-        ) (TimeSpan.FromMinutes 3.) |> ignore
+        let path = Paths.ProjectOutputFolder docGenerator DotNetFramework.NetCoreApp2_1
+        let generator = sprintf "%s.dll %s" docGenerator.Name Commandline.docsBranch
+
+        DotNetCli.RunCommand(fun p ->
+            { p with
+                WorkingDir = path
+            }) generator
 
     // TODO: hook documentation validation into the process
     let Validate() = 

build/scripts/Projects.fsx

@@ -16,13 +16,13 @@ module Projects =
     type DotNetFramework = 
         | Net46 
         | NetStandard1_3
-        | NetCoreApp2_0
+        | NetCoreApp2_1
         static member All = [Net46; NetStandard1_3] 
         member this.Identifier = 
             match this with
-            | Net46 -> { MSBuild = "v4.6"; Nuget = "net46"; DefineConstants = if not isMono then "TRACE;NET46" else "NET46"; }
-            | NetStandard1_3 -> { MSBuild = "netstandard1.3"; Nuget = "netstandard1.3"; DefineConstants = if not isMono then "TRACE;DOTNETCORE" else "DOTNETCORE"; }
-            | NetCoreApp2_0 -> { MSBuild = "netcoreapp2.0"; Nuget = "netcoreapp2.0"; DefineConstants = if not isMono then "TRACE;DOTNETCORE" else "DOTNETCORE"; }
+            | Net46 -> { MSBuild = "v4.6"; Nuget = "net46"; DefineConstants = ""; }
+            | NetStandard1_3 -> { MSBuild = "netstandard1.3"; Nuget = "netstandard1.3"; DefineConstants = ""; }
+            | NetCoreApp2_1 -> { MSBuild = "netcoreapp2.1"; Nuget = "netcoreapp2.1"; DefineConstants = ""; }
 
     type Project =
         | Nest

build/scripts/Targets.fsx

@@ -34,6 +34,7 @@ open Differ.Differ
 
 Commandline.parse()
 
+Target "Touch" <| fun _ -> traceHeader "Touching build"
 Target "Build" <| fun _ -> traceHeader "STARTING BUILD"
 Target "Start" <| fun _ -> 
     match (isMono, Commandline.validMonoTarget) with
@@ -79,7 +80,10 @@ Target "Release" <| fun _ ->
     Release.GenerateNotes()
 
 Target "TestNugetPackage" <| fun _ -> 
-    Tests.RunReleaseUnitTests()
+    //RunReleaseUnitTests restores the canary nugetpackages in tests, since these end up being cached
+    //its too evasive to run on development machines or TC, Run only on AppVeyor containers.
+    if buildServer <> AppVeyor then Tests.RunUnitTests()
+    else Tests.RunReleaseUnitTests()
 
 Target "Canary" <| fun _ -> 
     trace "Running canary build" 
@@ -102,7 +106,7 @@ Target "Diff" <| fun _ ->
   =?> ("Version", hasBuildParam "version")
   ==> "Restore"
   =?> ("FullBuild", Commandline.needsFullBuild)
-  =?> ("Test", (not Commandline.skipTests))
+  =?> ("Test", (not Commandline.skipTests && Commandline.target <> "canary"))
   =?> ("InternalizeDependencies", (not isMono))
   ==> "InheritDoc"
   =?> ("Documentation", (not Commandline.skipDocs))
@@ -120,7 +124,7 @@ Target "Diff" <| fun _ ->
 
 "Version"
   ==> "Release"
-  =?> ("TestNugetPackage", (not isMono))
+  =?> ("TestNugetPackage", (not isMono && not Commandline.skipTests))
   ==> "Canary"
 
 "Start"
@@ -132,6 +136,7 @@ Target "Diff" <| fun _ ->
 "Build"
   ==> "Release"
 
+"Touch"
 "Start"
   ==> "Clean"
   ==> "Diff"

build/scripts/Testing.fsx

@@ -43,8 +43,9 @@ module Tests =
         let command = 
             let p = ["xunit"; "-parallel"; "all"; "-xml"; "../.." @@ Paths.Output("TestResults-Desktop-Clr.xml")] 
             match (target, buildingOnTravis) with 
+            //make sure we don't test against net46 on mono or travis systems
             | (_, true) 
-            | (Commandline.MultiTarget.One, _) -> ["-framework"; "netcoreapp2.0"] |> List.append p
+            | (Commandline.MultiTarget.One, _) -> ["-framework"; "netcoreapp2.1"] |> List.append p
             | _  -> p
 
         let dotnet = Tooling.BuildTooling("dotnet")

build/scripts/Tooling.fsx

@@ -109,12 +109,12 @@ module Tooling =
 
     let jetBrainsTools = [{ 
                             DotTraceTool.Name = "JetBrains DotTrace Self-Profile API";
-                            Download = "https://download-cf.jetbrains.com/resharper/JetBrains.Profiler.SelfSdk.2016.3.2.zip";
+                            Download = "https://download-cf.jetbrains.com/resharper/JetBrains.Profiler.SelfSdk.2017.3.2.zip";
                             TargetDir = "dottrace-selfprofile";
                          };
                          { 
                             DotTraceTool.Name = "JetBrains DotTrace Commandline Tools";
-                            Download = "https://download-cf.jetbrains.com/resharper/JetBrains.dotTrace.CommandLineTools.2016.3.20170126.121657.zip";
+                            Download = "https://download-cf.jetbrains.com/resharper/JetBrains.dotTrace.CommandLineTools.2017.3.2.zip";
                             TargetDir = "dottrace-commandline";
                          }]
 
@@ -139,7 +139,6 @@ module Tooling =
         let toolPath = commandLineTool @@ path
         member this.Exec arguments = execAt Environment.CurrentDirectory toolPath arguments
 
-    let DotTraceProfiler = new ProfilerTooling("ConsoleProfiler.exe")
     let DotTraceReporter = new ProfilerTooling("Reporter.exe")
     let DotTraceSnapshotStats = new ProfilerTooling("SnapshotStat.exe")
 
@@ -165,5 +164,5 @@ module Tooling =
             let result = execProcessWithTimeout toolPath arguments (TimeSpan.FromMinutes 5.) "."
             if result <> 0 then failwith (sprintf "Failed to run diff tooling for %s args: %A" exe arguments)
 
-    let JustAssembly = DiffTooling("JustAssembly.CommandLineTool.exe")      
+    let JustAssembly = DiffTooling("JustAssembly.CommandLineTool.exe")
 

contributing.md

@@ -12,7 +12,7 @@ We do ask that you sign the [Elasticsearch CLA](http://www.elasticsearch.org/con
 
 ## Coding Styleguide
 
-Please install the [editorconfig vs extension](https://visualstudiogallery.msdn.microsoft.com/c8bccfe2-650c-4b42-bc5c-845e21f96328)
+Please install the [Editorconfig vs extension](https://visualstudiogallery.msdn.microsoft.com/c8bccfe2-650c-4b42-bc5c-845e21f96328)
 this will automatically switch to our indentation, whitespace, newlines settings while working on our project
 **while leaving your default settings intact**.
 

global.json

@@ -1,6 +1,6 @@
 {
   "sdk": {
-    "version": "2.0.3"
+    "version": "2.1.300"
   },
   "version": "5.6.2"
 }