From f83c0c3c8caed8c29133abc684a73134cfae4b07 Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Mon, 8 Jan 2018 19:49:43 +1100 Subject: [PATCH 1/3] Diff assemblies WIP --- build/scripts/Commandline.fsx | 16 ++- build/scripts/Differ.fsx | 247 ++++++++++++++++++++++++++++++++++ build/scripts/Targets.fsx | 49 ++++++- build/scripts/Tooling.fsx | 20 ++- build/scripts/scripts.fsproj | 1 + 5 files changed, 329 insertions(+), 4 deletions(-) create mode 100644 build/scripts/Differ.fsx diff --git a/build/scripts/Commandline.fsx b/build/scripts/Commandline.fsx index 4cbccf38c28..ee70672c772 100644 --- a/build/scripts/Commandline.fsx +++ b/build/scripts/Commandline.fsx @@ -28,6 +28,7 @@ Targets: * canary [apikey] [feed] - create a canary nuget package based on the current version if [feed] and [apikey] are provided also pushes to upstream (myget) +* diff NOTE: both the `test` and `integrate` targets can be suffixed with `-all` to force the tests against all suported TFM's @@ -43,6 +44,7 @@ module Commandline = let private args = getBuildParamOrDefault "cmdline" "build" |> split ' ' let skipTests = args |> List.exists (fun x -> x = "skiptests") + let skipDocs = args |> List.exists (fun x -> x = "skipdocs") || isMono let seed = match args |> List.tryFind (fun x -> x.StartsWith("seed:")) with | Some t -> t.Replace("seed:", "") @@ -57,7 +59,7 @@ module Commandline = args |> List.filter ( fun x -> - x <> "skiptests" && x <> "source_serialization" && not (x.StartsWith("seed:")) && not (x.StartsWith("random:")) + x <> "skiptests" && x <> "skipdocs" && x <> "source_serialization" && not (x.StartsWith("seed:")) && not (x.StartsWith("random:")) ) let multiTarget = @@ -102,6 +104,13 @@ module Commandline = match Uri.TryCreate(candidate, UriKind.RelativeOrAbsolute) with | true, _ -> Some candidate | _ -> None + + let private (|IsDiff|_|) (candidate:string) = + let c = candidate |> toLower + match c with + | "github" | "nuget" | "directories" | "assemblies" -> Some c + | _ -> failwith (sprintf "Unknown diff type: %s" candidate) + let parse () = setEnvironVar "FAKEBUILD" "1" @@ -162,6 +171,11 @@ module Commandline = setBuildParam "clusterfilter" "ConnectionReuse" setBuildParam "numberOfConnections" numberOfConnections + | ["diff"; IsDiff diffType; firstVersionOrPath; secondVersionOrPath] -> + setBuildParam "diffType" diffType + setBuildParam "first" firstVersionOrPath + setBuildParam "second" secondVersionOrPath + | ["temp"; ] -> ignore() | ["canary"; ] -> ignore() | ["canary"; apiKey ] -> diff --git a/build/scripts/Differ.fsx b/build/scripts/Differ.fsx new file mode 100644 index 00000000000..9c125819f48 --- /dev/null +++ b/build/scripts/Differ.fsx @@ -0,0 +1,247 @@ +#I @"../../packages/build/FAKE/tools" +#r @"FakeLib.dll" +#nowarn "0044" //TODO sort out FAKE 5 + +#load @"Paths.fsx" +#load @"Tooling.fsx" + +open System +open System.IO + +open Fake +open System +open System.IO +open System.Net +open System.Text +open Fake.Git.CommandHelper + +open Paths +open Projects +open Tooling + +module Differ = + + /// The format of the output + type Format = + | Xml + | Markdown + | Asciidoc + + /// The github project compilation target. Determines how to compile the github commit + type GitHubTarget = + | Command of command:string * args:string list * resolveAssembly:(string -> string) + + // A github commit to diff + type GitHubCommit = { + /// The commit to diff against + Commit: string; + /// The compilation target. If not specified, will use the first *.sln found + CompileTarget : GitHubTarget; + /// The build output target. If not specified, a diff will be performed on all assemblies in the build output directories + OutputTarget: string; + } + + /// Diff the build output of two github commits + type GitHub = { + /// The github repository url + Url: Uri; + /// A temporary directory in which to diff the commits. If the directory already exists, it will use that + TempDir: string; + /// The first commit to diff against + FirstCommit: GitHubCommit; + /// The second commit to diff against + SecondCommit: GitHubCommit; + } + + /// Diff the assemblies in two nuget package versions + type Nuget = { + /// The nuget package id + Package: string; + /// A temporary directory in which to diff the packages. If the directory already exists, will be deleted first. + TempDir: string; + /// The first package version to diff against + FirstVersion: string; + /// The second package version to diff against + SecondVersion: string; + /// The framework version of the package + FrameworkVersion: string; + /// The nuget package sources. Defaults to nuget v2 and v3 feeds if empty + Sources: string list; + } + + /// Diff two different assemblies + type Assemblies = { + /// The path to the first assembly + FirstPath: string; + /// The path to the second assembly + SecondPath: string; + } + + /// Diff the assemblies in two different directories + type Directories = { + /// The path to the first directory + FirstDir: string; + /// The path to the second directory + SecondDir: string; + } + + /// The diff operation to perform + type Diff = + | GitHub of GitHub + | Nuget of Nuget + | Assemblies of Assemblies + | Directories of Directories + + type AssemblyDiff = { + /// The path to the first assembly + FirstPath: string; + /// The path to the second assembly + SecondPath: string; + } + + let private tempDir = Path.GetTempPath() Path.GetRandomFileName() + + let private downloadNugetPackages nuget = + let tempDir = nuget.TempDir "nuget" + DeleteDir tempDir + CreateDir tempDir + let versions = [nuget.FirstVersion; nuget.SecondVersion] + versions + |> Seq.map(fun v -> tempDir v) + |> Seq.iter CreateDir + + let nugetExe = Tooling.Nuget + + let sources = + if List.isEmpty nuget.Sources then ["https://www.nuget.org/api/v2/"; "https://api.nuget.org/v3/index.json"] + else nuget.Sources + |> List.map (fun s -> sprintf "-Source %s" s) + |> String.concat " " + + let packageVersionPath dir packageVersion = + let desiredFrameworkVersion = Directory.GetDirectories dir + |> Array.tryFind (fun f -> nuget.FrameworkVersion = Path.GetFileName f) + match desiredFrameworkVersion with + | Some f -> f |> Path.GetFullPath + | _ -> failwith (sprintf "Nuget package %s, version %s, does not contain framework version %s in %s" + nuget.Package + packageVersion + nuget.FrameworkVersion + dir) + + versions + |> Seq.map(fun v -> + let workingDir = tempDir @@ v + let exitCode = nugetExe.ExecIn workingDir ["install"; nuget.Package; "-Version"; v; sources; "-ExcludeVersion -NonInteractive"] + if exitCode <> 0 then failwith (sprintf "Error downloading nuget package version: %s" v) + + // assumes DLLs are in the lib folder + let packageDirs = Directory.GetDirectories workingDir + |> Array.filter (fun f -> nuget.Package <> Path.GetFileName f) + |> Array.map(fun f -> (f @@ "lib") |> Path.GetFullPath) + + let targetPath = packageVersionPath (workingDir @@ nuget.Package @@ "lib") v + + // targeting an individual assembly or the directory of assemblies + let target = + let assemblyNamedAfterPackage = + Directory.EnumerateFiles(targetPath, "*.dll") + |> Seq.tryPick (fun f -> + let fileName = Path.GetFileNameWithoutExtension f + if String.Equals(fileName, nuget.Package, StringComparison.OrdinalIgnoreCase) + then Some f + else None) + match assemblyNamedAfterPackage with + | Some a -> a + | _ -> targetPath + + // copy all dependent package assemblies into target dir + for packageDir in packageDirs do + let path = packageVersionPath packageDir v + path |> Directory.GetFiles |> CopyFiles targetPath + + target + ) + |> Seq.toList + + let private cloneAndBuildGitRepo (git:GitHub) = + let fullTempPath = git.TempDir |> Path.GetFullPath + let gitDir = fullTempPath "nest-diff" + let repo = fullTempPath @@ "github" + + if (directoryExists repo |> not) then + CreateDir repo + directRunGitCommandAndFail repo (sprintf "clone %s ." git.Url.AbsoluteUri) + + let checkoutAndBuild (commit:GitHubCommit) = + directRunGitCommandAndFail repo "reset --hard" + directRunGitCommandAndFail repo (sprintf "checkout %s" commit.Commit) + let outputPath = fullTempPath @@ commit.Commit + + let out = match commit.CompileTarget with + | Command (c, a, f) -> + let failIfError exitCode = + if exitCode > 0 then + let message = sprintf "Command %s failed" c + traceError message + failwith message + + ExecProcess(fun p -> + p.WorkingDirectory <- repo + p.FileName <- c + p.Arguments <- String.concat " " a + ) (TimeSpan.FromMinutes 5.) + |> failIfError + + let buildOutputPath = f(repo) + CopyDir outputPath buildOutputPath (fun s -> true) + outputPath + + if isNullOrEmpty commit.OutputTarget then out + else out @@ commit.OutputTarget + + [git.FirstCommit; git.SecondCommit] |> List.map checkoutAndBuild + + let private convertToMarkdown path = trace "TODO: Convert to markdown" + + let private convertToAsciidoc path = trace "TODO: Convert to asciidoc" + + /// Generates a diff between assembly files, assembly directories, assemblies in nuget packages + let Generate(diff: Diff, format:Format) = + let pairAssembliesInDirectories directories = + let firstDirectory = directories |> Seq.head + let lastDirectory = directories |> Seq.last + [ for file in Directory.EnumerateFiles firstDirectory do + let otherFile = lastDirectory Path.GetFileName file + if fileExists otherFile then + yield { FirstPath = file; SecondPath = otherFile } ] + + let assemblies = + match diff with + | GitHub g -> + let checkouts = cloneAndBuildGitRepo g + if Seq.forall (fun t -> Path.HasExtension t && Path.GetExtension t = ".dll") checkouts + then [{ FirstPath = checkouts.Head; SecondPath = List.last checkouts }] + else pairAssembliesInDirectories checkouts + | Nuget n -> + let packages = downloadNugetPackages n + if Seq.forall (fun t -> Path.HasExtension t && Path.GetExtension t = ".dll") packages + then [{ FirstPath = packages.Head; SecondPath = List.last packages }] + else pairAssembliesInDirectories packages + | Assemblies a -> [{ FirstPath = a.FirstPath; SecondPath = a.SecondPath }] + | Directories d -> pairAssembliesInDirectories [d.FirstDir; d.SecondDir] + + let appendIfNotNullOrEmpty v f b = + if (String.IsNullOrEmpty v = false) then Printf.bprintf b f v + b + + for diff in assemblies do + let file = diff.FirstPath |> Path.GetFileNameWithoutExtension + let path = Paths.Output("diff") sprintf "%s.xml" file + Tooling.JustAssembly.Exec [diff.FirstPath; diff.SecondPath; path] + match format with + | Xml -> () + | Markdown -> convertToMarkdown path + | Asciidoc -> convertToAsciidoc path + + diff --git a/build/scripts/Targets.fsx b/build/scripts/Targets.fsx index f0f21dbed3e..4a6bf8db70d 100644 --- a/build/scripts/Targets.fsx +++ b/build/scripts/Targets.fsx @@ -11,6 +11,7 @@ #load @"Benchmarking.fsx" #load @"Profiling.fsx" #load @"XmlDocPatcher.fsx" +#load @"Differ.fsx" #nowarn "0044" //TODO sort out FAKE 5 open System @@ -28,6 +29,8 @@ open XmlDocPatcher open Documentation open Signing open Commandline +open Differ +open Differ.Differ Commandline.parse() @@ -82,6 +85,47 @@ Target "Canary" <| fun _ -> let apiKey = (getBuildParam "apikey"); let feed = (getBuildParamOrDefault "feed" "elasticsearch-net"); if (not (String.IsNullOrWhiteSpace apiKey) || apiKey = "ignore") then Release.PublishCanaryBuild apiKey feed + +Target "Diff" <| fun _ -> + let first = getBuildParam "first" + let second = getBuildParam "second" + let tempDir = System.IO.Path.GetTempPath() + let diff = + match getBuildParam "diffType" with + | "github" -> + let commit = { + Commit = "" + CompileTarget = Command("build.bat", ["skiptests"], fun o -> o @@ @"build\output\Nest\net46") + OutputTarget = "Nest.dll" + } + GitHub { + Url = new Uri(Paths.Repository) + TempDir = tempDir + FirstCommit = { commit with Commit = first } + SecondCommit = { commit with Commit = second } + } + | "nuget" -> + Nuget { + Package = "NEST" + TempDir = tempDir + FirstVersion = first + SecondVersion = second + FrameworkVersion = "net46" + Sources = [] + } + | "directories" -> + Directories { + FirstDir = first + SecondDir = second + } + | "assemblies" -> + Assemblies { + FirstPath = first + SecondPath = second + } + | d -> failwith (sprintf "Unknown diff type: %s" d) + tracefn "Performing diff using %A" diff + Differ.Generate(diff, Format.Xml) // Dependencies "Start" @@ -92,7 +136,7 @@ Target "Canary" <| fun _ -> =?> ("Test", (not Commandline.skipTests)) =?> ("InternalizeDependencies", (not isMono)) ==> "InheritDoc" - =?> ("Documentation", (not isMono)) + =?> ("Documentation", (not Commandline.skipDocs)) ==> "Build" "Start" @@ -119,5 +163,8 @@ Target "Canary" <| fun _ -> "Build" ==> "Release" +"Start" + ==> "Diff" + RunTargetOrListTargets() diff --git a/build/scripts/Tooling.fsx b/build/scripts/Tooling.fsx index 35614f468f4..945b7df4c0b 100644 --- a/build/scripts/Tooling.fsx +++ b/build/scripts/Tooling.fsx @@ -7,6 +7,7 @@ open System open System.IO open System.Diagnostics open System.Net +open System.Text.RegularExpressions #load @"Paths.fsx" @@ -143,11 +144,26 @@ module Tooling = let DotTraceSnapshotStats = new ProfilerTooling("SnapshotStat.exe") type DotNetTooling(exe) = - member this.Exec arguments = + member this.Exec arguments = this.ExecWithTimeout arguments (TimeSpan.FromMinutes 30.) member this.ExecWithTimeout arguments timeout = let result = execProcessWithTimeout exe arguments timeout "." if result <> 0 then failwith (sprintf "Failed to run dotnet tooling for %s args: %A" exe arguments) - let DotNet = DotNetTooling("dotnet.exe") \ No newline at end of file + let DotNet = DotNetTooling("dotnet.exe") + + type DiffTooling(exe) = + let installPath = "C:\Program Files (x86)\Progress\JustAssembly\Libraries" + let downloadPage = "https://www.telerik.com/download-trial-file/v2/justassembly" + let toolPath = installPath @@ exe + + member this.Exec arguments = + if (directoryExists installPath |> not) then + failwith (sprintf "JustAssembly is not installed in the default location %s. Download and install from %s" installPath downloadPage) + + let result = execProcessWithTimeout toolPath arguments (TimeSpan.FromMinutes 5.) "." + if result <> 0 then failwith (sprintf "Failed to run diff tooling for %s args: %A" exe arguments) + + let JustAssembly = DiffTooling("JustAssembly.CommandLineTool.exe") + \ No newline at end of file diff --git a/build/scripts/scripts.fsproj b/build/scripts/scripts.fsproj index 8ebfea164e9..cf911060062 100644 --- a/build/scripts/scripts.fsproj +++ b/build/scripts/scripts.fsproj @@ -45,6 +45,7 @@ + From 56ccada2566493d8da2102c8b15fa1228f9fc088 Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Tue, 9 Jan 2018 15:49:44 +1100 Subject: [PATCH 2/3] Output diff as Markdown --- build/scripts/Commandline.fsx | 37 +++++++-- build/scripts/Differ.fsx | 140 ++++++++++++++++++++++++++++------ build/scripts/Projects.fsx | 1 - build/scripts/Targets.fsx | 42 ++-------- 4 files changed, 154 insertions(+), 66 deletions(-) diff --git a/build/scripts/Commandline.fsx b/build/scripts/Commandline.fsx index ee70672c772..4070e1962ca 100644 --- a/build/scripts/Commandline.fsx +++ b/build/scripts/Commandline.fsx @@ -28,7 +28,7 @@ Targets: * canary [apikey] [feed] - create a canary nuget package based on the current version if [feed] and [apikey] are provided also pushes to upstream (myget) -* diff +* diff [format] NOTE: both the `test` and `integrate` targets can be suffixed with `-all` to force the tests against all suported TFM's @@ -110,7 +110,18 @@ module Commandline = match c with | "github" | "nuget" | "directories" | "assemblies" -> Some c | _ -> failwith (sprintf "Unknown diff type: %s" candidate) - + + let private (|IsProject|_|) (candidate:string) = + let c = candidate |> toLower + match c with + | "nest" | "elasticsearch.net" | "nest.jsonnetserializer" -> Some c + | _ -> None + + let private (|IsFormat|_|) (candidate:string) = + let c = candidate |> toLower + match c with + | "xml" | "markdown" | "asciidoc" -> Some c + | _ -> None let parse () = setEnvironVar "FAKEBUILD" "1" @@ -170,12 +181,28 @@ module Commandline = setBuildParam "esversions" esVersions setBuildParam "clusterfilter" "ConnectionReuse" setBuildParam "numberOfConnections" numberOfConnections - + + | ["diff"; IsDiff diffType; IsProject project; firstVersionOrPath; secondVersionOrPath; IsFormat format] -> + setBuildParam "diffType" diffType + setBuildParam "project" project + setBuildParam "first" firstVersionOrPath + setBuildParam "second" secondVersionOrPath + setBuildParam "format" format + | ["diff"; IsDiff diffType; IsProject project; firstVersionOrPath; secondVersionOrPath] -> + setBuildParam "diffType" diffType + setBuildParam "project" project + setBuildParam "first" firstVersionOrPath + setBuildParam "second" secondVersionOrPath + | ["diff"; IsDiff diffType; firstVersionOrPath; secondVersionOrPath; IsFormat format] -> + setBuildParam "diffType" diffType + setBuildParam "first" firstVersionOrPath + setBuildParam "second" secondVersionOrPath + setBuildParam "format" format | ["diff"; IsDiff diffType; firstVersionOrPath; secondVersionOrPath] -> setBuildParam "diffType" diffType setBuildParam "first" firstVersionOrPath - setBuildParam "second" secondVersionOrPath - + setBuildParam "second" secondVersionOrPath + | ["temp"; ] -> ignore() | ["canary"; ] -> ignore() | ["canary"; apiKey ] -> diff --git a/build/scripts/Differ.fsx b/build/scripts/Differ.fsx index 9c125819f48..d44f95c4b47 100644 --- a/build/scripts/Differ.fsx +++ b/build/scripts/Differ.fsx @@ -1,9 +1,11 @@ #I @"../../packages/build/FAKE/tools" #r @"FakeLib.dll" +#r @"System.Xml.Linq" #nowarn "0044" //TODO sort out FAKE 5 #load @"Paths.fsx" #load @"Tooling.fsx" +#load @"Projects.fsx" open System open System.IO @@ -13,6 +15,8 @@ open System open System.IO open System.Net open System.Text +open System.Text.RegularExpressions +open System.Xml.Linq open Fake.Git.CommandHelper open Paths @@ -92,6 +96,7 @@ module Differ = | Assemblies of Assemblies | Directories of Directories + /// The two assemblies to diff type AssemblyDiff = { /// The path to the first assembly FirstPath: string; @@ -99,19 +104,16 @@ module Differ = SecondPath: string; } - let private tempDir = Path.GetTempPath() Path.GetRandomFileName() - let private downloadNugetPackages nuget = let tempDir = nuget.TempDir "nuget" DeleteDir tempDir CreateDir tempDir let versions = [nuget.FirstVersion; nuget.SecondVersion] + versions |> Seq.map(fun v -> tempDir v) |> Seq.iter CreateDir - let nugetExe = Tooling.Nuget - let sources = if List.isEmpty nuget.Sources then ["https://www.nuget.org/api/v2/"; "https://api.nuget.org/v3/index.json"] else nuget.Sources @@ -122,7 +124,7 @@ module Differ = let desiredFrameworkVersion = Directory.GetDirectories dir |> Array.tryFind (fun f -> nuget.FrameworkVersion = Path.GetFileName f) match desiredFrameworkVersion with - | Some f -> f |> Path.GetFullPath + | Some f -> f |> Path.GetFullPath | _ -> failwith (sprintf "Nuget package %s, version %s, does not contain framework version %s in %s" nuget.Package packageVersion @@ -132,7 +134,7 @@ module Differ = versions |> Seq.map(fun v -> let workingDir = tempDir @@ v - let exitCode = nugetExe.ExecIn workingDir ["install"; nuget.Package; "-Version"; v; sources; "-ExcludeVersion -NonInteractive"] + let exitCode = Tooling.Nuget.ExecIn workingDir ["install"; nuget.Package; "-Version"; v; sources; "-ExcludeVersion -NonInteractive"] if exitCode <> 0 then failwith (sprintf "Error downloading nuget package version: %s" v) // assumes DLLs are in the lib folder @@ -166,7 +168,6 @@ module Differ = let private cloneAndBuildGitRepo (git:GitHub) = let fullTempPath = git.TempDir |> Path.GetFullPath - let gitDir = fullTempPath "nest-diff" let repo = fullTempPath @@ "github" if (directoryExists repo |> not) then @@ -190,7 +191,7 @@ module Differ = p.WorkingDirectory <- repo p.FileName <- c p.Arguments <- String.concat " " a - ) (TimeSpan.FromMinutes 5.) + ) (TimeSpan.FromMinutes 10.) |> failIfError let buildOutputPath = f(repo) @@ -202,19 +203,114 @@ module Differ = [git.FirstCommit; git.SecondCommit] |> List.map checkoutAndBuild - let private convertToMarkdown path = trace "TODO: Convert to markdown" + type DiffType = + | Deleted + | Modified + | New - let private convertToAsciidoc path = trace "TODO: Convert to asciidoc" + let private convertDiffType = function + | "Deleted" -> Deleted + | "Modified" -> Modified + | "New" -> New + | d -> failwith (sprintf "unknown diff type: %s" d) + + let private attributeValue name (element:XElement) = + let attribute = element.Attribute(XName.op_Implicit name) + if attribute <> null then attribute.Value else "" + + let private convertToMarkdown (path:string) first second = + let name = path |> Path.GetFileNameWithoutExtension + let doc = XDocument.Load path + let output = Path.ChangeExtension(path, "md") + use file = File.OpenWrite <| output + use writer = new StreamWriter(file) + writer.WriteLine(sprintf "# Breaking changes for %s between %s and %s" name first second) + writer.WriteLine() + + for element in doc.Descendants(XName.op_Implicit "Type") do + let typeName = element |> attributeValue "Name" |> replace (sprintf "%s." name) "" + let diffType = element |> attributeValue "DiffType" |> convertDiffType + match diffType with + | Deleted -> writer.WriteLine(sprintf "## `%s` is deleted" typeName) + | New -> writer.WriteLine(sprintf "## `%s` is added" typeName) + | Modified -> + let members = + Seq.append + (element.Elements(XName.op_Implicit "Method")) + (element.Elements(XName.op_Implicit "Property")) + + if Seq.isEmpty members |> not then + writer.WriteLine(sprintf "## `%s`" typeName) + for m in members do + let memberName = m |> attributeValue "Name" + if isNotNullOrEmpty memberName then + let diffType = m |> attributeValue "DiffType" + if isNotNullOrEmpty diffType then + match convertDiffType diffType with + | Deleted -> writer.WriteLine(sprintf "### `%s` is deleted" memberName) + | New -> writer.WriteLine(sprintf "### `%s` is added" memberName) + | Modified -> + match (m.Descendants(XName.op_Implicit "DiffItem") |> Seq.tryHead) with + | Some diffItem -> + writer.WriteLine(sprintf "### `%s`" memberName) + let diffDescription = diffItem.Value + writer.WriteLine(Regex.Replace(diffDescription, "changed from (.*?) to (.*).", "changed from `$1` to `$2`.")) + | None -> () + + let private convertToAsciidoc path first second = + trace "TODO: Convert to asciidoc" /// Generates a diff between assembly files, assembly directories, assemblies in nuget packages - let Generate(diff: Diff, format:Format) = + let Generate(diffType:string, project:string, first:string, second:string, format:string) = + let tempDir = Path.GetTempPath() "nest-diff" + + let targetProject = + match project |> toLower with + | "nest" -> (DotNetProject.Project Nest).Name + | "nest.jsonnetserializer" -> (DotNetProject.Project NestJsonNetSerializer).Name + | "elasticsearch.net" -> (DotNetProject.Project ElasticsearchNet).Name + | _ -> "" + + let targetFormat = + match format |> toLower with + | "xml" -> Xml + | "markdown" -> Markdown + | "asciidoc" -> Asciidoc + | _ -> Xml + + let diff = + match diffType with + | "github" -> + let commit = { + Commit = "" + CompileTarget = Command("build.bat", ["skiptests"], fun o -> o @@ @"build\output\Nest\net46") + OutputTarget = if isNotNullOrEmpty targetProject then sprintf "%s.dll" targetProject else targetProject + } + GitHub { + Url = new Uri(Paths.Repository) + TempDir = tempDir + FirstCommit = { commit with Commit = first } + SecondCommit = { commit with Commit = second } + } + | "nuget" -> + Nuget { + Package = if isNullOrEmpty targetProject then "NEST" else targetProject + TempDir = tempDir + FirstVersion = first + SecondVersion = second + FrameworkVersion = "net46" + Sources = [] + } + | "directories" -> Directories { FirstDir = first; SecondDir = second } + | "assemblies" -> Assemblies { FirstPath = first; SecondPath = second } + | d -> failwith (sprintf "Unknown diff type: %s" d) + let pairAssembliesInDirectories directories = let firstDirectory = directories |> Seq.head let lastDirectory = directories |> Seq.last - [ for file in Directory.EnumerateFiles firstDirectory do + [ for file in Directory.EnumerateFiles(firstDirectory, "*.dll") do let otherFile = lastDirectory Path.GetFileName file - if fileExists otherFile then - yield { FirstPath = file; SecondPath = otherFile } ] + if fileExists otherFile then yield { FirstPath = file; SecondPath = otherFile } ] let assemblies = match diff with @@ -231,17 +327,15 @@ module Differ = | Assemblies a -> [{ FirstPath = a.FirstPath; SecondPath = a.SecondPath }] | Directories d -> pairAssembliesInDirectories [d.FirstDir; d.SecondDir] - let appendIfNotNullOrEmpty v f b = - if (String.IsNullOrEmpty v = false) then Printf.bprintf b f v - b - for diff in assemblies do let file = diff.FirstPath |> Path.GetFileNameWithoutExtension - let path = Paths.Output("diff") sprintf "%s.xml" file - Tooling.JustAssembly.Exec [diff.FirstPath; diff.SecondPath; path] - match format with + let outputFile = Paths.Output("diff") sprintf "%s.xml" file |> Path.GetFullPath + let outputDir = outputFile |> Path.GetDirectoryName + if directoryExists outputDir |> not then CreateDir outputDir + Tooling.JustAssembly.Exec [diff.FirstPath; diff.SecondPath; outputFile] + match targetFormat with | Xml -> () - | Markdown -> convertToMarkdown path - | Asciidoc -> convertToAsciidoc path + | Markdown -> convertToMarkdown outputFile first second + | Asciidoc -> convertToAsciidoc outputFile first second diff --git a/build/scripts/Projects.fsx b/build/scripts/Projects.fsx index 082f530bd8c..3e3384a4a4c 100644 --- a/build/scripts/Projects.fsx +++ b/build/scripts/Projects.fsx @@ -35,7 +35,6 @@ module Projects = | Tests | DocGenerator - type DotNetProject = | Project of Project | PrivateProject of PrivateProject diff --git a/build/scripts/Targets.fsx b/build/scripts/Targets.fsx index 4a6bf8db70d..96229987ec2 100644 --- a/build/scripts/Targets.fsx +++ b/build/scripts/Targets.fsx @@ -87,45 +87,13 @@ Target "Canary" <| fun _ -> if (not (String.IsNullOrWhiteSpace apiKey) || apiKey = "ignore") then Release.PublishCanaryBuild apiKey feed Target "Diff" <| fun _ -> + let diffType = getBuildParam "diffType" + let project = getBuildParam "project" let first = getBuildParam "first" let second = getBuildParam "second" - let tempDir = System.IO.Path.GetTempPath() - let diff = - match getBuildParam "diffType" with - | "github" -> - let commit = { - Commit = "" - CompileTarget = Command("build.bat", ["skiptests"], fun o -> o @@ @"build\output\Nest\net46") - OutputTarget = "Nest.dll" - } - GitHub { - Url = new Uri(Paths.Repository) - TempDir = tempDir - FirstCommit = { commit with Commit = first } - SecondCommit = { commit with Commit = second } - } - | "nuget" -> - Nuget { - Package = "NEST" - TempDir = tempDir - FirstVersion = first - SecondVersion = second - FrameworkVersion = "net46" - Sources = [] - } - | "directories" -> - Directories { - FirstDir = first - SecondDir = second - } - | "assemblies" -> - Assemblies { - FirstPath = first - SecondPath = second - } - | d -> failwith (sprintf "Unknown diff type: %s" d) - tracefn "Performing diff using %A" diff - Differ.Generate(diff, Format.Xml) + let format = getBuildParam "format" + tracefn "Performing %s diff %s using %s with %s and %s" format project diffType first second + Differ.Generate(diffType, project, first, second, format) // Dependencies "Start" From 73ef01e2cb78be457b3bc2b887b3fe6a4f8de621 Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Wed, 10 Jan 2018 15:01:45 +1100 Subject: [PATCH 3/3] Output diff as asciidoc --- build/scripts/Differ.fsx | 145 ++++++++++++++++++++++++++------------ build/scripts/Targets.fsx | 1 + 2 files changed, 99 insertions(+), 47 deletions(-) diff --git a/build/scripts/Differ.fsx b/build/scripts/Differ.fsx index d44f95c4b47..3781471bbe9 100644 --- a/build/scripts/Differ.fsx +++ b/build/scripts/Differ.fsx @@ -13,9 +13,11 @@ open System.IO open Fake open System open System.IO +open System.Linq open System.Net open System.Text open System.Text.RegularExpressions +open System.Xml open System.Xml.Linq open Fake.Git.CommandHelper @@ -33,13 +35,13 @@ module Differ = /// The github project compilation target. Determines how to compile the github commit type GitHubTarget = - | Command of command:string * args:string list * resolveAssembly:(string -> string) + | Command of command:string * args:string list * resolveAssemblies:(string -> string) // A github commit to diff type GitHubCommit = { /// The commit to diff against Commit: string; - /// The compilation target. If not specified, will use the first *.sln found + /// The compilation target. CompileTarget : GitHubTarget; /// The build output target. If not specified, a diff will be performed on all assemblies in the build output directories OutputTarget: string; @@ -194,7 +196,7 @@ module Differ = ) (TimeSpan.FromMinutes 10.) |> failIfError - let buildOutputPath = f(repo) + let buildOutputPath = f repo CopyDir outputPath buildOutputPath (fun s -> true) outputPath @@ -212,63 +214,104 @@ module Differ = | "Deleted" -> Deleted | "Modified" -> Modified | "New" -> New - | d -> failwith (sprintf "unknown diff type: %s" d) + | d -> failwithf "unknown diff type: %s" d let private attributeValue name (element:XElement) = let attribute = element.Attribute(XName.op_Implicit name) if attribute <> null then attribute.Value else "" + let private elements name (element:XContainer) = element.Elements(XName.op_Implicit name) + + let private descendents name (element:XContainer) = element.Descendants(XName.op_Implicit name) + let private convertToMarkdown (path:string) first second = let name = path |> Path.GetFileNameWithoutExtension - let doc = XDocument.Load path - let output = Path.ChangeExtension(path, "md") - use file = File.OpenWrite <| output - use writer = new StreamWriter(file) - writer.WriteLine(sprintf "# Breaking changes for %s between %s and %s" name first second) - writer.WriteLine() - - for element in doc.Descendants(XName.op_Implicit "Type") do - let typeName = element |> attributeValue "Name" |> replace (sprintf "%s." name) "" - let diffType = element |> attributeValue "DiffType" |> convertDiffType - match diffType with - | Deleted -> writer.WriteLine(sprintf "## `%s` is deleted" typeName) - | New -> writer.WriteLine(sprintf "## `%s` is added" typeName) - | Modified -> - let members = - Seq.append - (element.Elements(XName.op_Implicit "Method")) - (element.Elements(XName.op_Implicit "Property")) - - if Seq.isEmpty members |> not then - writer.WriteLine(sprintf "## `%s`" typeName) - for m in members do - let memberName = m |> attributeValue "Name" - if isNotNullOrEmpty memberName then - let diffType = m |> attributeValue "DiffType" - if isNotNullOrEmpty diffType then - match convertDiffType diffType with - | Deleted -> writer.WriteLine(sprintf "### `%s` is deleted" memberName) - | New -> writer.WriteLine(sprintf "### `%s` is added" memberName) - | Modified -> - match (m.Descendants(XName.op_Implicit "DiffItem") |> Seq.tryHead) with - | Some diffItem -> - writer.WriteLine(sprintf "### `%s`" memberName) - let diffDescription = diffItem.Value - writer.WriteLine(Regex.Replace(diffDescription, "changed from (.*?) to (.*).", "changed from `$1` to `$2`.")) - | None -> () + try + let doc = XDocument.Load path + let output = Path.ChangeExtension(path, "md") + DeleteFile output + use file = File.OpenWrite <| output + use writer = new StreamWriter(file) + writer.WriteLine(sprintf "# Breaking changes for %s between %s and %s" name first second) + writer.WriteLine() + + for element in (doc |> descendents "Type") do + let typeName = element |> attributeValue "Name" |> replace (sprintf "%s." name) "" + let diffType = element |> attributeValue "DiffType" |> convertDiffType + match diffType with + | Deleted -> writer.WriteLine(sprintf "## `%s` is deleted" typeName) + | New -> writer.WriteLine(sprintf "## `%s` is added" typeName) + | Modified -> + let members = Seq.append (element |> elements "Method") (element |> elements "Property") + if Seq.isEmpty members |> not then + writer.WriteLine(sprintf "## `%s`" typeName) + for m in members do + let memberName = m |> attributeValue "Name" + if isNotNullOrEmpty memberName then + let diffType = m |> attributeValue "DiffType" + if isNotNullOrEmpty diffType then + match convertDiffType diffType with + | Deleted -> writer.WriteLine(sprintf "### `%s` is deleted" memberName) + | New -> writer.WriteLine(sprintf "### `%s` is added" memberName) + | Modified -> + match (m.Descendants(XName.op_Implicit "DiffItem") |> Seq.tryHead) with + | Some diffItem -> + writer.WriteLine(sprintf "### `%s`" memberName) + let diffDescription = diffItem.Value + writer.WriteLine(Regex.Replace(diffDescription, "changed from (.*?) to (.*).", "changed from `$1` to `$2`.")) + | None -> () + with + | :? XmlException -> ignore() let private convertToAsciidoc path first second = - trace "TODO: Convert to asciidoc" + let name = path |> Path.GetFileNameWithoutExtension + try + let doc = XDocument.Load path + let output = Path.ChangeExtension(path, "asciidoc") + DeleteFile output + use file = File.OpenWrite <| output + use writer = new StreamWriter(file) + writer.WriteLine(name |> replace "." "-" |> sprintf "[[%s-breaking-changes]]") + writer.WriteLine(sprintf "== Breaking changes for %s between %s and %s" name first second) + writer.WriteLine() + + for element in (doc |> descendents "Type") do + let typeName = element |> attributeValue "Name" |> replace (sprintf "%s." name) "" + let diffType = element |> attributeValue "DiffType" |> convertDiffType + match diffType with + | Deleted -> writer.WriteLine(sprintf "[float]%s=== `%s` is deleted" Environment.NewLine typeName) + | New -> writer.WriteLine(sprintf "[float]%s=== `%s` is added" Environment.NewLine typeName) + | Modified -> + let members = Seq.append (element |> elements "Method") (element |> elements "Property") + if Seq.isEmpty members |> not then + writer.WriteLine(sprintf "[float]%s=== `%s`" Environment.NewLine typeName) + for m in members do + let memberName = m |> attributeValue "Name" + if isNotNullOrEmpty memberName then + let diffType = m |> attributeValue "DiffType" + if isNotNullOrEmpty diffType then + match convertDiffType diffType with + | Deleted -> writer.WriteLine(sprintf "[float]%s==== `%s` is deleted" Environment.NewLine memberName) + | New -> writer.WriteLine(sprintf "[float]%s==== `%s` is added" Environment.NewLine memberName) + | Modified -> + match (m.Descendants(XName.op_Implicit "DiffItem") |> Seq.tryHead) with + | Some diffItem -> + writer.WriteLine(sprintf "[float]%s==== `%s`" Environment.NewLine memberName) + let diffDescription = diffItem.Value + writer.WriteLine(Regex.Replace(diffDescription, "changed from (.*?) to (.*).", "changed from `$1` to `$2`.")) + | None -> () + with + | :? XmlException -> ignore() - /// Generates a diff between assembly files, assembly directories, assemblies in nuget packages + /// Generates a diff between assemblies let Generate(diffType:string, project:string, first:string, second:string, format:string) = let tempDir = Path.GetTempPath() "nest-diff" let targetProject = match project |> toLower with - | "nest" -> (DotNetProject.Project Nest).Name - | "nest.jsonnetserializer" -> (DotNetProject.Project NestJsonNetSerializer).Name - | "elasticsearch.net" -> (DotNetProject.Project ElasticsearchNet).Name + | "nest" -> (Project Nest).Name + | "nest.jsonnetserializer" -> (Project NestJsonNetSerializer).Name + | "elasticsearch.net" -> (Project ElasticsearchNet).Name | _ -> "" let targetFormat = @@ -283,7 +326,14 @@ module Differ = | "github" -> let commit = { Commit = "" - CompileTarget = Command("build.bat", ["skiptests"], fun o -> o @@ @"build\output\Nest\net46") + CompileTarget = Command( + "build.bat", + ["skiptests"], + fun o -> + let outputDir = o @@ @"build\output\Nest\net46" + if directoryExists outputDir && Directory.EnumerateFileSystemEntries(outputDir).Any() then outputDir + else o @@ @"src\Nest\bin\Release\net46" + ) OutputTarget = if isNotNullOrEmpty targetProject then sprintf "%s.dll" targetProject else targetProject } GitHub { @@ -312,6 +362,7 @@ module Differ = let otherFile = lastDirectory Path.GetFileName file if fileExists otherFile then yield { FirstPath = file; SecondPath = otherFile } ] + /// returns a list of AssemblyDiff pairs let assemblies = match diff with | GitHub g -> @@ -329,7 +380,7 @@ module Differ = for diff in assemblies do let file = diff.FirstPath |> Path.GetFileNameWithoutExtension - let outputFile = Paths.Output("diff") sprintf "%s.xml" file |> Path.GetFullPath + let outputFile = Paths.Output("Diffs") sprintf "%s.xml" file |> Path.GetFullPath let outputDir = outputFile |> Path.GetDirectoryName if directoryExists outputDir |> not then CreateDir outputDir Tooling.JustAssembly.Exec [diff.FirstPath; diff.SecondPath; outputFile] diff --git a/build/scripts/Targets.fsx b/build/scripts/Targets.fsx index 96229987ec2..ddceaf177e7 100644 --- a/build/scripts/Targets.fsx +++ b/build/scripts/Targets.fsx @@ -132,6 +132,7 @@ Target "Diff" <| fun _ -> ==> "Release" "Start" + ==> "Clean" ==> "Diff" RunTargetOrListTargets()