From a89cdce2b17820cf25a464c8283c86c6faf1b132 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 1 Jul 2020 12:44:37 +0200 Subject: [PATCH 1/3] Introduce .ci/make.sh Initial stab at common entry script for all the Elasticsearch clients. This puts our build through a docker container so that folks/CI does not need to install dependencies on their machines to build artifacts. > ./.ci/make.sh release VERSION will output our packages, release notes, API reports in `.ci/output`. This also updates our build to use: https://github.com/nullean/nupkg-validator This is the tool version of our build script targets that - validate the dlls are build in release mode - validate the dll version numbers - validate the dll assemly identity - validate the dll assembly names. With the added benefit that the dependency on `sn` is gone which is not available on the dotnet base docker images that we use. `.ci/make.sh` is not done yet, will iterate on this as the flow in the unified release manager becomes more apparant. For example right now we bump the version after release but we might need to do so prior. --- .ci/DockerFile | 17 ++++---- .ci/Jenkins.csproj | 6 +++ .ci/make.sh | 44 +++++++++++++++++++++ .dockerignore | 8 +--- .gitignore | 2 + build/scripts/Commandline.fs | 8 +++- build/scripts/ReposTooling.fs | 5 +++ build/scripts/Targets.fs | 8 +++- build/scripts/Versioning.fs | 73 +++-------------------------------- build/scripts/scripts.fsproj | 1 + dotnet-tools.json | 6 +++ 11 files changed, 94 insertions(+), 84 deletions(-) create mode 100755 .ci/make.sh diff --git a/.ci/DockerFile b/.ci/DockerFile index 95c77cbbba2..74238516204 100644 --- a/.ci/DockerFile +++ b/.ci/DockerFile @@ -3,8 +3,12 @@ FROM mcr.microsoft.com/dotnet/core/sdk:${DOTNET_VERSION} AS elasticsearch-net-bu WORKDIR /sln + COPY ./*.sln ./nuget.config ./*.Build.props ./*.Build.targets ./ +COPY ./dotnet-tools.json ./ +RUN dotnet tool restore + # todo standardize on Build.props as Directory.Build.props needs that form COPY ./src/*.Build.props ./src/ COPY ./tests/*.Build.props ./tests/ @@ -21,13 +25,12 @@ RUN for file in $(find . -name "*.?sproj"); do mkdir -p $(dirname $file)/$(basen COPY build/scripts/scripts.fsproj ./build/scripts/ COPY .ci/Jenkins.csproj ./.ci/ -RUN ls -al -RUN ls -al tests -RUN ls -al src -RUN ls -al src/Elasticsearch.Net - +# Install app dependencies RUN dotnet restore -# Install app dependencies +# copy relevant files (see .dockerignore) +COPY . . + +# making sure enough git info is available inside the container +RUN git rev-parse HEAD . -COPY . . \ No newline at end of file diff --git a/.ci/Jenkins.csproj b/.ci/Jenkins.csproj index 17ca88b06a7..a6b212d6faa 100644 --- a/.ci/Jenkins.csproj +++ b/.ci/Jenkins.csproj @@ -4,4 +4,10 @@ netcoreapp2.2 + + + .dockerignore + + + diff --git a/.ci/make.sh b/.ci/make.sh new file mode 100755 index 00000000000..aaa0395b3a4 --- /dev/null +++ b/.ci/make.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# parameters are available to this script + + +# common build entry script for all elasticsearch clients + +# ./.ci/make.sh bump VERSION +# ./.ci/make.sh release VERSION + +script_path=$(dirname "$(realpath -s "$0")") +repo=$(realpath "$script_path/../") + +# shellcheck disable=SC1090 +cmd=$1 +VERSION=$2 +STACK_VERSION=$VERSION +source "$script_path/functions/imports.sh" +set -euo pipefail + +output_folder=".ci/output" +OUTPUT_DIR="$repo/${output_folder}" + +DOTNET_VERSION=${DOTNET_VERSION-3.0.100} + +echo -e "\033[34;1mINFO:\033[0m VERSION ${STACK_VERSION}\033[0m" +echo -e "\033[34;1mINFO:\033[0m OUTPUT_DIR ${OUTPUT_DIR}\033[0m" +echo -e "\033[34;1mINFO:\033[0m DOTNET_VERSION ${DOTNET_VERSION}\033[0m" + +echo -e "\033[1m>>>>> Build [elastic/elasticsearch-net container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" + +docker build --file .ci/DockerFile --tag elastic/elasticsearch-net . + +echo -e "\033[1m>>>>> Run [elastic/elasticsearch-net container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" + +mkdir -p "$OUTPUT_DIR" + +docker run \ + --network=${network_name} \ + --env "DOTNET_VERSION" \ + --name test-runner \ + --volume "${OUTPUT_DIR}:/sln/${output_folder}" \ + --rm \ + elastic/elasticsearch-net \ + ./build.sh release "$VERSION" "$output_folder" "skiptests" diff --git a/.dockerignore b/.dockerignore index d11d281070c..68c27f47b1c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,8 +1,4 @@ scripts/output -.git - - -.git **/Obj/ **/obj/ @@ -35,8 +31,6 @@ PublishProfiles/ *.vspx /.symbols nuget.exe -*net45.csproj -*k10.csproj App_Data/ bower_components node_modules @@ -48,3 +42,5 @@ node_modules launchSettings.json bin obj + +.git/objects/* diff --git a/.gitignore b/.gitignore index 531a83d3858..172e1dcc869 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,8 @@ build/tools/* !build/*.targets !build/scripts +.ci/output + /dep/Newtonsoft.Json.4.0.2 !docs/build diff --git a/build/scripts/Commandline.fs b/build/scripts/Commandline.fs index 7cf6513d0ec..3ca58b9a90f 100644 --- a/build/scripts/Commandline.fs +++ b/build/scripts/Commandline.fs @@ -7,6 +7,7 @@ namespace Scripts open System open System.Runtime.InteropServices open Fake.Core +open Fake.IO //this is ugly but a direct port of what used to be duplicated in our DOS and bash scripts module Commandline = @@ -80,7 +81,7 @@ Execution hints can be provided anywhere on the command line type MultiTarget = All | One - type VersionArguments = { Version: string; } + type VersionArguments = { Version: string; OutputLocation: string option } type TestArguments = { TrxExport: bool; CodeCoverage: bool; TestFilter: string option; } type IntegrationArguments = { TrxExport: bool; TestFilter: string option; ClusterFilter: string option; ElasticsearchVersions: string list; } @@ -197,7 +198,10 @@ Execution hints can be provided anywhere on the command line | ["profile"] -> parsed | "rest-spec-tests" :: tail -> { parsed with RemainingArguments = tail } - | ["release"; version] -> { parsed with CommandArguments = SetVersion { Version = version }; } + | ["release"; version] -> { parsed with CommandArguments = SetVersion { Version = version; OutputLocation = None }; } + | ["release"; version; path] -> + if (not <| System.IO.Directory.Exists path) then failwithf "'%s' is not an existing directory" (Path.getFullName path) + { parsed with CommandArguments = SetVersion { Version = version; OutputLocation = Some path }; } | ["canary"] -> { parsed with CommandArguments = Test { diff --git a/build/scripts/ReposTooling.fs b/build/scripts/ReposTooling.fs index de4d359c65c..6638d0cb485 100644 --- a/build/scripts/ReposTooling.fs +++ b/build/scripts/ReposTooling.fs @@ -57,4 +57,9 @@ module ReposTooling = let Rewriter args = restoreOnce.Force() Tooling.DotNet.ExecIn "." (List.append [assemblyRewriter] (List.ofSeq args)) |> ignore + + let private packageValidator = "nupkg-validator" + let PackageValidator args = + restoreOnce.Force() + Tooling.DotNet.ExecIn "." (List.append [packageValidator] (List.ofSeq args)) |> ignore diff --git a/build/scripts/Targets.fs b/build/scripts/Targets.fs index 29e9df9b5d0..40cab418d7d 100644 --- a/build/scripts/Targets.fs +++ b/build/scripts/Targets.fs @@ -10,6 +10,7 @@ open System.IO open Build open Commandline open Bullseye +open Octokit open ProcNet open Fake.Core @@ -90,7 +91,12 @@ module Main = //RELEASE command "release" releaseChain <| fun _ -> - printfn "Finished Release Build %O" artifactsVersion + let outputPath = match parsed.CommandArguments with | SetVersion c -> c.OutputLocation | _ -> None + match outputPath with + | None -> printfn "Finished Release Build %O, artifacts available at: %s" artifactsVersion Paths.BuildOutput + | Some path -> + Fake.IO.Shell.cp_r Paths.BuildOutput path + printfn "Finished Release Build %O, output copied to: %s" artifactsVersion path conditional "test-nuget-package" (not parsed.SkipTests && Environment.isWindows) <| fun _ -> // run release unit tests puts packages in the system cache prevent this from happening locally diff --git a/build/scripts/Versioning.fs b/build/scripts/Versioning.fs index bfc81296aa6..3e7919641dc 100644 --- a/build/scripts/Versioning.fs +++ b/build/scripts/Versioning.fs @@ -12,6 +12,7 @@ open Commandline open Fake.Core open Fake.IO open Fake.IO.Globbing.Operators +open Fake.Tools.Git open Newtonsoft.Json module Versioning = @@ -112,60 +113,7 @@ module Versioning = | NoChange n -> n | Update (newVersion, _) -> newVersion - let private sn () = - match notWindows with - | true -> "sn" - | false -> - let programFiles = Environment.environVar "PROGRAMFILES(X86)" - if not (Directory.Exists programFiles) then failwith "Can not locate 64 bit program files" - let windowsSdks = ["v10.0A"; "v8.1A"; "v8.1"; "v8.0"; "v7.0A";] - let dotNetVersion = ["4.7.2"; "4.7.1"; "4.7"; "4.6.2"; "4.6.1"; "4.0"] - let combinations = List.allPairs windowsSdks dotNetVersion - let winFolder w = Path.Combine(programFiles, "Microsoft SDKs", "Windows", w, "bin") - let sdkFolder w d = - let folder = sprintf "NETFX %s Tools" d - Path.Combine(winFolder w, folder) - let snExe w d = Path.Combine(sdkFolder w d, "sn.exe") - let sn = combinations |> List.map (fun (w, d) -> snExe w d) |> List.tryFind File.exists - match sn with - | Some sn -> sn - | None -> failwithf "Could not locate sn.exe" - let private officialToken = "96c599bbe3e70f5d" - let private keyFile = Paths.Keys "keypair.snk" - - let private validate dll name = - let sn = sn () - let out = Tooling.readQuiet sn ["-v"; dll;] - - // Mono StrongName - version 5.18.1.0 - // returns `is strongnamed` - let valid = (out.ExitCode, out.Output |> Seq.tryFindIndex(fun s -> s.Line.Contains("is valid") || s.Line.Contains("is strongnamed"))) - match valid with - | (0, Some i) when i >= 0 -> printfn "%s is strongnamed" name - | (_, _) -> failwithf "%s is NOT strongnamed" dll - - let out = Tooling.readQuiet sn ["-T"; dll;] - - let tokenMessage = (out.Output |> Seq.tryFind(fun s -> s.Line.Contains("Public key token", StringComparison.OrdinalIgnoreCase))); - - // Mono StrongName - version 5.18.1.0 - // returns `Key Token:` - let token = - match tokenMessage with - | Some s -> Some <| (s.Line.Replace("Public Key Token:", "").Replace("Public key token is", "")).Trim() - | None -> None - - let valid = (out.ExitCode, token) - match valid with - | (0, Some t) when t = officialToken -> printfn "%s was signed with official key token %s" name t - | (_, Some t) -> printfn "%s was not signed with the official token: %s but %s" name officialToken t - | (_, None) -> printfn "%s was not signed at all" name - - let private validateDllStrongName dll name = - match File.Exists dll with - | true -> validate dll name - | _ -> failwithf "Attempted to verify signature of %s but it was not found!" dll let BuiltArtifacts (version: AnchoredVersion) = let packages = @@ -179,7 +127,6 @@ module Versioning = packages let ValidateArtifacts version = - let fileVersion = version.AssemblyFile let tmp = "build/output/_packages/tmp" let packages = BuiltArtifacts version @@ -187,6 +134,9 @@ module Versioning = packages |> Seq.iter(fun p -> + let v = sprintf "%O+%s" version.Full (Information.getCurrentSHA1(".")) + ReposTooling.PackageValidator [p.Package; "-v"; v; "-a"; p.AssemblyName; "-k"; officialToken] |> ignore + Zip.unzip tmp p.Package let nugetId = p.NugetId @@ -200,22 +150,9 @@ module Versioning = let command = [ sprintf "previous-nuget|%s|%s|%s" nugetId (version.Full.ToString()) tfm; sprintf "directory|%s" fullPath - "-a"; "-f"; "github-comment";] + "-a"; "-f"; "github-comment"; "--output"; Paths.BuildOutput] ReposTooling.Differ command ) - - !! (sprintf "%s/**/*.dll" tmp) - |> Seq.iter(fun f -> - let fv = FileVersionInfo.GetVersionInfo(f) - let name = AssemblyName.GetAssemblyName(f) - let a = name.Version - printfn "Assembly: %A File: %s Product: %s => %s" a fv.FileVersion fv.ProductVersion f - if (a.Minor > 0 || a.Revision > 0 || a.Build > 0) then failwith (sprintf "%s assembly version is not sticky to its major component" f) - if (parse (fv.ProductVersion) <> version.Full) then - failwith (sprintf "Expected product info %s to match new version %O " fv.ProductVersion fileVersion) - - validateDllStrongName f f - ) Directory.delete tmp ) \ No newline at end of file diff --git a/build/scripts/scripts.fsproj b/build/scripts/scripts.fsproj index b5dd519e373..51b12bc5578 100644 --- a/build/scripts/scripts.fsproj +++ b/build/scripts/scripts.fsproj @@ -44,6 +44,7 @@ + diff --git a/dotnet-tools.json b/dotnet-tools.json index 675f50c9fe3..4d75caa706b 100644 --- a/dotnet-tools.json +++ b/dotnet-tools.json @@ -13,6 +13,12 @@ "commands": [ "assembly-differ" ] + }, + "nupkg-validator": { + "version": "0.3.1", + "commands": [ + "nupkg-validator" + ] } } } \ No newline at end of file From 5c2f735ed40735ec9122b19c260131d86c8613b8 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 21 Jul 2020 19:12:20 +0200 Subject: [PATCH 2/3] disable release mode validation on appveyor --- build/scripts/Versioning.fs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build/scripts/Versioning.fs b/build/scripts/Versioning.fs index 3e7919641dc..2d62494457e 100644 --- a/build/scripts/Versioning.fs +++ b/build/scripts/Versioning.fs @@ -135,7 +135,11 @@ module Versioning = packages |> Seq.iter(fun p -> let v = sprintf "%O+%s" version.Full (Information.getCurrentSHA1(".")) - ReposTooling.PackageValidator [p.Package; "-v"; v; "-a"; p.AssemblyName; "-k"; officialToken] |> ignore + // loading dlls is locked down on APPVEYOR so we can not assert release mode + let appVeyorArgs = if Environment.hasEnvironVar "APPVEYOR" then ["-r"; "true"] else [] + ReposTooling.PackageValidator + <| [p.Package; "-v"; v; "-a"; p.AssemblyName; "-k"; officialToken] @ appVeyorArgs + |> ignore Zip.unzip tmp p.Package let nugetId = p.NugetId From b92f60ef470e22c3c65792d4e48b23185c883f0b Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 22 Jul 2020 09:26:35 +0200 Subject: [PATCH 3/3] disable release mode validaton on azure devops and validation for versioned packages --- build/scripts/Versioning.fs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/build/scripts/Versioning.fs b/build/scripts/Versioning.fs index 2d62494457e..514ea230148 100644 --- a/build/scripts/Versioning.fs +++ b/build/scripts/Versioning.fs @@ -133,12 +133,17 @@ module Versioning = printf "%O" packages packages + // do not validate versioned packages for now + |> Seq.filter(fun f -> not <| f.NugetId.EndsWith(sprintf ".v%i" version.Assembly.Major)) |> Seq.iter(fun p -> let v = sprintf "%O+%s" version.Full (Information.getCurrentSHA1(".")) // loading dlls is locked down on APPVEYOR so we can not assert release mode - let appVeyorArgs = if Environment.hasEnvironVar "APPVEYOR" then ["-r"; "true"] else [] + let ciArgs = + let appVeyor = Environment.hasEnvironVar "APPVEYOR" + let azDevops = Environment.hasEnvironVar "TF_BUILD" + if appVeyor || azDevops then ["-r"; "true"] else [] ReposTooling.PackageValidator - <| [p.Package; "-v"; v; "-a"; p.AssemblyName; "-k"; officialToken] @ appVeyorArgs + <| [p.Package; "-v"; v; "-a"; p.AssemblyName; "-k"; officialToken] @ ciArgs |> ignore Zip.unzip tmp p.Package