From d294f97ec06e4128d1c9ed4a48cc5f62d1ef51ff Mon Sep 17 00:00:00 2001 From: Claudia Beresford Date: Fri, 15 Mar 2019 15:26:57 +0200 Subject: [PATCH] Move everything that needs GOPATH to a new GOPATH Also build and move grootfs and tardis binaries to GOPATH/bin in the test script instead of building them in guardian GQTs [#164336113] Co-authored-by: Danail Branekov Co-authored-by: Tom Godkin --- .envrc | 2 +- .gitignore | 1 + .gitmodules | 56 +++++++++---------- ci/helpers/gats-utils | 6 +- ci/scripts/containerd | 10 +++- ci/scripts/garden | 4 +- ci/scripts/gdn-linux | 4 +- ci/scripts/gdn-linux-standalone | 16 +++--- ci/scripts/guardian | 8 ++- ci/scripts/guardian-windows.ps1 | 8 ++- ci/scripts/non-nested-gats | 2 +- ci/scripts/performance-tests | 2 +- scripts/test | 4 +- src/build/grootfs.bash | 14 +++++ src/build/runc.bash | 12 ++-- src/code.cloudfoundry.org/guardian | 1 - src/gopath/bin/.gitkeep | 0 .../src}/code.cloudfoundry.org/archiver | 0 .../src}/code.cloudfoundry.org/garden | 0 .../garden-integration-tests | 0 .../src}/code.cloudfoundry.org/grootfs | 0 src/gopath/src/code.cloudfoundry.org/guardian | 1 + .../src}/code.cloudfoundry.org/lager | 0 src/{ => gopath/src}/github.com/bmizerany/pat | 0 .../src}/github.com/containerd/containerd | 0 .../src}/github.com/golang/protobuf | 0 src/{ => gopath/src}/github.com/onsi/ginkgo | 0 src/{ => gopath/src}/github.com/onsi/gomega | 0 .../src}/github.com/opencontainers/runc | 0 src/{ => gopath/src}/github.com/tedsuo/rata | 0 src/{ => gopath/src}/gopkg.in/yaml.v2 | 0 31 files changed, 89 insertions(+), 62 deletions(-) create mode 100644 src/build/grootfs.bash delete mode 160000 src/code.cloudfoundry.org/guardian create mode 100644 src/gopath/bin/.gitkeep rename src/{ => gopath/src}/code.cloudfoundry.org/archiver (100%) rename src/{ => gopath/src}/code.cloudfoundry.org/garden (100%) rename src/{ => gopath/src}/code.cloudfoundry.org/garden-integration-tests (100%) rename src/{ => gopath/src}/code.cloudfoundry.org/grootfs (100%) create mode 160000 src/gopath/src/code.cloudfoundry.org/guardian rename src/{ => gopath/src}/code.cloudfoundry.org/lager (100%) rename src/{ => gopath/src}/github.com/bmizerany/pat (100%) rename src/{ => gopath/src}/github.com/containerd/containerd (100%) rename src/{ => gopath/src}/github.com/golang/protobuf (100%) rename src/{ => gopath/src}/github.com/onsi/ginkgo (100%) rename src/{ => gopath/src}/github.com/onsi/gomega (100%) rename src/{ => gopath/src}/github.com/opencontainers/runc (100%) rename src/{ => gopath/src}/github.com/tedsuo/rata (100%) rename src/{ => gopath/src}/gopkg.in/yaml.v2 (100%) diff --git a/.envrc b/.envrc index f603dccd..7ecdaf04 100644 --- a/.envrc +++ b/.envrc @@ -1,4 +1,4 @@ -export GOPATH=$PWD +export GOPATH=$PWD/src/gopath export PATH=$PWD/bin:$PATH # Tests log level diff --git a/.gitignore b/.gitignore index 6ca26668..15dbca94 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,4 @@ blobs .vagrant/ /pkg /bin +src/gopath/bin diff --git a/.gitmodules b/.gitmodules index 35b633c2..e5477578 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,57 +1,57 @@ -[submodule "src/code.cloudfoundry.org/guardian"] - path = src/code.cloudfoundry.org/guardian +[submodule "src/gopath/src/code.cloudfoundry.org/guardian"] + path = src/gopath/src/code.cloudfoundry.org/guardian url = https://github.com/cloudfoundry/guardian branch = master -[submodule "src/github.com/onsi/gomega"] - path = src/github.com/onsi/gomega +[submodule "src/gopath/src/github.com/onsi/gomega"] + path = src/gopath/src/github.com/onsi/gomega url = https://github.com/onsi/gomega branch = master -[submodule "src/github.com/tedsuo/rata"] - path = src/github.com/tedsuo/rata +[submodule "src/gopath/src/github.com/tedsuo/rata"] + path = src/gopath/src/github.com/tedsuo/rata url = https://github.com/tedsuo/rata branch = master -[submodule "src/code.cloudfoundry.org/garden"] - path = src/code.cloudfoundry.org/garden +[submodule "src/gopath/src/code.cloudfoundry.org/garden"] + path = src/gopath/src/code.cloudfoundry.org/garden url = https://github.com/cloudfoundry/garden branch = master -[submodule "src/github.com/onsi/ginkgo"] - path = src/github.com/onsi/ginkgo +[submodule "src/gopath/src/github.com/onsi/ginkgo"] + path = src/gopath/src/github.com/onsi/ginkgo url = https://github.com/onsi/ginkgo branch = master -[submodule "src/github.com/bmizerany/pat"] - path = src/github.com/bmizerany/pat +[submodule "src/gopath/src/github.com/bmizerany/pat"] + path = src/gopath/src/github.com/bmizerany/pat url = https://github.com/bmizerany/pat branch = master -[submodule "src/gopkg.in/yaml.v2"] - path = src/gopkg.in/yaml.v2 +[submodule "src/gopath/src/gopkg.in/yaml.v2"] + path = src/gopath/src/gopkg.in/yaml.v2 url = https://gopkg.in/yaml.v2 branch = v2 -[submodule "src/code.cloudfoundry.org/lager"] - path = src/code.cloudfoundry.org/lager +[submodule "src/gopath/src/code.cloudfoundry.org/lager"] + path = src/gopath/src/code.cloudfoundry.org/lager url = https://github.com/cloudfoundry/lager branch = master -[submodule "src/code.cloudfoundry.org/garden-integration-tests"] - path = src/code.cloudfoundry.org/garden-integration-tests +[submodule "src/gopath/src/code.cloudfoundry.org/garden-integration-tests"] + path = src/gopath/src/code.cloudfoundry.org/garden-integration-tests url = https://github.com/cloudfoundry/garden-integration-tests branch = master -[submodule "src/github.com/golang/protobuf"] - path = src/github.com/golang/protobuf +[submodule "src/gopath/src/github.com/golang/protobuf"] + path = src/gopath/src/github.com/golang/protobuf url = https://github.com/golang/protobuf branch = master -[submodule "src/code.cloudfoundry.org/grootfs"] - path = src/code.cloudfoundry.org/grootfs +[submodule "src/gopath/src/code.cloudfoundry.org/grootfs"] + path = src/gopath/src/code.cloudfoundry.org/grootfs url = https://github.com/cloudfoundry/grootfs branch = master -[submodule "src/github.com/containerd/containerd"] - path = src/github.com/containerd/containerd +[submodule "src/gopath/src/github.com/containerd/containerd"] + path = src/gopath/src/github.com/containerd/containerd url = https://github.com/containerd/containerd branch = master -[submodule "src/code.cloudfoundry.org/archiver"] - path = src/code.cloudfoundry.org/archiver +[submodule "src/gopath/src/code.cloudfoundry.org/archiver"] + path = src/gopath/src/code.cloudfoundry.org/archiver url = https://github.com/cloudfoundry/archiver branch = master -[submodule "src/github.com/opencontainers/runc"] - path = src/github.com/opencontainers/runc +[submodule "src/gopath/src/github.com/opencontainers/runc"] + path = src/gopath/src/github.com/opencontainers/runc url = https://github.com/opencontainers/runc branch = master [submodule "src/idmapper"] diff --git a/ci/helpers/gats-utils b/ci/helpers/gats-utils index 79b9984b..808c1eb9 100644 --- a/ci/helpers/gats-utils +++ b/ci/helpers/gats-utils @@ -2,7 +2,7 @@ build_nstar() { local grreleasedevelopdir="$1" - pushd "$grreleasedevelopdir"/src/code.cloudfoundry.org/guardian/rundmc/nstar + pushd "$grreleasedevelopdir"/src/gopath/src/code.cloudfoundry.org/guardian/rundmc/nstar make mv nstar $GOPATH/bin/ popd @@ -13,7 +13,7 @@ build_grootfs() { local target="${GOPATH}/bin" ( - cd "$grreleasedevelopdir"/src/code.cloudfoundry.org/grootfs + cd "$grreleasedevelopdir"/src/gopath/src/code.cloudfoundry.org/grootfs unset GOPATH @@ -27,7 +27,7 @@ build_gdn() { local target="${GOPATH}/bin" ( - cd "${grreleasedevelop}/src/code.cloudfoundry.org/guardian" + cd "${grreleasedevelop}/src/gopath/src/code.cloudfoundry.org/guardian" unset GOPATH diff --git a/ci/scripts/containerd b/ci/scripts/containerd index 64786259..148483e4 100755 --- a/ci/scripts/containerd +++ b/ci/scripts/containerd @@ -8,18 +8,22 @@ source "$grr_base/ci/helpers/device-control" # shellcheck source=../../src/build/runc.bash source "$grr_base/src/build/runc.bash" +# shellcheck source=../../src/build/grootfs.bash +source "$grr_base/src/build/grootfs.bash" export GOROOT=/usr/local/go export PATH=$GOROOT/bin:$PATH -export GOPATH=$grr_base +export GOPATH="$grr_base/src/gopath" export PATH=$HOME/go/bin:$GOPATH/bin:$PATH +mkdir -p "${GOPATH}/bin" build_runc "$grr_base" "${GOPATH}/bin" +build_grootfs "$grr_base" "${GOPATH}/bin" #TODO: Figure out how to get rid of the containerd git-related errors below echo -e "=========================================\n\n\n\nDuring containerd compilation you will get git fatals that complain of containerd directory not being a git repository. You can safely ignore them :)\n\n\n\n=========================================" -pushd "$grr_base"/src/github.com/containerd/containerd +pushd "$GOPATH"/src/github.com/containerd/containerd BUILDTAGS=no_btrfs make && make install popd @@ -42,7 +46,7 @@ go version echo "compiling test process: $(date)" -cd "$grr_base"/src/code.cloudfoundry.org/guardian +cd "$grr_base"/src/gopath/src/code.cloudfoundry.org/guardian if [ -z "${NO_GO_VET:-}" ]; then go vet ./... diff --git a/ci/scripts/garden b/ci/scripts/garden index d5cfb26d..8305b66b 100755 --- a/ci/scripts/garden +++ b/ci/scripts/garden @@ -6,10 +6,10 @@ cd "$( dirname "$0" )/../.." export GOROOT=/usr/local/go export PATH=$GOROOT/bin:$PATH -export GOPATH=$PWD +export GOPATH=$PWD/src/gopath export PATH=$GOPATH/bin:$PATH -cd src/code.cloudfoundry.org/garden +cd src/gopath/src/code.cloudfoundry.org/garden go vet ./... ginkgo -r -p --race --keepGoing --nodes="${GINKGO_NODES:-8}" --failOnPending --randomizeSuites --randomizeAllSpecs "$@" diff --git a/ci/scripts/gdn-linux b/ci/scripts/gdn-linux index df2de4d4..7445d1a9 100755 --- a/ci/scripts/gdn-linux +++ b/ci/scripts/gdn-linux @@ -32,7 +32,7 @@ function prepare() { } function test() { - cd src/code.cloudfoundry.org/garden-integration-tests + cd src/gopath/src/code.cloudfoundry.org/garden-integration-tests cmd="ginkgo -p --nodes=${GINKGO_NODES:-8} --failOnPending --randomizeSuites --randomizeAllSpecs $*" if [ -n "${SLOW_MOBIUS_TIME_LIMIT_SECONDS:-}" ]; then @@ -44,7 +44,7 @@ function test() { cd "$( dirname "$0" )/../.." -export GOPATH=$PWD +export GOPATH=$PWD/src/gopath export PATH=$GOPATH/bin:$PATH export tmpdir=/tmp/dir diff --git a/ci/scripts/gdn-linux-standalone b/ci/scripts/gdn-linux-standalone index 729084b3..790ff0a7 100755 --- a/ci/scripts/gdn-linux-standalone +++ b/ci/scripts/gdn-linux-standalone @@ -9,14 +9,14 @@ set -ex # shellcheck source=../../src/build/runc.bash source "$( dirname "$0" )/../../src/build/runc.bash" -export GOPATH=$PWD/gr-release-develop +export GOPATH=$PWD/gr-release-develop/src/gopath export PATH=$PATH:$GOPATH/bin basedir=$PWD # If you want to fly execute this, you will have to pass DEV_MODE=true if [ -z "$DEV_MODE" ]; then - gdn_version=$(cd gr-release-develop/src/code.cloudfoundry.org/guardian && git rev-parse HEAD) + gdn_version=$(cd gr-release-develop/src/gopath/src/code.cloudfoundry.org/guardian && git rev-parse HEAD) gdn_build_ldflags="-X main.version=${gdn_version}" fi @@ -45,25 +45,25 @@ function build() { ( ( unset GOPATH - cd src/code.cloudfoundry.org/guardian + cd src/gopath/src/code.cloudfoundry.org/guardian go build -mod vendor -o "${LINUX_ASSETS}/bin/dadoo" ./cmd/dadoo ) build_runc "$PWD" "${LINUX_ASSETS}/bin" - pushd src/code.cloudfoundry.org/guardian/rundmc/nstar + pushd src/gopath/src/code.cloudfoundry.org/guardian/rundmc/nstar make mv nstar "${LINUX_ASSETS}/bin" popd - pushd src/code.cloudfoundry.org/guardian/cmd/init + pushd src/gopath/src/code.cloudfoundry.org/guardian/cmd/init gcc -static -o init init.c ignore_sigchild.c mv init "${LINUX_ASSETS}/bin" popd ( - cd src/code.cloudfoundry.org/grootfs + cd src/gopath/src/code.cloudfoundry.org/grootfs unset GOPATH @@ -73,10 +73,10 @@ function build() { ) popd - go-bindata -nomemcopy -pkg bindata -o gr-release-develop/src/code.cloudfoundry.org/guardian/bindata/bindata.go linux/... + go-bindata -nomemcopy -pkg bindata -o gr-release-develop/src/gopath/src/code.cloudfoundry.org/guardian/bindata/bindata.go linux/... ( - cd gr-release-develop/src/code.cloudfoundry.org/guardian + cd gr-release-develop/src/gopath/src/code.cloudfoundry.org/guardian unset GOPATH diff --git a/ci/scripts/guardian b/ci/scripts/guardian index 2d9030c9..132a4240 100755 --- a/ci/scripts/guardian +++ b/ci/scripts/guardian @@ -8,14 +8,18 @@ source "$grr_base/ci/helpers/device-control" # shellcheck source=../../src/build/runc.bash source "$grr_base/src/build/runc.bash" +# shellcheck source=../../src/build/grootfs.bash +source "$grr_base/src/build/grootfs.bash" export GOROOT=/usr/local/go export PATH=$GOROOT/bin:$PATH -export GOPATH=$grr_base +export GOPATH="${grr_base}/src/gopath" export PATH=$HOME/go/bin:$GOPATH/bin:$PATH +mkdir -p "${GOPATH}/bin" build_runc "$grr_base" "${GOPATH}/bin" +build_grootfs "$grr_base" "${GOPATH}/bin" if [[ "$CONTAINERD_ENABLED" == "true" ]] ; then #TODO: Figure out how to get rid of the containerd git-related errors below @@ -44,7 +48,7 @@ go version echo "compiling test process: $(date)" -cd "$grr_base"/src/code.cloudfoundry.org/guardian +cd "$grr_base"/src/gopath/src/code.cloudfoundry.org/guardian if [ -z "${NO_GO_VET:-}" ]; then go vet ./... diff --git a/ci/scripts/guardian-windows.ps1 b/ci/scripts/guardian-windows.ps1 index 4c0d15ad..78d787c8 100755 --- a/ci/scripts/guardian-windows.ps1 +++ b/ci/scripts/guardian-windows.ps1 @@ -3,16 +3,18 @@ trap { $host.SetShouldExit(1) } cd gr-release-develop -$env:GOPATH = $PWD +$env:PWD = (Get-Location) +$env:GOPATH = ${env:PWD} + "\src\gopath" $env:PATH = $env:GOPATH + "/bin;C:/go/bin;" + $env:PATH Write-Host "Installing Ginkgo" -go.exe install ./src/github.com/onsi/ginkgo/ginkgo +go.exe install ./src/gopath/src/github.com/onsi/ginkgo/ginkgo if ($LastExitCode -ne 0) { throw "Ginkgo installation process returned error code: $LastExitCode" } -cd src/code.cloudfoundry.org/guardian +$guardian_path = ${env:GOPATH} + "/src/code.cloudfoundry.org/guardian" +cd ${guardian_path} go version go vet ./... diff --git a/ci/scripts/non-nested-gats b/ci/scripts/non-nested-gats index a7ba011a..a8cf0280 100755 --- a/ci/scripts/non-nested-gats +++ b/ci/scripts/non-nested-gats @@ -5,7 +5,7 @@ base=$PWD cd "$( dirname "$0" )/../.." -export GOPATH=$PWD +export GOPATH=$PWD/src/gopath export PATH=$PWD/bin:$PATH cd src/code.cloudfoundry.org/garden-integration-tests diff --git a/ci/scripts/performance-tests b/ci/scripts/performance-tests index 87d7b8ad..c56e64a8 100755 --- a/ci/scripts/performance-tests +++ b/ci/scripts/performance-tests @@ -3,7 +3,7 @@ set -ex cd "$( dirname "$0" )/../.." -export GOPATH=$PWD +export GOPATH=$PWD/src/gopath export PATH=$PWD/bin:$PATH cd src/code.cloudfoundry.org/garden-integration-tests diff --git a/scripts/test b/scripts/test index 71a96ffd..9343931a 100755 --- a/scripts/test +++ b/scripts/test @@ -23,12 +23,12 @@ usage() { go_vet_and_fmt() { cd "$ROOT_DIR_PATH" - pushd src/code.cloudfoundry.org/guardian + pushd src/gopath/src/code.cloudfoundry.org/guardian GOOS=linux go vet ./... go fmt ./... popd - pushd src/code.cloudfoundry.org/garden + pushd src/gopath/src/code.cloudfoundry.org/garden GOOS=linux go vet ./... go fmt ./... popd diff --git a/src/build/grootfs.bash b/src/build/grootfs.bash new file mode 100644 index 00000000..6f24b574 --- /dev/null +++ b/src/build/grootfs.bash @@ -0,0 +1,14 @@ +function build_grootfs() { + local garden_runc_release_dir="$1" + local target="$2" + + ( + unset GOPATH + cd "$garden_runc_release_dir/src/gopath/src/code.cloudfoundry.org/grootfs" || exit + make all + + chmod 04755 build/tardis + + mv build/{grootfs,tardis} "$target" + ) +} diff --git a/src/build/runc.bash b/src/build/runc.bash index f625ede1..d8def781 100644 --- a/src/build/runc.bash +++ b/src/build/runc.bash @@ -2,12 +2,14 @@ source "$(dirname "${BASH_SOURCE[0]}")"/../patches/patch.bash function build_runc() { - garden_runc_release_dir="$1" - dest_dir="$2" + local garden_runc_release_dir="$1" + local target="$2" + + ( + cd "$garden_runc_release_dir/src/gopath/src/github.com/opencontainers/runc" || exit - pushd "$garden_runc_release_dir/src/github.com/opencontainers/runc" apply_patches "$garden_runc_release_dir/src/patches/runc" make BUILDTAGS='seccomp apparmor' static - mv runc "$dest_dir" - popd + mv runc "$target" + ) } diff --git a/src/code.cloudfoundry.org/guardian b/src/code.cloudfoundry.org/guardian deleted file mode 160000 index 22110f93..00000000 --- a/src/code.cloudfoundry.org/guardian +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 22110f9382a082e8443553c33cf84399cebfdb71 diff --git a/src/gopath/bin/.gitkeep b/src/gopath/bin/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/src/code.cloudfoundry.org/archiver b/src/gopath/src/code.cloudfoundry.org/archiver similarity index 100% rename from src/code.cloudfoundry.org/archiver rename to src/gopath/src/code.cloudfoundry.org/archiver diff --git a/src/code.cloudfoundry.org/garden b/src/gopath/src/code.cloudfoundry.org/garden similarity index 100% rename from src/code.cloudfoundry.org/garden rename to src/gopath/src/code.cloudfoundry.org/garden diff --git a/src/code.cloudfoundry.org/garden-integration-tests b/src/gopath/src/code.cloudfoundry.org/garden-integration-tests similarity index 100% rename from src/code.cloudfoundry.org/garden-integration-tests rename to src/gopath/src/code.cloudfoundry.org/garden-integration-tests diff --git a/src/code.cloudfoundry.org/grootfs b/src/gopath/src/code.cloudfoundry.org/grootfs similarity index 100% rename from src/code.cloudfoundry.org/grootfs rename to src/gopath/src/code.cloudfoundry.org/grootfs diff --git a/src/gopath/src/code.cloudfoundry.org/guardian b/src/gopath/src/code.cloudfoundry.org/guardian new file mode 160000 index 00000000..5708a933 --- /dev/null +++ b/src/gopath/src/code.cloudfoundry.org/guardian @@ -0,0 +1 @@ +Subproject commit 5708a9330d630cbb73b386d9b46eba6fc234e1eb diff --git a/src/code.cloudfoundry.org/lager b/src/gopath/src/code.cloudfoundry.org/lager similarity index 100% rename from src/code.cloudfoundry.org/lager rename to src/gopath/src/code.cloudfoundry.org/lager diff --git a/src/github.com/bmizerany/pat b/src/gopath/src/github.com/bmizerany/pat similarity index 100% rename from src/github.com/bmizerany/pat rename to src/gopath/src/github.com/bmizerany/pat diff --git a/src/github.com/containerd/containerd b/src/gopath/src/github.com/containerd/containerd similarity index 100% rename from src/github.com/containerd/containerd rename to src/gopath/src/github.com/containerd/containerd diff --git a/src/github.com/golang/protobuf b/src/gopath/src/github.com/golang/protobuf similarity index 100% rename from src/github.com/golang/protobuf rename to src/gopath/src/github.com/golang/protobuf diff --git a/src/github.com/onsi/ginkgo b/src/gopath/src/github.com/onsi/ginkgo similarity index 100% rename from src/github.com/onsi/ginkgo rename to src/gopath/src/github.com/onsi/ginkgo diff --git a/src/github.com/onsi/gomega b/src/gopath/src/github.com/onsi/gomega similarity index 100% rename from src/github.com/onsi/gomega rename to src/gopath/src/github.com/onsi/gomega diff --git a/src/github.com/opencontainers/runc b/src/gopath/src/github.com/opencontainers/runc similarity index 100% rename from src/github.com/opencontainers/runc rename to src/gopath/src/github.com/opencontainers/runc diff --git a/src/github.com/tedsuo/rata b/src/gopath/src/github.com/tedsuo/rata similarity index 100% rename from src/github.com/tedsuo/rata rename to src/gopath/src/github.com/tedsuo/rata diff --git a/src/gopkg.in/yaml.v2 b/src/gopath/src/gopkg.in/yaml.v2 similarity index 100% rename from src/gopkg.in/yaml.v2 rename to src/gopath/src/gopkg.in/yaml.v2