From 65663b0594ad2311471bb473bf2541a3eb7704e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Niewrza=C5=82?= Date: Mon, 15 Apr 2024 14:27:19 +0200 Subject: [PATCH] Jenkinsfile.public: align more with verify/premerge builds (#6905) --- Jenkinsfile.public | 228 +++++++++++++++++++++------------------------ 1 file changed, 108 insertions(+), 120 deletions(-) diff --git a/Jenkinsfile.public b/Jenkinsfile.public index 2f93c711577c..db8b6c3cc47e 100644 --- a/Jenkinsfile.public +++ b/Jenkinsfile.public @@ -4,7 +4,7 @@ pipeline { label 'main' image 'storjlabs/ci:latest' alwaysPull true - args '-u root:root --cap-add SYS_PTRACE -v "/tmp/gomod":/go/pkg/mod -v "/tmp/npm":/npm --tmpfs "/tmp:exec,mode=777"' + args '-u root:root --cap-add SYS_PTRACE -v "/tmp/gomod":/go/pkg/mod -v /tmp/golangci-lint:/root/.cache/golangci-lint -v /tmp/gocache:/root/.cache/go-build -v "/tmp/npm":/npm --tmpfs "/tmp:exec,mode=777"' } } options { @@ -73,51 +73,21 @@ pipeline { stage('Build') { parallel { - stage('go') { - steps { - // use go test to build all the packages, including tests - sh 'go test -v -p 16 -bench XYZXYZXYZXYZ -run XYZXYZXYZXYZ ./...' - } - } - stage('go -race') { - steps { - // use go test to build all the packages, including tests - sh 'go test -v -p 16 -bench XYZXYZXYZXYZ -run XYZXYZXYZXYZ -race ./...' - - // install storj-sim - sh 'go install -race -v storj.io/storj/cmd/satellite '+ - 'storj.io/storj/cmd/storagenode ' + - 'storj.io/storj/cmd/storj-sim ' + - 'storj.io/storj/cmd/versioncontrol ' + - 'storj.io/storj/cmd/uplink ' + - 'storj.io/storj/cmd/identity ' + - 'storj.io/storj/cmd/certificates ' + - 'storj.io/storj/cmd/multinode' - } - } - stage('go -race gateway') { - steps { - // install gateway for storj-sim - sh 'go install -race -v storj.io/gateway@latest' - } - } - stage('db') { steps { sh 'service postgresql start' dir('.build') { - sh 'cockroach start-single-node --insecure --store=type=mem,size=2GiB --listen-addr=localhost:26256 --http-addr=localhost:8086 --cache 512MiB --max-sql-memory 512MiB --background' - sh 'cockroach start-single-node --insecure --store=type=mem,size=2GiB --listen-addr=localhost:26257 --http-addr=localhost:8087 --cache 512MiB --max-sql-memory 512MiB --background' - sh 'cockroach start-single-node --insecure --store=type=mem,size=2GiB --listen-addr=localhost:26258 --http-addr=localhost:8088 --cache 512MiB --max-sql-memory 512MiB --background' - sh 'cockroach start-single-node --insecure --store=type=mem,size=2GiB --listen-addr=localhost:26259 --http-addr=localhost:8089 --cache 512MiB --max-sql-memory 512MiB --background' - sh 'cockroach start-single-node --insecure --store=type=mem,size=2GiB --listen-addr=localhost:26260 --http-addr=localhost:8090 --cache 256MiB --max-sql-memory 256MiB --background' - } + sh 'cockroach start-single-node --insecure --store=type=mem,size=3GiB --listen-addr=localhost:26256 --http-addr=localhost:8086 --cache 1024MiB --max-sql-memory 1024MiB --background' + sh 'cockroach start-single-node --insecure --store=type=mem,size=3GiB --listen-addr=localhost:26257 --http-addr=localhost:8087 --cache 1024MiB --max-sql-memory 1024MiB --background' + sh 'cockroach start-single-node --insecure --store=type=mem,size=3GiB --listen-addr=localhost:26258 --http-addr=localhost:8088 --cache 1024MiB --max-sql-memory 1024MiB --background' } } } } } - stage('Lint') { + stage('Lint') { + parallel { + stage('Storj Checks') { steps { sh 'check-mod-tidy' sh 'check-copyright' @@ -126,90 +96,101 @@ pipeline { sh 'check-atomic-align ./...' sh 'check-monkit ./...' sh 'check-errs ./...' - sh 'staticcheck ./...' - sh 'golangci-lint --config /go/ci/.golangci.yml -j=2 run' + sh 'check-deferloop ./...' sh 'check-downgrades' sh 'make check-monitoring' sh 'make test-wasm-size' - - sh 'protolock status' sh './scripts/check-package-lock.sh' - - // go-licenses by default has AGPL3 in the forbidden list, hence we need to explicitly allow `storj.io/storj`. - sh 'go-licenses check --ignore "storj.io/storj" ./...' } } - stage('Cross Compile') { + stage('Staticcheck') { steps { - // verify most of the commands, we cannot check everything since some of them - // have a C dependency and we don't have cross-compilation in storj/ci image - sh 'check-cross-compile storj.io/storj/cmd/uplink storj.io/storj/cmd/satellite storj.io/storj/cmd/storagenode-updater storj.io/storj/cmd/storj-sim' + sh 'staticcheck ./...' } } - stage('Tests') { - environment { - STORJ_TEST_HOST = '127.0.0.20;127.0.0.21;127.0.0.22;127.0.0.23;127.0.0.24;127.0.0.25' - STORJ_TEST_COCKROACH = 'cockroach://root@localhost:26256/testcockroach?sslmode=disable;' + - 'cockroach://root@localhost:26257/testcockroach?sslmode=disable;' + - 'cockroach://root@localhost:26258/testcockroach?sslmode=disable;' + - 'cockroach://root@localhost:26259/testcockroach?sslmode=disable' - STORJ_TEST_COCKROACH_NODROP = 'true' - STORJ_TEST_COCKROACH_ALT = 'cockroach://root@localhost:26260/testcockroach?sslmode=disable' - STORJ_TEST_POSTGRES = 'postgres://postgres@localhost/teststorj?sslmode=disable' - STORJ_TEST_LOG_LEVEL = 'info' - COVERFLAGS = "${ env.BRANCH_NAME == 'main' ? '-coverprofile=.build/coverprofile -coverpkg=storj.io/storj/private/...,storj.io/storj/satellite/...,storj.io/storj/storagenode/...,storj.io/storj/versioncontrol/...' : ''}" - GOEXPERIMENT = 'nocoverageredesign' + stage('Other') { + steps { + sh 'golangci-lint --config /go/ci/.golangci.yml -j=2 run' + sh 'protolock status' + sh 'go.min vet ./...' + dir("testsuite/storjscan") { + sh 'go vet ./...' + } } + } + stage('licenses check') { steps { - sh 'cockroach sql --insecure --host=localhost:26256 -e \'create database testcockroach;\'' - sh 'cockroach sql --insecure --host=localhost:26257 -e \'create database testcockroach;\'' - sh 'cockroach sql --insecure --host=localhost:26258 -e \'create database testcockroach;\'' - sh 'cockroach sql --insecure --host=localhost:26259 -e \'create database testcockroach;\'' - sh 'cockroach sql --insecure --host=localhost:26260 -e \'create database testcockroach;\'' + // go-licenses by default has AGPL3 in the forbidden list, hence we need to explicitly allow `storj.io/storj`. + sh 'go-licenses check --ignore "storj.io/storj" ./...' + } + } + } + } - sh 'psql -U postgres -c \'create database teststorj;\'' + stage('Cross Compile') { + steps { + // verify most of the commands, we cannot check everything since some of them + // have a C dependency and we don't have cross-compilation in storj/ci image + sh 'check-cross-compile storj.io/storj/cmd/uplink storj.io/storj/cmd/satellite storj.io/storj/cmd/storagenode-updater storj.io/storj/cmd/storj-sim' + } + } - sh 'use-ports -from 1024 -to 10000 &' + stage('Tests') { + environment { + STORJ_TEST_HOST = '127.0.0.20;127.0.0.21;127.0.0.22;127.0.0.23;127.0.0.24;127.0.0.25' + STORJ_TEST_COCKROACH = 'cockroach://root@localhost:26256/testcockroach?sslmode=disable;' + + 'cockroach://root@localhost:26257/testcockroach?sslmode=disable' + STORJ_TEST_COCKROACH_NODROP = 'true' + STORJ_TEST_POSTGRES = 'omit' + STORJ_TEST_LOG_LEVEL = 'info' + COVERFLAGS = "${ env.BRANCH_NAME == 'main' ? '-coverprofile=.build/coverprofile -coverpkg=storj.io/storj/private/...,storj.io/storj/satellite/...,storj.io/storj/storagenode/...,storj.io/storj/versioncontrol/...' : ''}" + GOEXPERIMENT = 'nocoverageredesign' + } + steps { + sh 'cockroach sql --insecure --host=localhost:26256 -e \'create database testcockroach;\'' + sh 'cockroach sql --insecure --host=localhost:26257 -e \'create database testcockroach;\'' - sh 'go test -parallel 4 -p 6 -vet=off $COVERFLAGS -timeout 32m -json -race ./... 2>&1 | tee .build/tests.json | xunit -out .build/tests.xml' - } + sh 'use-ports -from 1024 -to 10000 &' - post { - always { - archiveArtifacts artifacts: '.build/tests.json' - sh script: 'cat .build/tests.json | tparse -all -slow 100', returnStatus: true - junit '.build/tests.xml' - - script { - if(fileExists(".build/coverprofile")){ - sh script: 'filter-cover-profile < .build/coverprofile > .build/clean.coverprofile', returnStatus: true - sh script: 'gocov convert .build/clean.coverprofile > .build/cover.json', returnStatus: true - sh script: 'gocov-xml < .build/cover.json > .build/cobertura.xml', returnStatus: true - sh script: 'gzip .build/clean.coverprofile -c > .build/clean.coverprofile.gz' - archiveArtifacts artifacts: '.build/clean.coverprofile.gz' - cobertura coberturaReportFile: '.build/cobertura.xml', - lineCoverageTargets: '70, 60, 50', - autoUpdateHealth: false, - autoUpdateStability: false, - failUnhealthy: true - } - } + sh 'go test -parallel 6 -p 12 -vet=off $COVERFLAGS -timeout 32m -json -race ./... 2>&1 | tee .build/tests.json | xunit -out .build/tests.xml' + } + + post { + always { + archiveArtifacts artifacts: '.build/tests.json' + sh script: 'cat .build/tests.json | tparse -all -slow 100', returnStatus: true + junit '.build/tests.xml' + + script { + if(fileExists(".build/coverprofile")){ + sh script: 'filter-cover-profile < .build/coverprofile > .build/clean.coverprofile', returnStatus: true + sh script: 'gocov convert .build/clean.coverprofile > .build/cover.json', returnStatus: true + sh script: 'gocov-xml < .build/cover.json > .build/cobertura.xml', returnStatus: true + cobertura coberturaReportFile: '.build/cobertura.xml', + lineCoverageTargets: '70, 60, 50', + autoUpdateHealth: false, + autoUpdateStability: false, + failUnhealthy: true } } } + } + } - stage('Check Benchmark') { - environment { - STORJ_TEST_COCKROACH = 'omit' - STORJ_TEST_POSTGRES = 'postgres://postgres@localhost/benchstorj?sslmode=disable' - } - steps { - sh 'psql -U postgres -c \'create database benchstorj;\'' - sh 'go test -parallel 1 -p 1 -vet=off -timeout 20m -short -run XYZXYZXYZXYZ -bench . -benchtime 1x ./...' - } - } + stage('Check Benchmark') { + environment { + STORJ_TEST_COCKROACH = 'omit' + STORJ_TEST_POSTGRES = 'postgres://postgres@localhost/benchstorj?sslmode=disable' + } + steps { + sh 'psql -U postgres -c \'create database benchstorj;\'' + sh 'go test -parallel 1 -p 1 -vet=off -timeout 20m -short -run XYZXYZXYZXYZ -bench . -benchtime 1x ./...' + } + } - stage('Integration') { + stage('Integration') { + parallel { + stage('Postgres') { environment { // use different hostname to avoid port conflicts STORJ_NETWORK_HOST4 = '127.0.0.2' @@ -226,7 +207,7 @@ pipeline { } } - stage('Cockroach Integration') { + stage('Cockroach') { environment { STORJ_NETWORK_HOST4 = '127.0.0.4' STORJ_NETWORK_HOST6 = '127.0.0.4' @@ -240,24 +221,12 @@ pipeline { sh 'cockroach sql --insecure --host=localhost:26257 -e \'drop database testcockroach4;\'' } } + } + } - stage('Integration Redis unavailability') { - environment { - // use different hostname to avoid port conflicts - STORJ_NETWORK_HOST4 = '127.0.0.6' - STORJ_NETWORK_HOST6 = '127.0.0.6' - STORJ_REDIS_PORT = '7379' - - STORJ_SIM_POSTGRES = 'postgres://postgres@localhost/teststorj6?sslmode=disable' - } - - steps { - sh 'psql -U postgres -c \'create database teststorj6;\'' - sh 'make test-sim-redis-unavailability' - } - } - - stage('Backwards Compatibility') { + stage('Backwards Compatibility') { + parallel { + stage('Postgres') { environment { STORJ_NETWORK_HOST4 = '127.0.0.3' STORJ_NETWORK_HOST6 = '127.0.0.3' @@ -272,7 +241,7 @@ pipeline { } } - stage('Cockroach Backwards Compatibility') { + stage('Cockroach') { environment { STORJ_NETWORK_HOST4 = '127.0.0.5' STORJ_NETWORK_HOST6 = '127.0.0.5' @@ -287,6 +256,25 @@ pipeline { sh 'cockroach sql --insecure --host=localhost:26257 -e \'drop database testcockroach5;\'' } } + } + } + + stage('Integration Redis unavailability') { + environment { + // use different hostname to avoid port conflicts + STORJ_NETWORK_HOST4 = '127.0.0.6' + STORJ_NETWORK_HOST6 = '127.0.0.6' + STORJ_REDIS_PORT = '7379' + + STORJ_SIM_POSTGRES = 'postgres://postgres@localhost/teststorj6?sslmode=disable' + } + + steps { + sh 'psql -U postgres -c \'create database teststorj6;\'' + sh 'make test-sim-redis-unavailability' + } + } + stage('Post') { parallel { stage('Lint') {