Skip to content

Commit

Permalink
Jenkinsfile.public: align more with verify/premerge builds (#6905)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mniewrzal
mniewrzal committed Apr 15, 2024
1 parent 21bd84e commit 65663b0
Showing 1 changed file with 108 additions and 120 deletions.
228 changes: 108 additions & 120 deletions Jenkinsfile.public
View file
Expand Up @@ -4,7 +4,7 @@ pipeline {
label 'main'
image 'storjlabs/ci:latest'
alwaysPull true
args '-u root:root --cap-add SYS_PTRACE -v "/tmp/gomod":/go/pkg/mod -v "/tmp/npm":/npm --tmpfs "/tmp:exec,mode=777"'
args '-u root:root --cap-add SYS_PTRACE -v "/tmp/gomod":/go/pkg/mod -v /tmp/golangci-lint:/root/.cache/golangci-lint -v /tmp/gocache:/root/.cache/go-build -v "/tmp/npm":/npm --tmpfs "/tmp:exec,mode=777"'
}
}
options {
Expand Down Expand Up @@ -73,51 +73,21 @@ pipeline {

stage('Build') {
parallel {
stage('go') {
steps {
// use go test to build all the packages, including tests
sh 'go test -v -p 16 -bench XYZXYZXYZXYZ -run XYZXYZXYZXYZ ./...'
}
}
stage('go -race') {
steps {
// use go test to build all the packages, including tests
sh 'go test -v -p 16 -bench XYZXYZXYZXYZ -run XYZXYZXYZXYZ -race ./...'

// install storj-sim
sh 'go install -race -v storj.io/storj/cmd/satellite '+
'storj.io/storj/cmd/storagenode ' +
'storj.io/storj/cmd/storj-sim ' +
'storj.io/storj/cmd/versioncontrol ' +
'storj.io/storj/cmd/uplink ' +
'storj.io/storj/cmd/identity ' +
'storj.io/storj/cmd/certificates ' +
'storj.io/storj/cmd/multinode'
}
}
stage('go -race gateway') {
steps {
// install gateway for storj-sim
sh 'go install -race -v storj.io/gateway@latest'
}
}

stage('db') {
steps {
sh 'service postgresql start'
dir('.build') {
sh 'cockroach start-single-node --insecure --store=type=mem,size=2GiB --listen-addr=localhost:26256 --http-addr=localhost:8086 --cache 512MiB --max-sql-memory 512MiB --background'
sh 'cockroach start-single-node --insecure --store=type=mem,size=2GiB --listen-addr=localhost:26257 --http-addr=localhost:8087 --cache 512MiB --max-sql-memory 512MiB --background'
sh 'cockroach start-single-node --insecure --store=type=mem,size=2GiB --listen-addr=localhost:26258 --http-addr=localhost:8088 --cache 512MiB --max-sql-memory 512MiB --background'
sh 'cockroach start-single-node --insecure --store=type=mem,size=2GiB --listen-addr=localhost:26259 --http-addr=localhost:8089 --cache 512MiB --max-sql-memory 512MiB --background'
sh 'cockroach start-single-node --insecure --store=type=mem,size=2GiB --listen-addr=localhost:26260 --http-addr=localhost:8090 --cache 256MiB --max-sql-memory 256MiB --background'
}
sh 'cockroach start-single-node --insecure --store=type=mem,size=3GiB --listen-addr=localhost:26256 --http-addr=localhost:8086 --cache 1024MiB --max-sql-memory 1024MiB --background'
sh 'cockroach start-single-node --insecure --store=type=mem,size=3GiB --listen-addr=localhost:26257 --http-addr=localhost:8087 --cache 1024MiB --max-sql-memory 1024MiB --background'
sh 'cockroach start-single-node --insecure --store=type=mem,size=3GiB --listen-addr=localhost:26258 --http-addr=localhost:8088 --cache 1024MiB --max-sql-memory 1024MiB --background' }
}
}
}
}

stage('Lint') {
stage('Lint') {
parallel {
stage('Storj Checks') {
steps {
sh 'check-mod-tidy'
sh 'check-copyright'
Expand All @@ -126,90 +96,101 @@ pipeline {
sh 'check-atomic-align ./...'
sh 'check-monkit ./...'
sh 'check-errs ./...'
sh 'staticcheck ./...'
sh 'golangci-lint --config /go/ci/.golangci.yml -j=2 run'
sh 'check-deferloop ./...'
sh 'check-downgrades'
sh 'make check-monitoring'
sh 'make test-wasm-size'

sh 'protolock status'
sh './scripts/check-package-lock.sh'

// go-licenses by default has AGPL3 in the forbidden list, hence we need to explicitly allow `storj.io/storj`.
sh 'go-licenses check --ignore "storj.io/storj" ./...'
}
}
stage('Cross Compile') {
stage('Staticcheck') {
steps {
// verify most of the commands, we cannot check everything since some of them
// have a C dependency and we don't have cross-compilation in storj/ci image
sh 'check-cross-compile storj.io/storj/cmd/uplink storj.io/storj/cmd/satellite storj.io/storj/cmd/storagenode-updater storj.io/storj/cmd/storj-sim'
sh 'staticcheck ./...'
}
}
stage('Tests') {
environment {
STORJ_TEST_HOST = '127.0.0.20;127.0.0.21;127.0.0.22;127.0.0.23;127.0.0.24;127.0.0.25'
STORJ_TEST_COCKROACH = 'cockroach://root@localhost:26256/testcockroach?sslmode=disable;' +
'cockroach://root@localhost:26257/testcockroach?sslmode=disable;' +
'cockroach://root@localhost:26258/testcockroach?sslmode=disable;' +
'cockroach://root@localhost:26259/testcockroach?sslmode=disable'
STORJ_TEST_COCKROACH_NODROP = 'true'
STORJ_TEST_COCKROACH_ALT = 'cockroach://root@localhost:26260/testcockroach?sslmode=disable'
STORJ_TEST_POSTGRES = 'postgres://postgres@localhost/teststorj?sslmode=disable'
STORJ_TEST_LOG_LEVEL = 'info'
COVERFLAGS = "${ env.BRANCH_NAME == 'main' ? '-coverprofile=.build/coverprofile -coverpkg=storj.io/storj/private/...,storj.io/storj/satellite/...,storj.io/storj/storagenode/...,storj.io/storj/versioncontrol/...' : ''}"
GOEXPERIMENT = 'nocoverageredesign'
stage('Other') {
steps {
sh 'golangci-lint --config /go/ci/.golangci.yml -j=2 run'
sh 'protolock status'
sh 'go.min vet ./...'
dir("testsuite/storjscan") {
sh 'go vet ./...'
}
}
}
stage('licenses check') {
steps {
sh 'cockroach sql --insecure --host=localhost:26256 -e \'create database testcockroach;\''
sh 'cockroach sql --insecure --host=localhost:26257 -e \'create database testcockroach;\''
sh 'cockroach sql --insecure --host=localhost:26258 -e \'create database testcockroach;\''
sh 'cockroach sql --insecure --host=localhost:26259 -e \'create database testcockroach;\''
sh 'cockroach sql --insecure --host=localhost:26260 -e \'create database testcockroach;\''
// go-licenses by default has AGPL3 in the forbidden list, hence we need to explicitly allow `storj.io/storj`.
sh 'go-licenses check --ignore "storj.io/storj" ./...'
}
}
}
}

sh 'psql -U postgres -c \'create database teststorj;\''
stage('Cross Compile') {
steps {
// verify most of the commands, we cannot check everything since some of them
// have a C dependency and we don't have cross-compilation in storj/ci image
sh 'check-cross-compile storj.io/storj/cmd/uplink storj.io/storj/cmd/satellite storj.io/storj/cmd/storagenode-updater storj.io/storj/cmd/storj-sim'
}
}

sh 'use-ports -from 1024 -to 10000 &'
stage('Tests') {
environment {
STORJ_TEST_HOST = '127.0.0.20;127.0.0.21;127.0.0.22;127.0.0.23;127.0.0.24;127.0.0.25'
STORJ_TEST_COCKROACH = 'cockroach://root@localhost:26256/testcockroach?sslmode=disable;' +
'cockroach://root@localhost:26257/testcockroach?sslmode=disable'
STORJ_TEST_COCKROACH_NODROP = 'true'
STORJ_TEST_POSTGRES = 'omit'
STORJ_TEST_LOG_LEVEL = 'info'
COVERFLAGS = "${ env.BRANCH_NAME == 'main' ? '-coverprofile=.build/coverprofile -coverpkg=storj.io/storj/private/...,storj.io/storj/satellite/...,storj.io/storj/storagenode/...,storj.io/storj/versioncontrol/...' : ''}"
GOEXPERIMENT = 'nocoverageredesign'
}
steps {
sh 'cockroach sql --insecure --host=localhost:26256 -e \'create database testcockroach;\''
sh 'cockroach sql --insecure --host=localhost:26257 -e \'create database testcockroach;\''

sh 'go test -parallel 4 -p 6 -vet=off $COVERFLAGS -timeout 32m -json -race ./... 2>&1 | tee .build/tests.json | xunit -out .build/tests.xml'
}
sh 'use-ports -from 1024 -to 10000 &'

post {
always {
archiveArtifacts artifacts: '.build/tests.json'
sh script: 'cat .build/tests.json | tparse -all -slow 100', returnStatus: true
junit '.build/tests.xml'

script {
if(fileExists(".build/coverprofile")){
sh script: 'filter-cover-profile < .build/coverprofile > .build/clean.coverprofile', returnStatus: true
sh script: 'gocov convert .build/clean.coverprofile > .build/cover.json', returnStatus: true
sh script: 'gocov-xml < .build/cover.json > .build/cobertura.xml', returnStatus: true
sh script: 'gzip .build/clean.coverprofile -c > .build/clean.coverprofile.gz'
archiveArtifacts artifacts: '.build/clean.coverprofile.gz'
cobertura coberturaReportFile: '.build/cobertura.xml',
lineCoverageTargets: '70, 60, 50',
autoUpdateHealth: false,
autoUpdateStability: false,
failUnhealthy: true
}
}
sh 'go test -parallel 6 -p 12 -vet=off $COVERFLAGS -timeout 32m -json -race ./... 2>&1 | tee .build/tests.json | xunit -out .build/tests.xml'
}

post {
always {
archiveArtifacts artifacts: '.build/tests.json'
sh script: 'cat .build/tests.json | tparse -all -slow 100', returnStatus: true
junit '.build/tests.xml'

script {
if(fileExists(".build/coverprofile")){
sh script: 'filter-cover-profile < .build/coverprofile > .build/clean.coverprofile', returnStatus: true
sh script: 'gocov convert .build/clean.coverprofile > .build/cover.json', returnStatus: true
sh script: 'gocov-xml < .build/cover.json > .build/cobertura.xml', returnStatus: true
cobertura coberturaReportFile: '.build/cobertura.xml',
lineCoverageTargets: '70, 60, 50',
autoUpdateHealth: false,
autoUpdateStability: false,
failUnhealthy: true
}
}
}
}
}

stage('Check Benchmark') {
environment {
STORJ_TEST_COCKROACH = 'omit'
STORJ_TEST_POSTGRES = 'postgres://postgres@localhost/benchstorj?sslmode=disable'
}
steps {
sh 'psql -U postgres -c \'create database benchstorj;\''
sh 'go test -parallel 1 -p 1 -vet=off -timeout 20m -short -run XYZXYZXYZXYZ -bench . -benchtime 1x ./...'
}
}
stage('Check Benchmark') {
environment {
STORJ_TEST_COCKROACH = 'omit'
STORJ_TEST_POSTGRES = 'postgres://postgres@localhost/benchstorj?sslmode=disable'
}
steps {
sh 'psql -U postgres -c \'create database benchstorj;\''
sh 'go test -parallel 1 -p 1 -vet=off -timeout 20m -short -run XYZXYZXYZXYZ -bench . -benchtime 1x ./...'
}
}

stage('Integration') {
stage('Integration') {
parallel {
stage('Postgres') {
environment {
// use different hostname to avoid port conflicts
STORJ_NETWORK_HOST4 = '127.0.0.2'
Expand All @@ -226,7 +207,7 @@ pipeline {
}
}

stage('Cockroach Integration') {
stage('Cockroach') {
environment {
STORJ_NETWORK_HOST4 = '127.0.0.4'
STORJ_NETWORK_HOST6 = '127.0.0.4'
Expand All @@ -240,24 +221,12 @@ pipeline {
sh 'cockroach sql --insecure --host=localhost:26257 -e \'drop database testcockroach4;\''
}
}
}
}

stage('Integration Redis unavailability') {
environment {
// use different hostname to avoid port conflicts
STORJ_NETWORK_HOST4 = '127.0.0.6'
STORJ_NETWORK_HOST6 = '127.0.0.6'
STORJ_REDIS_PORT = '7379'

STORJ_SIM_POSTGRES = 'postgres://postgres@localhost/teststorj6?sslmode=disable'
}

steps {
sh 'psql -U postgres -c \'create database teststorj6;\''
sh 'make test-sim-redis-unavailability'
}
}

stage('Backwards Compatibility') {
stage('Backwards Compatibility') {
parallel {
stage('Postgres') {
environment {
STORJ_NETWORK_HOST4 = '127.0.0.3'
STORJ_NETWORK_HOST6 = '127.0.0.3'
Expand All @@ -272,7 +241,7 @@ pipeline {
}
}

stage('Cockroach Backwards Compatibility') {
stage('Cockroach') {
environment {
STORJ_NETWORK_HOST4 = '127.0.0.5'
STORJ_NETWORK_HOST6 = '127.0.0.5'
Expand All @@ -287,6 +256,25 @@ pipeline {
sh 'cockroach sql --insecure --host=localhost:26257 -e \'drop database testcockroach5;\''
}
}
}
}

stage('Integration Redis unavailability') {
environment {
// use different hostname to avoid port conflicts
STORJ_NETWORK_HOST4 = '127.0.0.6'
STORJ_NETWORK_HOST6 = '127.0.0.6'
STORJ_REDIS_PORT = '7379'

STORJ_SIM_POSTGRES = 'postgres://postgres@localhost/teststorj6?sslmode=disable'
}

steps {
sh 'psql -U postgres -c \'create database teststorj6;\''
sh 'make test-sim-redis-unavailability'
}
}

stage('Post') {
parallel {
stage('Lint') {
Expand Down

0 comments on commit 65663b0

Please sign in to comment.