Skip to content

Commit

Permalink
feat: filter changes using go list output (#17397) (#17748)
Browse files Browse the repository at this point in the history 
* feat: filter changes using go list output

* fix: auditbeat trigger
  • Loading branch information
@kuisathaverat
kuisathaverat committed Apr 20, 2020
1 parent c268fcb commit 81ce564
Showing 1 changed file with 53 additions and 49 deletions.
102 changes: 53 additions & 49 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ pipeline {
BASE_DIR = 'src/github.com/elastic/beats'
GOX_FLAGS = "-arch amd64"
DOCKER_COMPOSE_VERSION = "1.21.0"
PIPELINE_LOG_LEVEL = "INFO"
}
options {
timeout(time: 2, unit: 'HOURS')
Expand All @@ -17,7 +18,6 @@ pipeline {
disableResume()
durabilityHint('PERFORMANCE_OPTIMIZED')
disableConcurrentBuilds()
// checkoutToSubdirectory "${env.BASE_DIR}"
}
triggers {
issueCommentTrigger('(?i).*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*')
Expand All @@ -27,6 +27,7 @@ pipeline {
booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.')
booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.')
booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps')
booleanParam(name: 'dry_run', defaultValue: false, description: 'Skip build steps, it is for testing pipeline flow')
}
stages {
/**
Expand Down Expand Up @@ -620,7 +621,9 @@ def withBeatsEnv(boolean archive, Closure body) {
sh(label: "Install docker-compose ${DOCKER_COMPOSE_VERSION}", script: ".ci/scripts/install-docker-compose.sh")
sh(label: "Install Mage", script: "make mage")
try {
body()
if(!params.dry_run){
body()
}
} finally {
if (archive) {
catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
Expand Down Expand Up @@ -653,7 +656,9 @@ def withBeatsEnvWin(Closure body) {
dir("${env.BASE_DIR}"){
bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat")
try {
body()
if(!params.dry_run){
body()
}
} finally {
catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
junit(allowEmptyResults: true, keepLongStdio: true, testResults: "**\\build\\TEST*.xml")
Expand Down Expand Up @@ -789,19 +794,21 @@ def isChanged(patterns){
}

def isChangedOSSCode(patterns) {
def always = [
def allPatterns = [
"^Jenkinsfile",
"^vendor/.*",
"^libbeat/.*",
"^testing/.*",
"^dev-tools/.*",
"^\\.ci/.*",
]
return isChanged(always + patterns)
allPatterns.addAll(patterns)
allPatterns.addAll(getVendorPatterns('libbeat'))
return isChanged(allPatterns)
}

def isChangedXPackCode(patterns) {
def always = [
def allPatterns = [
"^Jenkinsfile",
"^vendor/.*",
"^libbeat/.*",
Expand All @@ -810,86 +817,83 @@ def isChangedXPackCode(patterns) {
"^x-pack/libbeat/.*",
"^\\.ci/.*",
]
return isChanged(always + patterns)
allPatterns.addAll(patterns)
allPatterns.addAll(getVendorPatterns('x-pack/libbeat'))
return isChanged(allPatterns)
}

def loadConfigEnvVars(){
def empty = []
env.GO_VERSION = readFile(".go-version").trim()

// Libbeat is the core framework of Beats. It has no additional dependencies
// on other projects in the Beats repository.
env.BUILD_LIBBEAT = isChangedOSSCode([])
env.BUILD_LIBBEAT_XPACK = env.BUILD_LIBBEAT || isChangedXPackCode([])
env.BUILD_LIBBEAT = isChangedOSSCode(empty)
env.BUILD_LIBBEAT_XPACK = isChangedXPackCode(empty)

// Auditbeat depends on metricbeat as framework, but does not include any of
// the modules from Metricbeat.
// The Auditbeat x-pack build contains all functionality from OSS Auditbeat.
env.BUILD_AUDITBEAT = isChangedOSSCode([
"^metricbeat/.*",
"^auditbeat/.*",
])
env.BUILD_AUDITBEAT_XPACK = env.BUILD_AUDITBEAT || isChangedXPackCode([
"^x-pack/auditbeat/.*",
])
env.BUILD_AUDITBEAT = isChangedOSSCode(getVendorPatterns('auditbeat'))
env.BUILD_AUDITBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/auditbeat'))

// Dockerlogbeat is a standalone Beat that only relies on libbeat.
env.BUILD_DOCKERLOGBEAT_XPACK = isChangedXPackCode([
"^x-pack/dockerlogbeat/.*",
])
env.BUILD_DOCKERLOGBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/dockerlogbeat'))

// Filebeat depends on libbeat only.
// The Filebeat x-pack build contains all functionality from OSS Filebeat.
env.BUILD_FILEBEAT = isChangedOSSCode(["^filebeat/.*"])
env.BUILD_FILEBEAT_XPACK = env.BUILD_FILEBEAT || isChangedXPackCode([
"^x-pack/filebeat/.*",
])
env.BUILD_FILEBEAT = isChangedOSSCode(getVendorPatterns('filebeat'))
env.BUILD_FILEBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/filebeat'))

// Metricbeat depends on libbeat only.
// The Metricbeat x-pack build contains all functionality from OSS Metricbeat.
env.BUILD_METRICBEAT = isChangedOSSCode(["^metricbeat/.*"])
env.BUILD_METRICBEAT_XPACK = env.BUILD_METRICBEAT || isChangedXPackCode([
"^x-pack/metricbeat/.*",
])
env.BUILD_METRICBEAT = isChangedOSSCode(getVendorPatterns('metricbeat'))
env.BUILD_METRICBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/metricbeat'))

// Functionbeat is a standalone beat that depends on libbeat only.
// Functionbeat is available as x-pack build only.
env.BUILD_FUNCTIONBEAT_XPACK = isChangedXPackCode([
"^x-pack/functionbeat/.*",
])
env.BUILD_FUNCTIONBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/functionbeat'))

// Heartbeat depends on libbeat only.
// The Heartbeat x-pack build contains all functionality from OSS Heartbeat.
env.BUILD_HEARTBEAT = isChangedOSSCode(["^heartbeat/.*"])
env.BUILD_HEARTBEAT_XPACK = env.BUILD_HEARTBEAT || isChangedXPackCode([
"^x-pack/heartbeat/.*",
])

env.BUILD_HEARTBEAT = isChangedOSSCode(getVendorPatterns('heartbeat'))
env.BUILD_HEARTBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/heartbeat'))

// Journalbeat depends on libbeat only.
// The Journalbeat x-pack build contains all functionality from OSS Journalbeat.
env.BUILD_JOURNALBEAT = isChangedOSSCode(["^journalbeat/.*"])
env.BUILD_JOURNALBEAT_XPACK = env.BUILD_JOURNALBEAT || isChangedXPackCode([
"^x-pack/journalbeat/.*",
])
env.BUILD_JOURNALBEAT = isChangedOSSCode(getVendorPatterns('journalbeat'))
env.BUILD_JOURNALBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/journalbeat'))

// Packetbeat depends on libbeat only.
// The Packetbeat x-pack build contains all functionality from OSS Packetbeat.
env.BUILD_PACKETBEAT = isChangedOSSCode(["^packetbeat/.*"])
env.BUILD_PACKETBEAT_XPACK = env.BUILD_PACKETBEAT || isChangedXPackCode([
"^x-pack/packetbeat/.*",
])
env.BUILD_PACKETBEAT = isChangedOSSCode(getVendorPatterns('packetbeat'))
env.BUILD_PACKETBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/packetbeat'))

// Winlogbeat depends on libbeat only.
// The Winlogbeat x-pack build contains all functionality from OSS Winlogbeat.
env.BUILD_WINLOGBEAT = isChangedOSSCode(["^winlogbeat/.*"])
env.BUILD_WINLOGBEAT_XPACK = env.BUILD_WINLOGBEAT || isChangedXPackCode([
"^x-pack/winlogbeat/.*",
])
env.BUILD_WINLOGBEAT = isChangedOSSCode(getVendorPatterns('winlogbeat'))
env.BUILD_WINLOGBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/winlogbeat'))

// The Kubernetes test use Filebeat and Metricbeat, but only need to be run
// if the deployment scripts have been updated. No Beats specific testing is
// involved.
env.BUILD_KUBERNETES = isChanged(["^deploy/kubernetes/.*"])

env.BUILD_GENERATOR = isChangedOSSCode(["^generator/.*"])
env.BUILD_GENERATOR = isChangedOSSCode(getVendorPatterns('generator'))
}

env.GO_VERSION = readFile(".go-version").trim()
/**
This method grab the dependencies of a Go module and transform them on regexp
*/
def getVendorPatterns(beatName){
def output = ""
docker.image("golang:${GO_VERSION}").inside{
output = sh(label: 'Get vendor dependency patterns', returnStdout: true, script: """
export HOME=${WORKSPACE}/${BASE_DIR}
go list -mod=vendor -f '{{ .ImportPath }}{{ "\\n" }}{{ join .Deps "\\n" }}' ./${beatName}\
|awk '{print \$1"/.*"}'\
|sed -e "s#github.com/elastic/beats/v7/##g"
""")
}
return output?.split('\n').collect{ item -> item as String }
}

0 comments on commit 81ce564

Please sign in to comment.