Parallelize tests to shorten total time

.ci/Jenkinsfile

@@ -48,7 +48,7 @@ pipeline {
         /**
         Check the source code.
         */
-        stage('Check') {
+        stage('Check static') {
           steps {
             cleanup()
             dir("${BASE_DIR}"){
@@ -82,22 +82,27 @@ pipeline {
             }
           }
         }
-        stage('Integration Tests') {
+        stage('Integration tests') {
           failFast true
           options { skipDefaultCheckout() }
           steps {
-            withGithubNotify(context: 'Integration Tests', tab: 'tests') {
+            withGithubNotify(context: 'Integration tests', tab: 'tests') {
               cleanup()
               dir("${BASE_DIR}") {
                 script {
-                  parallel([
+                  def basicTasks = [
                     'stack-command-default': generateTestCommandStage(command: 'test-stack-command-default', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*']),
                     'stack-command-7x': generateTestCommandStage(command: 'test-stack-command-7x', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*']),
                     'stack-command-8x': generateTestCommandStage(command: 'test-stack-command-8x', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*']),
-                    'check-packages': generateTestCommandStage(command: 'test-check-packages', artifacts: ['build/test-results/*.xml', 'build/kubectl-dump.txt', 'build/elastic-stack-dump/check/logs/*.log', 'build/elastic-stack-dump/check/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true),
+                    'check-packages-with-kind': generateTestCommandStage(command: 'test-check-packages-with-kind', artifacts: ['build/test-results/*.xml', 'build/kubectl-dump.txt', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true),
+                    'check-packages-other': generateTestCommandStage(command: 'test-check-packages-other', artifacts: ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true),
                     'build-zip': generateTestCommandStage(command: 'test-build-zip', artifacts: ['build/elastic-stack-dump/build-zip/logs/*.log', 'build/integrations/*.sig']),
-                    'profiles-command': generateTestCommandStage(command: 'test-profiles-command'),
-                  ])
+                    'profiles-command': generateTestCommandStage(command: 'test-profiles-command')
+                  ]
+
+                  def checkSinglePackageTasks = generateTestCheckSinglePackageStage(artifacts: ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true)
+                  def tasks = basicTasks + checkSinglePackageTasks
+                  parallel tasks
                 }
               }
             }
@@ -137,14 +142,60 @@ def cleanup(){
   unstash 'source'
 }
 
+def generateTestCheckSinglePackageStage(Map args = [:]) {
+  def artifacts = args.get('artifacts') ? args.get('artifacts') : []
+  def junitArtifacts = args.get('junitArtifacts') ? args.get('junitArtifacts') : false
+  def publishCoverage = args.get('publishCoverage') ? args.get('publishCoverage') : false
+
+  def integrations = [:]
+  dir("test/packages/parallel") {
+      // Include hack to skip temporary files with "@tmp" suffix.
+      // For reference: https://issues.jenkins.io/browse/JENKINS-52750
+      findFiles()?.findAll{ !it.name.endsWith('@tmp') }?.collect{ it.name }?.sort()?.each {
+        integrations["check-package-${it}"] = {
+         withNode(labels: "ubuntu-20 && immutable", sleepMax: 20, forceWorkspace: true) {
+           cleanup()
+           try {
+             dir("${BASE_DIR}"){
+               withMageEnv(){
+                 withCloudTestEnv() {
+                   sh(label: 'Build elastic-package',script: "make build")
+                   sh(label: 'Build elastic-package',script: "make PACKAGE_UNDER_TEST=${it} test-check-packages-with-parallel")
+                 }
+               }
+             }
+           } finally {
+             dir("${BASE_DIR}") {
+               artifacts.each { artifact ->
+                 archiveArtifacts(allowEmptyArchive: true, artifacts: "${artifact}")
+               }
+
+               if (junitArtifacts) {
+                 junit(allowEmptyResults: true,
+                  keepLongStdio: true,
+                  testResults: "build/test-results/*.xml")
+               }
+
+               if (publishCoverage) {
+                 stashCoverageReport()
+               }
+             }
+           }
+         }
+        }
+      }
+  }
+  return integrations
+}
+
 def generateTestCommandStage(Map args = [:]){
   def command = args.get('command')
   def artifacts = args.get('artifacts') ? args.get('artifacts') : []
   def junitArtifacts = args.get('junitArtifacts') ? args.get('junitArtifacts') : false
   def publishCoverage = args.get('publishCoverage') ? args.get('publishCoverage') : false
 
   return {
-    withNode(labels: "ubuntu-18 && immutable", sleepMax: 20, forceWorkspace: true) {
+    withNode(labels: "ubuntu-20 && immutable", sleepMax: 20, forceWorkspace: true) {
       cleanup()
       try {
         dir("${BASE_DIR}"){

Makefile

@@ -53,8 +53,16 @@ test-stack-command-8x:
 
 test-stack-command: test-stack-command-default test-stack-command-7x test-stack-command-8x
 
-test-check-packages:
-	./scripts/test-check-packages.sh
+test-check-packages: test-check-packages-with-kind test-check-packages-with-other test-check-packages-with-parallel
+
+test-check-packages-with-kind:
+	PACKAGE_TEST_TYPE=with-kind ./scripts/test-check-packages.sh
+
+test-check-packages-other:
+	PACKAGE_TEST_TYPE=other ./scripts/test-check-packages.sh
+
+test-check-packages-with-parallel:
+	PACKAGE_TEST_TYPE=parallel ./scripts/test-check-packages.sh
 
 test-build-zip:
 	./scripts/test-build-zip.sh

internal/fields/validate_test.go

@@ -17,23 +17,23 @@ type results struct {
 }
 
 func TestValidate_NoWildcardFields(t *testing.T) {
-	validator, err := CreateValidatorForDataStream("../../test/packages/aws/data_stream/elb_logs")
+	validator, err := CreateValidatorForDataStream("../../test/packages/parallel/aws/data_stream/elb_logs")
 	require.NoError(t, err)
 	require.NotNil(t, validator)
 
-	f := readTestResults(t, "../../test/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log-expected.json")
+	f := readTestResults(t, "../../test/packages/parallel/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log-expected.json")
 	for _, e := range f.Expected {
 		errs := validator.ValidateDocumentBody(e)
 		require.Empty(t, errs)
 	}
 }
 
 func TestValidate_WithWildcardFields(t *testing.T) {
-	validator, err := CreateValidatorForDataStream("../../test/packages/aws/data_stream/sns")
+	validator, err := CreateValidatorForDataStream("../../test/packages/parallel/aws/data_stream/sns")
 	require.NoError(t, err)
 	require.NotNil(t, validator)
 
-	e := readSampleEvent(t, "../../test/packages/aws/data_stream/sns/sample_event.json")
+	e := readSampleEvent(t, "../../test/packages/parallel/aws/data_stream/sns/sample_event.json")
 	errs := validator.ValidateDocumentBody(e)
 	require.Empty(t, errs)
 }
@@ -240,12 +240,12 @@ func readSampleEvent(t *testing.T, path string) json.RawMessage {
 }
 
 func TestValidate_geo_point(t *testing.T) {
-	validator, err := CreateValidatorForDataStream("../../test/packages/fields_tests/data_stream/first")
+	validator, err := CreateValidatorForDataStream("../../test/packages/other/fields_tests/data_stream/first")
 
 	require.NoError(t, err)
 	require.NotNil(t, validator)
 
-	e := readSampleEvent(t, "../../test/packages/fields_tests/data_stream/first/sample_event.json")
+	e := readSampleEvent(t, "../../test/packages/other/fields_tests/data_stream/first/sample_event.json")
 	errs := validator.ValidateDocumentBody(e)
 	require.Empty(t, errs)
 }

scripts/test-build-zip.sh

@@ -12,7 +12,7 @@ cleanup() {
   elastic-package stack down -v
 
   # Clean used resources
-  for d in test/packages/*/; do
+  for d in test/packages/*/*/; do
     (
       cd $d
       elastic-package clean -v
@@ -29,7 +29,7 @@ OLDPWD=$PWD
 export ELASTIC_PACKAGE_SIGNER_PRIVATE_KEYFILE="$OLDPWD/scripts/gpg-private.asc"
 export ELASTIC_PACKAGE_SIGNER_PASSPHRASE=$(cat "$OLDPWD/scripts/gpg-pass.txt")
 
-for d in test/packages/*/; do
+for d in test/packages/*/*/; do
   (
     cd $d
     elastic-package build --zip --sign -v
@@ -45,7 +45,7 @@ eval "$(elastic-package stack shellinit)"
 elastic-package stack up -d -v
 
 # Install zipped packages
-for d in test/packages/*/; do
+for d in test/packages/*/*/; do
   (
     cd $d
     elastic-package install -v

scripts/test-check-packages.sh

@@ -6,20 +6,22 @@ cleanup() {
   r=$?
 
   # Dump stack logs
-  elastic-package stack dump -v --output build/elastic-stack-dump/check
+  elastic-package stack dump -v --output "build/elastic-stack-dump/check-${PACKAGE_UNDER_TEST:-${PACKAGE_TEST_TYPE:-any}}"
 
-  # Dump kubectl details
-  kubectl describe pods --all-namespaces > build/kubectl-dump.txt
-  kubectl logs -l app=elastic-agent -n kube-system >> build/kubectl-dump.txt
+  if [ "${PACKAGE_TEST_TYPE:-other}" == "with-kind" ]; then
+    # Dump kubectl details
+    kubectl describe pods --all-namespaces > build/kubectl-dump.txt
+    kubectl logs -l app=elastic-agent -n kube-system >> build/kubectl-dump.txt
 
-  # Take down the kind cluster
-  kind delete cluster
+    # Take down the kind cluster
+    kind delete cluster
+  fi
 
   # Take down the stack
   elastic-package stack down -v
 
   # Clean used resources
-  for d in test/packages/*/; do
+  for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do
     (
       cd $d
       elastic-package clean -v
@@ -33,7 +35,7 @@ trap cleanup EXIT
 
 OLDPWD=$PWD
 # Build/check packages
-for d in test/packages/*/; do
+for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do
   (
     cd $d
     elastic-package check -v
@@ -47,13 +49,15 @@ elastic-package stack update -v
 # Boot up the stack
 elastic-package stack up -d -v
 
-# Boot up the kind cluster
-kind create cluster --config $PWD/scripts/kind-config.yaml
+if [ "${PACKAGE_TEST_TYPE:-other}" == "with-kind" ]; then
+  # Boot up the kind cluster
+  kind create cluster --config $PWD/scripts/kind-config.yaml
+fi
 
 # Run package tests
 eval "$(elastic-package stack shellinit)"
 
-for d in test/packages/*/; do
+for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do
   (
     cd $d
     elastic-package install -v

scripts/test-stack-command.sh

@@ -13,14 +13,6 @@ cleanup() {
   # Take down the stack
   elastic-package stack down -v
 
-  # Clean used resources
-  for d in test/packages/*/; do
-    (
-      cd $d
-      elastic-package clean -v
-    )
-  done
-
   exit $r
 }