[7.17](backport #35780) Upgrade to Go 1.19.10 and handle changes in g…

…olang Debian base image (#35840)

* ci: use ubuntu:22 workers (#34315)

(cherry picked from commit a5f15b6)

# Conflicts:
#	.ci/beats-tester.groovy
#	.ci/build-docker-images.groovy
#	.ci/heartbeat-synthetics.groovy
#	x-pack/filebeat/Jenkinsfile.yml
#	x-pack/metricbeat/Jenkinsfile.yml

* remove unexisting files in 7.17 when backporting another PR

* fix merge conflicts

* Upgrade to Go 1.19.10 and handle changes in golang Debian base image (#35780)

* Explicitly install netcat-openbsd everywhere.

The plain netcat package appears to have been removed in the latest
golang:1.19.10 base image.

* Setup a virtualenv in Docker containers.

This fixes the PEP668 error about "externally managed environments"
which prevents modifying the system python installation.

* Revert "Revert "[Automation] Bump Golang version to 1.19.10 (#35751)" (#35784)"

This reverts commit 27b1799.

* Reword CHANGELOG.next.asciidoc

Co-authored-by: subham sarkar <sarkar.subhams2@gmail.com>

* Improve PEP668 comments.

* Install latest PyYAML in test containers.

* Add check to see if already in a venv.

* Upgrade pytest and py.

Remove dependency on pytest-otel as it prevented upgrading pytest.

* Check for pre-set virtualenv earlier.

* Always create a fresh virtualenv in Docker.

Remove duplicate dependencies from the docker files.

* Fix metricbeat integration tests.

* Force install Python dependencies for Metricbeat.

---------

Co-authored-by: subham sarkar <sarkar.subhams2@gmail.com>
(cherry picked from commit 62374dd)

# Conflicts:
#	dev-tools/mage/pytest.go
#	libbeat/docs/version.asciidoc
#	libbeat/tests/system/requirements.txt
#	libbeat/tests/system/requirements_aix.txt
#	metricbeat/Dockerfile

* Remove extra entry in CHANGELOG.next.asciidoc

* Fix conflict in version.asciidoc

* Fix conflict in requirements.txt

* Fix conflict in requirements_aix.txt

* Fix conflict in metricbeat Dockerfile

* Fix conflict in pytest.go

* Add missing force create virtual env option.

* python tests: compatibility and stability improvements (#31588)

This commit improves the compatibility and stability of Python tests
for Filebeat. It addresses two main probmes:
1. Some deprecated TLS functions are now raising deprecation
exceptions (since Python 3.10.0)
2. Some tests, on failure, leave Filebeat running and bound to port
8080, this creates a cascading failure of the subsequent tests that
require the same port to be available.

The main changes are:
 - Ensuring Filebeat process is always terminated, even if the test
 fails
 - Using the replacement methods/objects for the deprecated TLS
 functions.

(cherry picked from commit 682045a)

# Conflicts:
#	libbeat/tests/system/beat/beat.py

* Remove duplicate entries in CHANGELOG-developer.next.asciidoc

* Resolve conflicts in beat.py

---------

Co-authored-by: Victor Martinez <victormartinezrubio@gmail.com>
Co-authored-by: Craig MacKenzie <craig.mackenzie@elastic.co>
Co-authored-by: Tiago Queiroz <tiago.queiroz@elastic.co>

.ci/apm-beats-update.groovy

@@ -31,7 +31,7 @@ pipeline {
   }
   stages {
     stage('Filter build') {
-      agent { label 'ubuntu-18 && immutable' }
+      agent { label 'ubuntu-22 && immutable' }
       when {
         beforeAgent true
         anyOf {

.ci/beats-tester.groovy

@@ -24,7 +24,7 @@ pipeline {
   }
   stages {
     stage('Filter build') {
-      agent { label 'ubuntu && immutable' }
+      agent { label 'ubuntu-22' }
       when {
         beforeAgent true
         anyOf {

.ci/ironbank-validation.groovy

@@ -3,7 +3,7 @@
 @Library('apm@current') _
 
 pipeline {
-  agent { label 'ubuntu-20 && immutable' }
+  agent { label 'ubuntu-22 && immutable' }
   environment {
     REPO = 'beats'
     BASE_DIR = "src/github.com/elastic/${env.REPO}"

.ci/packaging.groovy

@@ -46,7 +46,7 @@ pipeline {
   stages {
     stage('Filter build') {
       options { skipDefaultCheckout() }
-      agent { label 'ubuntu-20 && immutable' }
+      agent { label 'ubuntu-22 && immutable' }
       when {
         beforeAgent true
         anyOf {
@@ -290,7 +290,7 @@ def generateArmStep(beat) {
 
 def generateLinuxStep(beat) {
   return {
-    withNode(labels: 'ubuntu-20.04 && immutable') {
+    withNode(labels: 'ubuntu-22.04 && immutable') {
       withEnv(["HOME=${env.WORKSPACE}", "PLATFORMS=${linuxPlatforms()}", "BEATS_FOLDER=${beat}"]) {
         withGithubNotify(context: "Packaging Linux ${beat}") {
           deleteDir()

.go-version

@@ -1 +1 @@
-1.19.9
+1.19.10

.golangci.yml

@@ -102,7 +102,7 @@ linters-settings:
 
   gosimple:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.19.9"
+    go: "1.19.10"
 
   nakedret:
     # make an issue if func has more lines of code than this setting and it has naked returns; default is 30
@@ -120,19 +120,19 @@ linters-settings:
 
   staticcheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.19.9"
+    go: "1.19.10"
     checks: ["all"]
 
   stylecheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.19.9"
+    go: "1.19.10"
     # Disabled:
     # ST1005: error strings should not be capitalized
     checks: ["all", "-ST1005"]
 
   unused:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.19.9"
+    go: "1.19.10"
 
   gosec:
     excludes:

CHANGELOG-developer.next.asciidoc

@@ -120,6 +120,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Update Go version to 1.17.1. {pull}27543[27543]
 - Whitelist `GCP_*` environment variables in dev tools {pull}28364[28364]
 - Improve tests files with shorter statements. {pull}35667[35667]
+- Improve compatibility and reduce flakyness of Python tests {pull}31588[31588]
 
 ==== Deprecated
 

CHANGELOG.next.asciidoc

@@ -9,6 +9,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 ==== Breaking changes
 
 *Affecting all Beats*
+- Update Go version to 1.19.10 {pull}35751[35751]
 
 *Auditbeat*
 

Jenkinsfile

@@ -3,7 +3,7 @@
 @Library('apm@current') _
 
 pipeline {
-  agent { label 'ubuntu-18 && immutable' }
+  agent { label 'ubuntu-22 && immutable' }
   environment {
     AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth'
     AWS_REGION = "${params.awsRegion}"

auditbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.19.9
+FROM golang:1.19.10
 
 RUN \
     apt-get update \
@@ -10,6 +10,10 @@ RUN \
          iproute2 \
       && rm -rf /var/lib/apt/lists/*
 
+# Use a virtualenv to avoid the PEP668 "externally managed environment" error caused by conflicts
+# with the system Python installation. golang:1.19.10 uses Debian 12 which now enforces PEP668.
+ENV VIRTUAL_ENV=/opt/venv
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
 RUN pip3 install --upgrade pip==20.1.1
-RUN pip3 install --upgrade setuptools==47.3.2
-RUN pip3 install --upgrade docker-compose==1.23.2

auditbeat/Jenkinsfile.yml

@@ -11,7 +11,7 @@ when:
     parameters:                ## when parameter was selected in the UI.
         - "auditbeat"
     tags: true                 ## for all the tags
-platform: "immutable && ubuntu-18" ## default label for all the stages
+platform: "immutable && ubuntu-22" ## default label for all the stages
 stages:
     arm:
         mage: "mage build unitTest"

auditbeat/scripts/mage/docs.go

@@ -71,7 +71,7 @@ func ModuleDocs() error {
 	}
 
 	// Run the docs_collector.py script.
-	ve, err := devtools.PythonVirtualenv()
+	ve, err := devtools.PythonVirtualenv(false)
 	if err != nil {
 		return err
 	}

deploy/kubernetes/Jenkinsfile.yml

@@ -10,7 +10,7 @@ when:
     parameters:                ## when parameter was selected in the UI.
         - "kubernetes"
     tags: true                 ## for all the tags
-platform: "immutable && ubuntu-18" ## default label for all the stages
+platform: "immutable && ubuntu-22" ## default label for all the stages
 stages:
     lint:
         make: |

dev-tools/mage/docs.go

@@ -70,7 +70,7 @@ var Docs = docsBuilder{}
 // FieldDocs generates docs/fields.asciidoc from the specified fields.yml file.
 func (docsBuilder) FieldDocs(fieldsYML string) error {
 	// Run the docs_collector.py script.
-	ve, err := PythonVirtualenv()
+	ve, err := PythonVirtualenv(false)
 	if err != nil {
 		return err
 	}

dev-tools/mage/fmt.go

@@ -96,7 +96,7 @@ func PythonAutopep8() error {
 	}
 
 	fmt.Println(">> fmt - autopep8: Formatting Python code")
-	ve, err := PythonVirtualenv()
+	ve, err := PythonVirtualenv(false)
 	if err != nil {
 		return err
 	}

dev-tools/mage/pytest.go

@@ -85,6 +85,7 @@ type PythonTestArgs struct {
 	Files               []string          // Globs used to find tests.
 	XUnitReportFile     string            // File to write the XUnit XML test report to.
 	CoverageProfileFile string            // Test coverage profile file.
+	ForceCreateVenv     bool              // Set to true to always install required dependencies in the test virtual environment.
 }
 
 func makePythonTestArgs(name string) PythonTestArgs {
@@ -114,7 +115,8 @@ func DefaultPythonTestIntegrationArgs() PythonTestArgs { return makePythonTestAr
 func PythonTest(params PythonTestArgs) error {
 	fmt.Println(">> python test:", params.TestName, "Testing")
 
-	ve, err := PythonVirtualenv()
+	// Only activate the virtualenv if necessary.
+	ve, err := PythonVirtualenv(params.ForceCreateVenv)
 	if err != nil {
 		return err
 	}
@@ -196,7 +198,7 @@ func PythonTestForModule(params PythonTestArgs) error {
 // PythonVirtualenv constructs a virtualenv that contains the given modules as
 // defined in the requirements file pointed to by requirementsTxt. It returns
 // the path to the virtualenv.
-func PythonVirtualenv() (string, error) {
+func PythonVirtualenv(forceCreate bool) (string, error) {
 	pythonVirtualenvLock.Lock()
 	defer pythonVirtualenvLock.Unlock()
 
@@ -217,7 +219,7 @@ func PythonVirtualenv() (string, error) {
 	// Only execute if requirements.txt is newer than the virtualenv activate
 	// script.
 	activate := virtualenvPath(ve, "activate")
-	if IsUpToDate(activate, reqs...) {
+	if !forceCreate && IsUpToDate(activate, reqs...) {
 		return pythonVirtualenvDir, nil
 	}
 
@@ -277,6 +279,12 @@ func pythonVirtualenvPath() (string, error) {
 		return pythonVirtualenvDir, nil
 	}
 
+	// If VIRTUAL_ENV is set we are already in a virtual environment.
+	pythonVirtualenvDir = os.Getenv("VIRTUAL_ENV")
+	if pythonVirtualenvDir != "" {
+		return pythonVirtualenvDir, nil
+	}
+
 	// PYTHON_ENV can override the default location. This is used by CI to
 	// shorten the overall shebang interpreter path below the path length limits.
 	pythonVirtualenvDir = os.Getenv("PYTHON_ENV")
@@ -288,6 +296,7 @@ func pythonVirtualenvPath() (string, error) {
 
 		pythonVirtualenvDir = info.RootDir
 	}
+
 	pythonVirtualenvDir = filepath.Join(pythonVirtualenvDir, "build/ve")
 
 	// Use OS and docker specific virtualenv's because the interpreter in

dev-tools/mage/target/compose/compose.go

@@ -59,7 +59,7 @@ func (c Compose) composeForEachVariant(action, message string) error {
 		return errors.Wrap(err, "finding supported versions files")
 	}
 
-	virtualenv, err := devtools.PythonVirtualenv()
+	virtualenv, err := devtools.PythonVirtualenv(false)
 	if err != nil {
 		return errors.Wrap(err, "configuring Python virtual environment")
 	}

dev-tools/mage/target/integtest/integtest.go

@@ -86,6 +86,10 @@ func PythonIntegTest(ctx context.Context) error {
 	}
 	return runner.Test("pythonIntegTest", func() error {
 		mg.Deps(devtools.BuildSystemTestBinary)
-		return devtools.PythonTest(devtools.DefaultPythonTestIntegrationArgs())
+		args := devtools.DefaultPythonTestIntegrationArgs()
+		// Always create a fresh virtual environment when running tests in a container, until we get
+		// get the requirements installed as part of the container build.
+		args.ForceCreateVenv = true
+		return devtools.PythonTest(args)
 	})
 }

filebeat/Jenkinsfile.yml

@@ -11,7 +11,7 @@ when:
     parameters:                ## when parameter was selected in the UI.
         - "filebeat"
     tags: true                 ## for all the tags
-platform: "immutable && ubuntu-18" ## default label for all the stages
+platform: "immutable && ubuntu-22" ## default label for all the stages
 stages:
     arm:
         mage: "mage build unitTest"

filebeat/scripts/mage/docs.go

@@ -26,7 +26,7 @@ import (
 // CollectDocs executes the Filebeat docs_collector script to collect/generate
 // documentation from each module.
 func CollectDocs() error {
-	ve, err := devtools.PythonVirtualenv()
+	ve, err := devtools.PythonVirtualenv(false)
 	if err != nil {
 		return err
 	}

filebeat/tests/system/test_syslog.py

@@ -30,6 +30,7 @@ def test_syslog_with_tcp(self):
         )
 
         filebeat = self.start_beat()
+        self.addCleanup(filebeat.kill_and_wait)
 
         self.wait_until(lambda: self.log_contains("Started listening for TCP connection"))
 
@@ -72,6 +73,7 @@ def test_syslog_with_tcp_invalid_message(self):
         )
 
         filebeat = self.start_beat()
+        self.addCleanup(filebeat.kill_and_wait)
 
         self.wait_until(lambda: self.log_contains("Started listening for TCP connection"))
 
@@ -112,6 +114,7 @@ def test_syslog_with_udp(self):
         )
 
         filebeat = self.start_beat()
+        self.addCleanup(filebeat.kill_and_wait)
 
         self.wait_until(lambda: self.log_contains("Started listening for UDP connection"))
 
@@ -172,6 +175,7 @@ def run_filebeat_and_send_using_socket(self, socket_type, send_over_socket):
             )
 
             filebeat = self.start_beat()
+            self.addCleanup(filebeat.kill_and_wait)
 
             self.wait_until(lambda: self.log_contains("Started listening for UNIX connection"))
 
@@ -232,6 +236,7 @@ def run_filebeat_and_send_invalid_message_using_socket(self, socket_type, send_o
             )
 
             filebeat = self.start_beat()
+            self.addCleanup(filebeat.kill_and_wait)
 
             self.wait_until(lambda: self.log_contains("Started listening for UNIX connection"))