Upgrade to Go 1.19.10 and handle changes in golang Debian base image (#…

…35780)

* Explicitly install netcat-openbsd everywhere.

The plain netcat package appears to have been removed in the latest
golang:1.19.10 base image.

* Setup a virtualenv in Docker containers.

This fixes the PEP668 error about "externally managed environments"
which prevents modifying the system python installation.

* Revert "Revert "[Automation] Bump Golang version to 1.19.10 (#35751)" (#35784)"

This reverts commit 27b1799.

* Reword CHANGELOG.next.asciidoc

Co-authored-by: subham sarkar <sarkar.subhams2@gmail.com>

* Improve PEP668 comments.

* Install latest PyYAML in test containers.

* Add check to see if already in a venv.

* Upgrade pytest and py.

Remove dependency on pytest-otel as it prevented upgrading pytest.

* Check for pre-set virtualenv earlier.

* Always create a fresh virtualenv in Docker.

Remove duplicate dependencies from the docker files.

* Fix metricbeat integration tests.

* Force install Python dependencies for Metricbeat.

---------

Co-authored-by: subham sarkar <sarkar.subhams2@gmail.com>

.go-version

@@ -1 +1 @@
-1.19.9
+1.19.10

.golangci.yml

@@ -108,7 +108,7 @@ linters-settings:
 
   gosimple:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.19.9"
+    go: "1.19.10"
 
   nakedret:
     # make an issue if func has more lines of code than this setting and it has naked returns; default is 30
@@ -126,19 +126,19 @@ linters-settings:
 
   staticcheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.19.9"
+    go: "1.19.10"
     checks: ["all"]
 
   stylecheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.19.9"
+    go: "1.19.10"
     # Disabled:
     # ST1005: error strings should not be capitalized
     checks: ["all", "-ST1005"]
 
   unused:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.19.9"
+    go: "1.19.10"
 
   gosec:
     excludes:

CHANGELOG.next.asciidoc

@@ -9,6 +9,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 ==== Breaking changes
 
 *Affecting all Beats*
+- Update Go version to 1.19.10 {pull}35751[35751]
 - Fix status reporting to Elastic-Agent when output configuration is invalid running under Elastic-Agent {pull}35719[35719]
 
 *Auditbeat*

auditbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.19.9
+FROM golang:1.19.10
 
 RUN \
     apt-get update \
@@ -10,6 +10,10 @@ RUN \
          iproute2 \
       && rm -rf /var/lib/apt/lists/*
 
+# Use a virtualenv to avoid the PEP668 "externally managed environment" error caused by conflicts
+# with the system Python installation. golang:1.19.10 uses Debian 12 which now enforces PEP668.
+ENV VIRTUAL_ENV=/opt/venv
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
 RUN pip3 install --upgrade pip==20.1.1
-RUN pip3 install --upgrade setuptools==47.3.2
-RUN pip3 install --upgrade docker-compose==1.23.2

dev-tools/mage/pytest.go

@@ -86,6 +86,7 @@ type PythonTestArgs struct {
 	Files               []string          // Globs used to find tests.
 	XUnitReportFile     string            // File to write the XUnit XML test report to.
 	CoverageProfileFile string            // Test coverage profile file.
+	ForceCreateVenv     bool              // Set to true to always install required dependencies in the test virtual environment.
 }
 
 func makePythonTestArgs(name string) PythonTestArgs {
@@ -126,7 +127,7 @@ func PythonTest(params PythonTestArgs) error {
 	fmt.Println(">> python test:", params.TestName, "Testing")
 
 	// Only activate the virtualenv if necessary.
-	ve, err := PythonVirtualenv(false)
+	ve, err := PythonVirtualenv(params.ForceCreateVenv)
 	if err != nil {
 		return err
 	}
@@ -298,6 +299,12 @@ func pythonVirtualenvPath() (string, error) {
 		return pythonVirtualenvDir, nil
 	}
 
+	// If VIRTUAL_ENV is set we are already in a virtual environment.
+	pythonVirtualenvDir = os.Getenv("VIRTUAL_ENV")
+	if pythonVirtualenvDir != "" {
+		return pythonVirtualenvDir, nil
+	}
+
 	// PYTHON_ENV can override the default location. This is used by CI to
 	// shorten the overall shebang interpreter path below the path length limits.
 	pythonVirtualenvDir = os.Getenv("PYTHON_ENV")
@@ -309,6 +316,7 @@ func pythonVirtualenvPath() (string, error) {
 
 		pythonVirtualenvDir = info.RootDir
 	}
+
 	pythonVirtualenvDir = filepath.Join(pythonVirtualenvDir, "build/ve")
 
 	// Use OS and docker specific virtualenv's because the interpreter in

dev-tools/mage/target/integtest/integtest.go

@@ -90,6 +90,10 @@ func PythonIntegTest(ctx context.Context) error {
 	}
 	return runner.Test("pythonIntegTest", func() error {
 		mg.Deps(devtools.BuildSystemTestBinary)
-		return devtools.PythonTest(devtools.DefaultPythonTestIntegrationArgs())
+		args := devtools.DefaultPythonTestIntegrationArgs()
+		// Always create a fresh virtual environment when running tests in a container, until we get
+		// get the requirements installed as part of the container build.
+		args.ForceCreateVenv = true
+		return devtools.PythonTest(args)
 	})
 }

heartbeat/Dockerfile

@@ -1,17 +1,21 @@
-FROM golang:1.19.9
+FROM golang:1.19.10
 
 RUN \
     apt-get update \
       && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-         netcat \
+         netcat-openbsd \
          python3 \
          python3-pip \
          python3-venv \
       && rm -rf /var/lib/apt/lists/*
 
+# Use a virtualenv to avoid the PEP668 "externally managed environment" error caused by conflicts
+# with the system Python installation. golang:1.19.10 uses Debian 12 which now enforces PEP668.
+ENV VIRTUAL_ENV=/opt/venv
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
 RUN pip3 install --upgrade pip==20.1.1
-RUN pip3 install --upgrade setuptools==47.3.2
-RUN pip3 install --upgrade docker-compose==1.23.2
 
 # Setup work environment
 ENV HEARTBEAT_PATH /go/src/github.com/elastic/beats/heartbeat

libbeat/docs/version.asciidoc

@@ -1,6 +1,6 @@
 :stack-version: 8.9.0
 :doc-branch: master
-:go-version: 1.19.9
+:go-version: 1.19.10
 :release-state: unreleased
 :python: 3.7
 :docker: 1.12

libbeat/tests/system/requirements.txt

@@ -29,12 +29,11 @@ ordered-set==3.1.1
 packaging==20.4
 parameterized==0.7.0
 pluggy==0.13.1
-py==1.10.0
+py==1.11.0
 pycodestyle==2.6.0
 pyparsing==2.4.7
 pyrsistent==0.16.0
-pytest==7.1.3
-pytest-otel==1.3.0
+pytest==7.3.2
 pytest-rerunfailures==9.1.1
 pytest-timeout==1.4.2
 PyYAML==5.4.1

libbeat/tests/system/requirements_aix.txt

@@ -28,12 +28,11 @@ ordered-set==3.1.1
 packaging==20.4
 parameterized==0.7.0
 pluggy==0.13.1
-py==1.10.0
+py==1.11.0
 pycodestyle==2.6.0
 pyparsing==2.4.7
 pyrsistent==0.16.0
-pytest==7.1.3
-pytest-otel==1.3.0
+pytest==7.3.2
 pytest-rerunfailures==9.1.1
 pytest-timeout==1.4.2
 PyYAML==5.4.1

metricbeat/Dockerfile

@@ -1,9 +1,9 @@
-FROM golang:1.19.9
+FROM golang:1.19.10
 
 RUN \
     apt update \
       && DEBIAN_FRONTEND=noninteractive apt-get install -qq -y --no-install-recommends \
-         netcat \
+         netcat-openbsd \
          python3 \
          python3-dev \
          python3-pip \
@@ -12,9 +12,16 @@ RUN \
          unzip \
       && rm -rf /var/lib/apt/lists/*
 
+# Use a virtualenv to avoid the PEP668 "externally managed environment" error caused by conflicts
+# with the system Python installation. golang:1.19.10 uses Debian 12 which now enforces PEP668.
+ENV VIRTUAL_ENV=/opt/venv
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
 RUN pip3 install --upgrade pip==20.1.1
-RUN pip3 install --upgrade setuptools==47.3.2
 RUN pip3 install --upgrade docker-compose==1.23.2
+RUN pip3 install --upgrade setuptools==47.3.2
+RUN pip3 install --upgrade PyYAML==6.0.0
 
 # Oracle instant client
 RUN cd /usr/lib \
@@ -23,7 +30,5 @@ RUN cd /usr/lib \
   && rm instantclient-basic-linux.zip
 ENV LD_LIBRARY_PATH=/usr/lib/instantclient_19_6
 
-ENV PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/oracle/12.2/client64/bin
-
 # Add healthcheck for the docker/healthcheck metricset to check during testing.
 HEALTHCHECK CMD exit 0

metricbeat/magefile.go

@@ -240,6 +240,10 @@ func PythonIntegTest(ctx context.Context) error {
 	}
 	return runner.Test("pythonIntegTest", func() error {
 		mg.Deps(devtools.BuildSystemTestBinary)
-		return devtools.PythonTestForModule(devtools.DefaultPythonTestIntegrationArgs())
+		args := devtools.DefaultPythonTestIntegrationArgs()
+		// Always create a fresh virtual environment when running tests in a container, until we get
+		// get the requirements installed as part of the container build.
+		args.ForceCreateVenv = true
+		return devtools.PythonTestForModule(args)
 	})
 }

metricbeat/module/aerospike/_meta/Dockerfile

@@ -1,5 +1,5 @@
 ARG AEROSPIKE_VERSION
 FROM aerospike:${AEROSPIKE_VERSION}
 
-RUN apt-get update && apt-get install -y netcat
+RUN apt-get update && apt-get install -y netcat-openbsd
 HEALTHCHECK --interval=1s --retries=90 CMD nc -z localhost 3000

metricbeat/module/haproxy/_meta/Dockerfile

@@ -1,6 +1,6 @@
 ARG HAPROXY_VERSION
 FROM haproxy:${HAPROXY_VERSION}
-RUN apt-get update && apt-get install -y netcat
+RUN apt-get update && apt-get install -y netcat-openbsd
 
 HEALTHCHECK --interval=1s --retries=90 CMD nc -z localhost 14567 && nc -z localhost 14570
 

metricbeat/module/kafka/_meta/Dockerfile

@@ -11,9 +11,9 @@ ENV TERM=linux
 # TODO: use newer base
 RUN sed -i s/deb.debian.org/archive.debian.org/g /etc/apt/sources.list \
     && sed -i 's|security.debian.org|archive.debian.org/|g' /etc/apt/sources.list \
-    && sed -i '/stretch-updates/d' /etc/apt/sources.list 
-    
-RUN apt-get update && apt-get install -y curl openjdk-8-jre-headless netcat dnsutils
+    && sed -i '/stretch-updates/d' /etc/apt/sources.list
+
+RUN apt-get update && apt-get install -y curl openjdk-8-jre-headless netcat-openbsd dnsutils
 
 RUN mkdir -p ${KAFKA_LOGS_DIR} && mkdir -p ${KAFKA_HOME} && \
     curl -J -L -s -f -o - https://github.com/kadwanev/retry/releases/download/1.0.1/retry-1.0.1.tar.gz | tar xfz - -C /usr/local/bin && \

metricbeat/module/mongodb/_meta/Dockerfile

@@ -1,4 +1,4 @@
 ARG MONGODB_VERSION
 FROM mongo:${MONGODB_VERSION}
-RUN apt update && yes | apt install netcat
+RUN apt update && yes | apt install netcat-openbsd
 HEALTHCHECK --interval=1s --retries=90 CMD nc -z localhost 27017

metricbeat/module/munin/_meta/Dockerfile

@@ -1,7 +1,7 @@
 FROM ubuntu:16.04
 
 RUN apt-get update && \
-    apt-get install -y munin-node netcat && \
+    apt-get install -y munin-node netcat-openbsd && \
     apt-get clean && rm rm -rf /var/lib/apt/lists/*
 
 EXPOSE 4949

metricbeat/module/rabbitmq/_meta/Dockerfile

@@ -1,6 +1,6 @@
 ARG RABBITMQ_VERSION
 FROM rabbitmq:${RABBITMQ_VERSION}-management
 
-RUN apt-get update && apt-get install -y netcat && apt-get clean
+RUN apt-get update && apt-get install -y netcat-openbsd && apt-get clean
 HEALTHCHECK --interval=1s --retries=90 CMD nc -w 1 -v 127.0.0.1 15672 </dev/null
 EXPOSE 15672

packetbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.19.9
+FROM golang:1.19.10
 
 RUN \
     apt-get update \
@@ -7,10 +7,14 @@ RUN \
          python3-pip \
          python3-venv \
          librpm-dev \
-         netcat \
+         netcat-openbsd \
          libpcap-dev \
       && rm -rf /var/lib/apt/lists/*
 
+# Use a virtualenv to avoid the PEP668 "externally managed environment" error caused by conflicts
+# with the system Python installation. golang:1.19.10 uses Debian 12 which now enforces PEP668.
+ENV VIRTUAL_ENV=/opt/venv
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
 RUN pip3 install --upgrade pip==20.1.1
-RUN pip3 install --upgrade setuptools==47.3.2
-RUN pip3 install --upgrade docker-compose==1.23.2

testing/environments/docker/kafka/Dockerfile

@@ -10,7 +10,7 @@ ENV KAFKA_VERSION 2.2.2
 ENV _JAVA_OPTIONS "-Djava.net.preferIPv4Stack=true"
 ENV TERM=linux
 
-RUN apt-get update && apt-get install -y curl openjdk-11-jre-headless netcat
+RUN apt-get update && apt-get install -y curl openjdk-11-jre-headless netcat-openbsd
 
 RUN mkdir -p ${KAFKA_LOGS_DIR} && mkdir -p ${KAFKA_HOME} && \
     curl -J -L -s -f -o - https://github.com/kadwanev/retry/releases/download/1.0.1/retry-1.0.1.tar.gz | tar xfz - -C /usr/local/bin && \

x-pack/functionbeat/Dockerfile

@@ -1,18 +1,22 @@
-FROM golang:1.19.9
+FROM golang:1.19.10
 
 RUN \
     apt-get update \
       && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-         netcat \
+         netcat-openbsd \
          rsync \
          python3 \
          python3-pip \
          python3-venv \
       && rm -rf /var/lib/apt/lists/*
 
+# Use a virtualenv to avoid the PEP668 "externally managed environment" error caused by conflicts
+# with the system Python installation. golang:1.19.10 uses Debian 12 which now enforces PEP668.
+ENV VIRTUAL_ENV=/opt/venv
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
 RUN pip3 install --upgrade pip==20.1.1
-RUN pip3 install --upgrade setuptools==47.3.2
-RUN pip3 install --upgrade docker-compose==1.23.2
 
 # Setup work environment
 ENV FUNCTIONBEAT_PATH /go/src/github.com/elastic/beats/x-pack/functionbeat

x-pack/metricbeat/magefile.go

@@ -256,6 +256,9 @@ func PythonIntegTest(ctx context.Context) error {
 	return runner.Test("pythonIntegTest", func() error {
 		mg.Deps(BuildSystemTestBinary)
 		args := devtools.DefaultPythonTestIntegrationArgs()
+		// Always create a fresh virtual environment when running tests in a container, until we get
+		// get the requirements installed as part of the container build.
+		args.ForceCreateVenv = true
 		// On Windows 32-bit converage is not enabled.
 		if isWindows32bitRunner() {
 			args.Env["TEST_COVERAGE"] = "false"