Merge remote-tracking branch 'upstream/main' into feature/test-opente…

…lemetry-actions

* upstream/main: (148 commits)
  chore(deps): bump jsonwebtoken and @azure/msal-node (elastic#3087)
  fix bitrot.js dev-util to work for Next.js versions (elastic#3086)
  synchronize json schema specs (elastic#3082)
  chore(deps): bump json5 from 1.0.1 to 1.0.2 (elastic#3085)
  synchronize json schema specs (elastic#3078)
  chore(deps-dev): bump fastify from 4.10.2 to 4.11.0 (elastic#3083)
  chore(deps): bump next (elastic#3081)
  docs: fix header name with sampled flag (elastic#3069)
  3.41.1 (elastic#3077)
  fix: span compression bug where a buffered span would not be sent when an incompressible sibling ended (elastic#3076)
  chore(deps-dev): bump wait-on from 6.0.1 to 7.0.1 (elastic#3075)
  chore(deps-dev): bump undici from 5.12.0 to 5.14.0 (elastic#3068)
  chore(deps-dev): bump got from 11.8.5 to 11.8.6 (elastic#3067)
  chore(deps-dev): bump koa from 2.13.4 to 2.14.1 (elastic#3066)
  chore(deps-dev): bump @hapi/hapi from 21.0.0 to 21.1.0 (elastic#3058)
  chore(deps-dev): bump @fastify/formbody from 7.3.0 to 7.4.0 (elastic#3057)
  3.41.0 (elastic#3064)
  Support publishing a snapshot build for each commit to main (elastic#3050)
  fix: Add `tracestate` to the `TransactionOptions` TypeScript type (elastic#3063)
  fix: avoid IPv4 vs IPv6 ambiguity in default 'serverUrl' by using '127.0.0.1' rather than 'localhost' (elastic#3049)
  ...

.ci/.jenkins_nightly_nodejs.yml

@@ -9,4 +9,4 @@
 # made, these will stop and there will be no value in testing v17 nightlies.
 #
 NODEJS_VERSION:
-  - "19"
+  - "20"

.ci/.jenkins_nodejs.yml

@@ -1,7 +1,7 @@
 NODEJS_VERSION:
-  # - "19" # Uncomment this when there are v19.x releases at https://nodejs.org/download/release/
+  - "19"
   - "18"
-  # - "18.0"  # Uncomment this when "18" is 18.1 or greater.
+  - "18.0"
   - "16"
   - "16.0"
   - "14"

.ci/.jenkins_rc_nodejs.yml

@@ -4,7 +4,7 @@
 # The node.js project *sometimes* produces "rc" builds leading up to a new
 # release. They get uploaded to: https://nodejs.org/download/rc/
 NODEJS_VERSION:
+  - "19"
   - "18"
   - "16"
   - "14"
-  - "12"

.ci/.jenkins_tav.yml

@@ -26,9 +26,11 @@ TAV:
   - mongodb-core
   - mysql
   - mysql2
+  - next
   - pg
   - pug
   - redis
   - restify
   - tedious
+  - undici
   - ws

.ci/.jenkins_tav_nodejs.yml

@@ -1,5 +1,5 @@
 NODEJS_VERSION:
-  # - "19" # Uncomment this when there are v19.x releases at https://nodejs.org/en/about/releases/
+  - "19"
   - "18"
   - "16"
   - "14"

.ci/Jenkinsfile

@@ -8,15 +8,13 @@ pipeline {
     BASE_DIR = "src/github.com/elastic/${env.REPO}"
     PIPELINE_LOG_LEVEL='INFO'
     JOB_GCS_BUCKET = credentials('gcs-bucket')
-    GITHUB_CHECK_ITS_NAME = 'Integration Tests'
-    ITS_PIPELINE = 'apm-integration-tests-selector-mbp/main'
     GITHUB_CHECK = 'true'
     RELEASE_URL_MESSAGE = "(<https://github.com/elastic/${env.REPO}/releases/tag/${env.TAG_NAME}|${env.TAG_NAME}>)"
     SLACK_CHANNEL = '#apm-agent-node'
     NOTIFY_TO = 'build-apm+apm-agent-nodejs@elastic.co'
     NPMRC_SECRET = 'secret/jenkins-ci/npmjs/elasticmachine'
     TOTP_SECRET = 'totp/code/npmjs-elasticmachine'
-    BUILD_NODE_VERSION = 'v16.15.0'
+    BUILD_NODE_VERSION = 'v16.15.1'
     DOCKER_REGISTRY = 'docker.elastic.co'
     DOCKER_SECRET = 'secret/apm-team/ci/docker-registry/prod'
   }
@@ -92,7 +90,7 @@ pipeline {
               def parallelTasks = [:]
               node['NODEJS_VERSION'].each{ version ->
                 parallelTasks["Node.js-${version}"] = generateStep(version: version)
-                parallelTasks["Node.js-${version}-async-hooks-false"] = generateStep(version: version, disableAsyncHooks: true)
+                parallelTasks["Node.js-${version}-noasynchooks"] = generateStep(version: version, disableAsyncHooks: true)
                 // TODO: to be enabled if required.
                 // parallelTasks["Windows-Node.js-${version}"] = generateStepForWindows(version: version)
               }
@@ -193,18 +191,18 @@ pipeline {
             }
           }
         }
-        stage('Nightly Test - No async hooks') {
+        stage('Nightly Test - noasynchooks') {
           agent { label 'linux && immutable' }
           steps {
-            withGithubNotify(context: 'Nightly No Async Hooks Test', tab: 'tests') {
+            withGithubNotify(context: 'Nightly noasynchooks Test', tab: 'tests') {
               deleteDir()
               unstash 'source'
               dir("${BASE_DIR}"){
                 script {
                   def node = readYaml(file: '.ci/.jenkins_nightly_nodejs.yml')
                   def parallelTasks = [:]
                   node['NODEJS_VERSION'].each { version ->
-                    parallelTasks["Node.js-${version}-nightly-no-async-hooks"] = generateStep(version: version, buildType: 'nightly', disableAsyncHooks: true)
+                    parallelTasks["Node.js-${version}-nightly-noasynchooks"] = generateStep(version: version, buildType: 'nightly', disableAsyncHooks: true)
                   }
                   parallel(parallelTasks)
                 }
@@ -231,18 +229,18 @@ pipeline {
             }
           }
         }
-        stage('RC Test - No async hooks') {
+        stage('RC Test - noasynchooks') {
           agent { label 'linux && immutable' }
           steps {
-            withGithubNotify(context: 'RC No Async Hooks Test', tab: 'tests') {
+            withGithubNotify(context: 'RC noasynchooks Test', tab: 'tests') {
               deleteDir()
               unstash 'source'
               dir("${BASE_DIR}"){
                 script {
                   def node = readYaml(file: '.ci/.jenkins_rc_nodejs.yml')
                   def parallelTasks = [:]
                   node['NODEJS_VERSION'].each { version ->
-                    parallelTasks["Node.js-${version}-rc-no-async-hooks"] = generateStep(version: version, buildType: 'rc', disableAsyncHooks: true)
+                    parallelTasks["Node.js-${version}-rc-noasynchooks"] = generateStep(version: version, buildType: 'rc', disableAsyncHooks: true)
                   }
                   parallel(parallelTasks)
                 }
@@ -252,29 +250,39 @@ pipeline {
         }
       }
     }
-    stage('Integration Tests') {
-      agent none
-      when {
-        beforeAgent true
-        allOf {
-          not { tag pattern: 'v\\d+\\.\\d+\\.\\d+', comparator: 'REGEXP' }
-          expression { return env.ONLY_DOCS == "false" }
-          anyOf {
-            changeRequest()
-            expression { return !params.Run_As_Main_Branch }
-          }
-        }
+
+    /**
+    Publish a snapshot. A "snapshot" is a packaging of the latest *unreleased* APM agent,
+    published to a known GCS bucket for use in edge demo/test environments.
+    */
+    stage('Publish snapshot') {
+      options { skipDefaultCheckout() }
+      environment {
+        BUCKET_NAME = 'oblt-artifacts'
+        DOCKER_REGISTRY = 'docker.elastic.co'
+        DOCKER_REGISTRY_SECRET = 'secret/observability-team/ci/docker-registry/prod'
+        GCS_ACCOUNT_SECRET = 'secret/observability-team/ci/snapshoty'
       }
+      when { branch 'main' }
       steps {
-        build(job: env.ITS_PIPELINE, propagate: false, wait: false,
-              parameters: [string(name: 'INTEGRATION_TEST', value: 'Node.js'),
-                           string(name: 'BUILD_OPTS', value: "--nodejs-agent-package ${env.CHANGE_FORK?.trim() ?: 'elastic' }/${env.REPO}#${env.GIT_BASE_COMMIT} --opbeans-node-agent-branch ${env.GIT_BASE_COMMIT}"),
-                           string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_ITS_NAME),
-                           string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
-                           string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT)])
-        githubNotify(context: "${env.GITHUB_CHECK_ITS_NAME}", description: "${env.GITHUB_CHECK_ITS_NAME} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${env.ITS_PIPELINE.replaceAll('/','+')}")
+        withGithubNotify(context: 'Publish snapshot packages') {
+          deleteDir()
+          unstash name: 'source'
+          withNodeJSEnv(version: env.BUILD_NODE_VERSION) {
+            dir(env.BASE_DIR) {
+              sh(label: 'package snapshot', script: 'npm run package:snapshot')
+              snapshoty(
+                bucket: env.BUCKET_NAME,
+                gcsAccountSecret: env.GCS_ACCOUNT_SECRET,
+                dockerRegistry: env.DOCKER_REGISTRY,
+                dockerSecret: env.DOCKER_REGISTRY_SECRET
+              )
+            }
+          }
+        }
       }
     }
+
     stage('Release') {
       options { skipDefaultCheckout() }
       when {
@@ -371,7 +379,7 @@ pipeline {
       The result JSON files are also archive into Jenkins.
     */
     stage('Benchmarks') {
-      agent { label 'metal' }
+      agent { label 'microbenchmarks-pool' }
       options { skipDefaultCheckout() }
       environment {
         HOME = "${env.WORKSPACE}"
@@ -424,22 +432,23 @@ def generateStep(Map params = [:]){
   def version = params?.version
   def tav = params.containsKey('tav') ? params.tav : ''
   def buildType = params.containsKey('buildType') ? params.buildType : 'release'
-  def ELASTIC_APM_ASYNC_HOOKS = String.valueOf(!params.get('disableAsyncHooks', false))
+  def contextManager = params.get('disableAsyncHooks', false) ? 'patch' : ''
   return {
     withNode(labels: 'linux && immutable', forceWorkspace: true, forceWorker: true) {
-      withEnv(["VERSION=${version}", "ELASTIC_APM_ASYNC_HOOKS=${ELASTIC_APM_ASYNC_HOOKS}"]) {
+      withEnv(["VERSION=${version}", "ELASTIC_APM_CONTEXT_MANAGER=${contextManager}"]) {
         deleteDir()
         unstash 'source'
-        dir("${BASE_DIR}"){
-          try {
-            retryWithSleep(retries: 2, seconds: 5, backoff: true) {
-              sh(label: "Run Tests", script: """.ci/scripts/test.sh -b "${buildType}" -t "${tav}" "${version}" """)
+        // Grab the current docker context for helping to troubleshoot the docker containers using filebeat and metricbeat
+        dockerContext(filebeatOutput: "docker-${version}-${buildType}.log", metricbeatOutput: "docker-${version}-${buildType}-metricbeat.log", archiveOnlyOnFail: true){
+          dir("${BASE_DIR}"){
+            try {
+              retryWithSleep(retries: 2, seconds: 5, backoff: true) {
+                sh(label: "Run Tests", script: """.ci/scripts/test.sh -b "${buildType}" -t "${tav}" "${version}" """)
+              }
+            } finally {
+              junit(testResults: "test_output/*.junit.xml", allowEmptyResults: true, keepLongStdio: true)
+              archiveArtifacts(artifacts: "test_output/*.tap", allowEmptyArchive: true)
             }
-          } catch(e){
-            error(e.toString())
-          } finally {
-            junit(testResults: "test_output/*.junit.xml", allowEmptyResults: true, keepLongStdio: true)
-            archiveArtifacts(artifacts: "test_output/*.tap", allowEmptyArchive: true)
           }
         }
       }
@@ -516,15 +525,15 @@ def getSmartTAVContext() {
 
 def generateStepForWindows(Map params = [:]){
   def version = params?.version
-  def ELASTIC_APM_ASYNC_HOOKS = String.valueOf(!params.get('disableAsyncHooks', false))
+  def contextManager = params.get('disableAsyncHooks', false) ? 'patch' : ''
   return {
     sh label: 'Prepare services', script: ".ci/scripts/windows/prepare-test.sh ${version}"
     def linuxIp = grabWorkerIP()
     withNode(labels: 'windows-2019-docker-immutable', forceWorkspace: true, forceWorker: true) {
       // When installing with choco the PATH might not be updated within the already connected worker.
       withEnv(["PATH=${PATH};C:\\Program Files\\nodejs",
                "VERSION=${version}",
-               "ELASTIC_APM_ASYNC_HOOKS=${ELASTIC_APM_ASYNC_HOOKS}",
+               "ELASTIC_APM_CONTEXT_MANAGER=${contextManager}",
                "CASSANDRA_HOST=${linuxIp}",
                "ES_HOST=${linuxIp}",
                "LOCALSTACK_HOST=${linuxIp}",

.ci/Makefile

@@ -58,7 +58,7 @@ publish: validate-layer-name validate-aws-default-region
 		--layer-name "$(ELASTIC_LAYER_NAME)" \
 		--description "AWS Lambda Extension Layer for the Elastic APM Node.js Agent" \
 		--license "Apache-2.0" \
-		--compatible-runtimes nodejs16.x nodejs14.x nodejs12.x nodejs10.x \
+		--compatible-runtimes nodejs18.x nodejs16.x nodejs14.x nodejs12.x nodejs10.x \
 		--zip-file "fileb://./$(AWS_FOLDER)/elastic-apm-node-lambda-layer-$(BRANCH_NAME).zip"
 
 # Grant public access to the given LAYER in the given AWS region

.ci/docker/docker-compose-node-edge-test.yml

@@ -9,7 +9,7 @@ services:
         - NODE_VERSION=${NODE_VERSION}
         - NODE_FULL_VERSION=${NODE_FULL_VERSION}
         - NVM_NODEJS_ORG_MIRROR=${NVM_NODEJS_ORG_MIRROR}
-        - ELASTIC_APM_ASYNC_HOOKS=${ELASTIC_APM_ASYNC_HOOKS}
+        - ELASTIC_APM_CONTEXT_MANAGER=${ELASTIC_APM_CONTEXT_MANAGER}
     working_dir: /app
     command: >
       /bin/bash -c ".ci/scripts/docker-test.sh"
@@ -27,7 +27,7 @@ services:
       NODE_VERSION: ${NODE_VERSION}
       NODE_FULL_VERSION: ${NODE_FULL_VERSION}
       NVM_NODEJS_ORG_MIRROR: ${NVM_NODEJS_ORG_MIRROR}
-      ELASTIC_APM_ASYNC_HOOKS: ${ELASTIC_APM_ASYNC_HOOKS}
+      ELASTIC_APM_CONTEXT_MANAGER: ${ELASTIC_APM_CONTEXT_MANAGER}
       TAV: ${TAV_MODULE}
       HOME: /tmp
     volumes:

.ci/docker/docker-compose-node-test.yml

@@ -23,7 +23,7 @@ services:
       LOCALSTACK_HOST: 'localstack'
       NODE_VERSION: ${NODE_VERSION}
       TAV: ${TAV_MODULE}
-      ELASTIC_APM_ASYNC_HOOKS: ${ELASTIC_APM_ASYNC_HOOKS}
+      ELASTIC_APM_CONTEXT_MANAGER: ${ELASTIC_APM_CONTEXT_MANAGER}
       HOME: /tmp
       PATH: /app/node_modules/.bin:./node_modules/.bin:/app/node_modules:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
     volumes:

.ci/docker/docker-compose.yml

@@ -19,7 +19,7 @@ services:
       retries: 30
 
   mongodb:
-    image: mongo
+    image: mongo:5
     ports:
       - "27017:27017"
     volumes:

.ci/docker/node-edge-container/Dockerfile

@@ -14,11 +14,11 @@ RUN apt-get -qq update \
 ARG NODE_VERSION
 ARG NODE_FULL_VERSION
 ARG NVM_NODEJS_ORG_MIRROR
-ARG ELASTIC_APM_ASYNC_HOOKS
+ARG ELASTIC_APM_CONTEXT_MANAGER
 ENV NODE_VERSION=${NODE_VERSION}
 ENV NODE_FULL_VERSION=${NODE_FULL_VERSION}
 ENV NVM_NODEJS_ORG_MIRROR=${NVM_NODEJS_ORG_MIRROR}
-ENV ELASTIC_APM_ASYNC_HOOKS=${ELASTIC_APM_ASYNC_HOOKS}
+ENV ELASTIC_APM_CONTEXT_MANAGER=${ELASTIC_APM_CONTEXT_MANAGER}
 
 # nvm environment variables
 ENV NVM_VERSION v0.34.0

.ci/scripts/docker-test.sh

@@ -31,7 +31,7 @@ npm_install() {
   local retries=2
   local count=0
 
-  until npm install; do
+  until npm install --no-save; do
     exit=$?
     wait=$((2 ** $count))
     count=$(($count + 1))

.ci/scripts/test.sh

@@ -226,11 +226,9 @@ else
   DOCKER_COMPOSE_FILE=docker-compose-all.yml
 fi
 
-ELASTIC_APM_ASYNC_HOOKS=${ELASTIC_APM_ASYNC_HOOKS:-true}
-
 set +e
 NVM_NODEJS_ORG_MIRROR=${NVM_NODEJS_ORG_MIRROR} \
-ELASTIC_APM_ASYNC_HOOKS=${ELASTIC_APM_ASYNC_HOOKS} \
+ELASTIC_APM_CONTEXT_MANAGER=${ELASTIC_APM_CONTEXT_MANAGER} \
 NODE_VERSION=${NODE_VERSION} \
 NODE_FULL_VERSION=${NODE_FULL_VERSION} \
 TAV_MODULE=${TAV_MODULE} \
@@ -250,7 +248,7 @@ fi
 
 set -e
 NVM_NODEJS_ORG_MIRROR=${NVM_NODEJS_ORG_MIRROR} \
-ELASTIC_APM_ASYNC_HOOKS=${ELASTIC_APM_ASYNC_HOOKS} \
+ELASTIC_APM_CONTEXT_MANAGER=${ELASTIC_APM_CONTEXT_MANAGER} \
 NODE_VERSION=${NODE_VERSION} \
 NODE_FULL_VERSION=${NODE_FULL_VERSION} \
 TAV_MODULE=${TAV_MODULE} \

.ci/snapshoty.yml

@@ -0,0 +1,36 @@
+---
+
+# Version of configuration to use
+version: '1.0'
+
+# You can define a Google Cloud Account to use
+account:
+  # Project id of the service account
+  project: '${GCS_PROJECT}'
+  # Private key id of the service account
+  private_key_id: '${GCS_PRIVATE_KEY_ID}'
+  # Private key of the service account
+  private_key: '${GCS_PRIVATE_KEY}'
+  # Email of the service account
+  client_email: '${GCS_CLIENT_EMAIL}'
+  # URI token
+  token_uri: 'https://oauth2.googleapis.com/token'
+
+# List of artifacts
+artifacts:
+  # Path to use for artifacts discovery 
+  - path: './build/snapshot'
+    # Files pattern to match
+    files_pattern: 'elastic-apm-node-(?P<app_version>\d+\.\d+\.\d+)\.tgz'
+    # File layout on GCS bucket
+    output_pattern: '{project}/{jenkins_branch_name}/elastic-apm-node-{app_version}-{jenkins_git_commit_short}.tgz'
+    # List of metadata processors to use.
+    metadata:
+      # Define static custom metadata
+      - name: 'custom'
+        data:
+          project: 'apm-agent-nodejs'
+      # Add git metadata
+      - name: 'git'
+      # Add jenkins metadata
+      - name: 'jenkins'

.eslintrc.json

@@ -11,12 +11,18 @@
     "license-header/header": [ "error", "./dev-utils/license-header.js" ]
   },
   "ignorePatterns": [
+    "/*.example.js", // a pattern for uncommited local dev files to avoid linting
     "/.nyc_output",
     "/build",
     "node_modules",
+    "/examples/esbuild/dist",
+    "/examples/typescript/dist",
+    "/examples/nextjs",
     "/lib/opentelemetry-bridge/opentelemetry-core-mini",
     "/test/babel/out.js",
     "/test/lambda/fixtures/esbuild-bundled-handler/hello.js",
+    "/test/instrumentation/modules/next/a-nextjs-app/pages",
+    "/test/instrumentation/modules/next/a-nextjs-app/components",
     "/test/sourcemaps/fixtures/lib",
     "/test/sourcemaps/fixtures/src",
     "/test/stacktraces/fixtures/dist",