Merge remote-tracking branch 'upstream/master' into feature/jjbb-upda…

…te-schema-json-1

* upstream/master: (111 commits)
  Introduce metricset.name (elastic#4857)
  processor/otel: test service.version handling (elastic#4853)
  docs: Add PHP agent information to shared docs (elastic#4740)
  Script for faster development workflow (elastic#4731)
  Update to elastic/beats@1b31c26 (elastic#4763)
  backport: add 7.12 to .backportrc.json (elastic#4807)
  backport: enable auto-merge on backport PRs (elastic#4777)
  Support for Node.js profiles (elastic#4728)
  docs: readds .NET link (elastic#4764)
  [DOCS] Fixes URLs on Secure communication with APM Agents page (elastic#4746)
  ci: set proper parameters for the tar step (elastic#4696)
  docs: add 7.11.1 release notes (elastic#4727)
  Disable sourcemap upload endpoint when data streams enabled (elastic#4735)
  Add service name to dataset field (elastic#4674)
  Update to elastic/beats@ba423212a660 (elastic#4733)
  sampling: require a default policy (elastic#4729)
  processor/otel: add unit test for span status (elastic#4734)
  Add support for consuming OTLP/gRPC metrics (elastic#4722)
  [apmpackage] Add config options supported in ESS (elastic#4690)
  Use the apm-server version everywhere* (elastic#4725)
  ...

.backportrc.json

@@ -1,5 +1,7 @@
 {
   "upstream": "elastic/apm-server",
-  "branches": [{ "name": "7.x", "checked": true }, "7.10", "7.9", "6.8"],
-  "labels": ["backport"]
+  "branches": [{ "name": "7.x", "checked": true }, "7.12", "7.11", "7.10", "7.9", "6.8"],
+  "labels": ["backport"],
+  "autoMerge": true,
+  "autoMergeMethod": "squash"
 }

.ci/docker/Makefile

@@ -45,6 +45,7 @@ test-golang-mage: prepare-test ## Run the tests for the specific app
 	cp -f ../../go.* golang-mage
 	mkdir -p golang-mage/approvaltest && cp -f ../../approvaltest/go.* golang-mage/approvaltest
 	mkdir -p golang-mage/systemtest && cp -f ../../systemtest/go.* golang-mage/systemtest
+	mkdir -p golang-mage/internal/otel_collector && cp -f ../../internal/otel_collector/go.* golang-mage/internal/otel_collector
 	@DOCKERFILE=golang-mage bats-core/bin/bats --tap tests | tee target/results.tap
 	@$(MAKE) -s convert-tests-results
 

.ci/docker/golang-mage/Dockerfile

@@ -7,10 +7,12 @@ WORKDIR $TOOLS
 COPY go.mod go.sum ./
 COPY approvaltest/go.mod approvaltest/go.sum ./approvaltest/
 COPY systemtest/go.mod systemtest/go.sum ./systemtest/
+COPY internal/otel_collector/go.mod internal/otel_collector/go.sum ./internal/otel_collector/
 
 RUN go mod download
 RUN cd approvaltest && go mod download
 RUN cd systemtest && go mod download
+RUN cd internal/otel_collector && go mod download
 
 RUN apt-get update -y -qq \
   && apt-get install -y -qq  python3 python3-pip python3-venv \

.ci/scripts/docker-get-logs.sh

@@ -3,7 +3,7 @@ set -euo pipefail
 
 STEP=${1:-""}
 
-DOCKER_INFO_DIR="docker-info/${STEP}"
+DOCKER_INFO_DIR="build/docker-info/${STEP}"
 mkdir -p ${DOCKER_INFO_DIR}
 cp docker-compose*.yml ${DOCKER_INFO_DIR}
 cd ${DOCKER_INFO_DIR}

.github/pull_request_template.md

@@ -3,56 +3,48 @@
 If this is your first contribution, please review and sign our contributor agreement -
 https://www.elastic.co/contributor-agreement.
 
-Before creating the PR, ensure that:
-
-1. Your branch is rebased on top of the latest master.
-   Squash your initial commits into meaningful commits.
-   After creating a PR, do not rebase of force push any longer.
-2. Nothing is broken, by running the test suite (at least unit tests).
-   See https://github.com/elastic/apm-server/blob/master/TESTING.md for details.
-3. Your code follows the style guidelines of this project:
-   run `make check-full` for static code checks and linting.
-
-A few suggestions about filling out this PR:
-
-1. Use a descriptive title for the PR.
-2. If this pull request is work in progress, create a draft PR.
-3. Please label this PR with at least one of the following labels:
-   - bug fix
-   - breaking change
-   - enhancement
-4. Reference the related issue, and make use of magic keywords where it makes sense
-   https://help.github.com/articles/closing-issues-using-keywords/.
-5. Do not remove any checklist items, strike through the ones that don't apply
-   (by using tildes, e.g. ~scratch this ~).
-6. Explain how this PR can be tested by the reviewer: commands, dependencies, steps, etc.
-7. Submit the pull request:
-   Push your changes to your forked copy of the repository and submit a pull request
-   (https://help.github.com/articles/using-pull-requests).
-8. Please be patient. We might not be able to immediately review your code,
-   but we'll do our best to dedicate to it the attention it deserves.
-   Your effort is much appreciated!
+Guidelines:
+ - Prefer small PRs, and split changes into multiple logical commits where they must
+   be delivered in a single PR.
+ - If the PR is incomplete and not yet ready for review, open it as a Draft.
+ - Once the PR is marked ready for review it is expected to pass all tests and linting,
+   and you should not force-push any changes.
 
 See also https://github.com/elastic/apm-server/blob/master/CONTRIBUTING.md for more tips on contributing.
 -->
 
 ## Motivation/summary
 
+<!--
+Describe your change in the title and description, and provide a motivation for the
+change and rationale for the approach taken.
+-->
+
 ## Checklist
 
-- [ ] I have signed the [Contributor License Agreement](https://www.elastic.co/contributor-agreement/).
-- [ ] I have updated [CHANGELOG.asciidoc](https://github.com/elastic/apm-server/blob/master/CHANGELOG.asciidoc)
+<!--
+Delete irrelevant items. The changelog should only be updated for user-facing changes.
+Once the PR is ready for review there should be no unticked boxes.
+-->
+
+- [ ] Update [CHANGELOG.asciidoc](https://github.com/elastic/apm-server/blob/master/CHANGELOG.asciidoc)
+- [ ] Documentation has been updated
 
-I have considered changes for:
-- [ ] documentation
-- [ ] logging (add log lines, choose appropriate log selector, etc.)
-- [ ] metrics and monitoring (create issue for Kibana team to add metrics to visualizations, e.g. [Kibana#44001](https://github.com/elastic/kibana/issues/44001))
-- [ ] automated tests (add tests for the code changes, all [**unit** tests](https://github.com/elastic/apm-server/blob/master/TESTING.md) pass locally)
-- [ ] telemetry
-- [ ] Elasticsearch Service (https://cloud.elastic.co)
-- [ ] Elastic Cloud Enterprise (https://www.elastic.co/products/ece)
-- [ ] Elastic Cloud on Kubernetes (https://www.elastic.co/elastic-cloud-kubernetes)
+For functional changes, consider:
+- Is it observable through the addition of either **logging** or **metrics**?
+- Is its use being published in **telemetry** to enable product improvement?
+- Have system tests been added to avoid regression?
 
 ## How to test these changes
 
+<!--
+Explain how this PR can be tested by the reviewer: commands, dependencies, steps, etc.
+If it is self-explanatory, delete this section.
+-->
+
 ## Related issues
+
+<!--
+Reference the related issue(s), and make use of magic keywords where it makes sense
+https://help.github.com/articles/closing-issues-using-keywords/.
+-->

.gitignore

@@ -4,6 +4,7 @@
 /.ci/docker/golang-mage/go.*
 /.ci/docker/golang-mage/approvaltest/
 /.ci/docker/golang-mage/systemtest/
+/.ci/docker/golang-mage/internal/otel_collector/
 **/*.idea
 /build
 /data

.go-version

@@ -1 +1 @@
-1.14.7
+1.15.8

CHANGELOG.asciidoc

@@ -1,4 +1,5 @@
 include::./changelogs/head.asciidoc[]
+include::./changelogs/7.11.asciidoc[]
 include::./changelogs/7.10.asciidoc[]
 include::./changelogs/7.9.asciidoc[]
 include::./changelogs/7.8.asciidoc[]

CONTRIBUTING.md

@@ -1,25 +1,32 @@
 # Contributing to the APM Server
 
-The APM Server is open source and we love to receive contributions from our community — you!
+APM Server is open source, and we love to receive contributions from our community — you!
 
 There are many ways to contribute,
 from writing tutorials or blog posts,
 improving the documentation,
-submitting bug reports and feature requests or writing code.
+submitting bug reports and feature requests, or writing code.
 
-You can get in touch with us through [Discuss](https://discuss.elastic.co/c/apm),
+If you want to be rewarded for your contributions, sign up for the
+[Elastic Contributor Program](https://www.elastic.co/community/contributor).
+Each time you make a valid contribution, you’ll earn points that increase your chances of winning prizes and being recognized as a top contributor.
+
+## Questions
+
+GitHub is reserved for bug reports and feature requests; it is not the place
+for general questions. If you have a question or an unconfirmed bug, please
+visit our [discussion forum](https://discuss.elastic.co/c/apm);
 feedback and ideas are always welcome.
 
 ## Code contributions
 
-If you have a bugfix or new feature that you would like to contribute,
-please find or open an issue about it first.
-Talk about what you would like to do.
-It may be that somebody is already working on it,
-or that there are particular issues that you should know about before implementing the change.
+If you have a bug fix or new feature that you would like to contribute,
+please find or open an issue first.
+It's important to talk about what you would like to do,
+as there may already be someone working on it,
+or there may be context to be aware of before implementing the change.
 
-You will have to fork the `apm-server` repo,
-please follow the instructions in the [readme](README.md).
+Development instructions are available in the project [readme](README.md#apm-server-development).
 
 ### Submitting your changes
 
@@ -41,9 +48,9 @@ a committer will tag the pull request with the target version(s).
 Once a version is released,
 new features are frozen for that minor version and will not be backported.
 For example,
-if 6.2 was just released,
-the soonest a new feature will be released is 6.3,
-not 6.2.1.
+if 7.10 was just released,
+the soonest a new feature will be released is 7.11,
+not 7.10.1.
 Breaking changes may need to wait until the next major version.
 See [semver](https://semver.org/) for general information about major/minor versions.
 Bug fixes may be backported on a case by case basis.
@@ -56,3 +63,9 @@ and merged with "Create a merge commit".
 Straightforward backports may be merged without review.
 
 [Backport](https://github.com/sqren/backport) is recommended for automating the backport process.
+
+### Examples
+
+This is a collection of example PRs for additions occuring somewhat frequently.
+
+* [Adding a new field to the Intake API and index it in Elasticsearch](https://github.com/elastic/apm-server/pull/4626#issue-555484976)

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.14.7
+FROM golang:1.15.8
 MAINTAINER Nicolas Ruflin <ruflin@elastic.co>
 
 RUN set -x && \

Jenkinsfile

@@ -251,13 +251,15 @@ pipeline {
           }
           post {
             always {
-              coverageReport("${BASE_DIR}/build/coverage")
-              junit(allowEmptyResults: true,
-                keepLongStdio: true,
-                testResults: "${BASE_DIR}/build/junit-*.xml"
-              )
-              catchError(buildResult: 'SUCCESS', message: 'Failed to grab test results tar files', stageResult: 'SUCCESS') {
-                tar(file: "coverage-files.tgz", archive: true, dir: "coverage", pathPrefix: "${BASE_DIR}/build")
+              dir("${BASE_DIR}/build"){
+                coverageReport("coverage")
+                junit(allowEmptyResults: true,
+                  keepLongStdio: true,
+                  testResults: "junit-*.xml"
+                )
+                catchError(buildResult: 'SUCCESS', message: 'Failed to grab test results tar files', stageResult: 'SUCCESS') {
+                  tar(file: "coverage-files.tgz", archive: true, dir: "coverage")
+                }
               }
               codecov(repo: env.REPO, basedir: "${BASE_DIR}", secret: "${CODECOV_SECRET}")
             }
@@ -293,17 +295,18 @@ pipeline {
           }
           post {
             always {
-              dir("${BASE_DIR}"){
+              dir("${BASE_DIR}/build"){
                 archiveArtifacts(allowEmptyArchive: true,
                   artifacts: "docker-info/**",
-                  defaultExcludes: false)
-                  junit(allowEmptyResults: true,
-                    keepLongStdio: true,
-                    testResults: "**/build/TEST-*.xml"
-                  )
-              }
-              catchError(buildResult: 'SUCCESS', message: 'Failed to grab test results tar files', stageResult: 'SUCCESS') {
-                tar(file: "system-tests-linux-files.tgz", archive: true, dir: "system-tests", pathPrefix: "${BASE_DIR}/build")
+                  defaultExcludes: false
+                )
+                junit(allowEmptyResults: true,
+                  keepLongStdio: true,
+                  testResults: "**/TEST-*.xml"
+                )
+                catchError(buildResult: 'SUCCESS', message: 'Failed to grab test results tar files', stageResult: 'SUCCESS') {
+                  tar(file: "system-tests-linux-files.tgz", archive: true, dir: "system-tests")
+                }
               }
             }
           }
@@ -336,9 +339,9 @@ pipeline {
               golang(){
                 dir("${BASE_DIR}"){
                   sh(label: 'Run benchmarks', script: './.ci/scripts/bench.sh')
-                  sendBenchmarks(file: 'bench.out', index: "benchmark-server")
                 }
               }
+              sendBenchmarks(file: "${BASE_DIR}/bench.out", index: "benchmark-server")
             }
           }
         }