From 017ebd8254a2c1f4f05885afd77c6e5feadd6f90 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 8 Mar 2024 15:47:22 +0100 Subject: [PATCH 01/18] ci: run the release when merges to main --- .github/workflows/release.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a2a48b62b..266b7dda6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,6 +4,8 @@ on: push: tags: - "v*.*.*" + branches: + - main permissions: contents: read @@ -31,10 +33,16 @@ jobs: with: name: packages path: dist - - name: Upload + - name: Upload pipy + if: startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@2f6f737ca5f74c637829c0f5c3acd0e29ea5e8bf with: repository-url: https://upload.pypi.org/legacy/ + - name: Upload pipy test + if: github.ref == 'refs/heads/main' + uses: pypa/gh-action-pypi-publish@2f6f737ca5f74c637829c0f5c3acd0e29ea5e8bf + with: + repository-url: https://test.pypi.org/legacy/ build-distribution: uses: ./.github/workflows/build-distribution.yml @@ -59,6 +67,7 @@ jobs: name: build-distribution path: ./build - name: Publish lambda layers to AWS + if: startsWith(github.ref, 'refs/tags') run: | # Convert v1.2.3 to ver-1-2-3 VERSION=${GITHUB_REF_NAME/v/ver-} @@ -66,6 +75,7 @@ jobs: ELASTIC_LAYER_NAME="elastic-apm-python-${VERSION}" .ci/publish-aws.sh - uses: actions/upload-artifact@v4 + if: startsWith(github.ref, 'refs/tags') with: name: arn-file path: ".arn-file.md" @@ -106,6 +116,7 @@ jobs: ${{ steps.setup-docker.outputs.name }}:${{ steps.setup-docker.outputs.tag }} ${{ steps.setup-docker.outputs.name }}:latest - name: Docker push + if: startsWith(github.ref, 'refs/tags') run: |- docker push ${{ steps.setup-docker.outputs.name }}:${{ steps.setup-docker.outputs.tag }} docker push ${{ steps.setup-docker.outputs.name }}:latest @@ -115,6 +126,7 @@ jobs: contents: write needs: - publish-lambda-layers + if: startsWith(github.ref, 'refs/tags') runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -133,7 +145,7 @@ jobs: notify: runs-on: ubuntu-latest - if: always() + if: ${{ always() && startsWith(github.ref, 'refs/tags') }} needs: - publish-lambda-layers - publish-pypi From b7675a6bae64aca1985bc5d527182c78e8639764 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 8 Mar 2024 16:01:12 +0100 Subject: [PATCH 02/18] avoid wrong docker-tag name --- .github/workflows/release.yml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 266b7dda6..1bd27900e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -85,6 +85,8 @@ jobs: needs: - build-distribution runs-on: ubuntu-latest + env: + DOCKER_IMAGE_NAME: docker.elastic.co/observability/apm-agent-python steps: - uses: actions/checkout@v4 - uses: elastic/apm-pipeline-library/.github/actions/docker-login@current @@ -101,25 +103,29 @@ jobs: - id: setup-docker name: Set up docker variables run: |- - # version without v prefix (e.g. 1.2.3) - echo "tag=${GITHUB_REF_NAME/v/}" >> "${GITHUB_OUTPUT}" - echo "name=docker.elastic.co/observability/apm-agent-python" >> "${GITHUB_OUTPUT}" + if [ "${{ startsWith(github.ref, 'refs/tags') }}" == "false" ] ; then + # for testing purposes + echo "tag=test" >> "${GITHUB_OUTPUT}" + else + # version without v prefix (e.g. 1.2.3) + echo "tag=${GITHUB_REF_NAME/v/}" >> "${GITHUB_OUTPUT}" + fi - name: Docker build run: >- docker build - -t ${{ steps.setup-docker.outputs.name }}:${{ steps.setup-docker.outputs.tag }} + -t ${{ env.DOCKER_IMAGE_NAME }}:${{ steps.setup-docker.outputs.tag }} --build-arg AGENT_DIR=./build/dist/package/python . - name: Docker retag run: >- docker tag - ${{ steps.setup-docker.outputs.name }}:${{ steps.setup-docker.outputs.tag }} - ${{ steps.setup-docker.outputs.name }}:latest + ${{ env.DOCKER_IMAGE_NAME }}:${{ steps.setup-docker.outputs.tag }} + ${{ env.DOCKER_IMAGE_NAME }}:latest - name: Docker push if: startsWith(github.ref, 'refs/tags') run: |- - docker push ${{ steps.setup-docker.outputs.name }}:${{ steps.setup-docker.outputs.tag }} - docker push ${{ steps.setup-docker.outputs.name }}:latest + docker push ${{ env.DOCKER_IMAGE_NAME }}:${{ steps.setup-docker.outputs.tag }} + docker push ${{ env.DOCKER_IMAGE_NAME }}:latest github-draft: permissions: From 21dfc0f4d3b298f1af5ffb7ef35d9cecfd09ea42 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 8 Mar 2024 16:04:21 +0100 Subject: [PATCH 03/18] use the opposite --- .github/workflows/release.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1bd27900e..93de4b691 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,22 +6,18 @@ on: - "v*.*.*" branches: - main + - test/release permissions: contents: read jobs: - test: - uses: ./.github/workflows/test.yml - with: - full-matrix: true packages: uses: ./.github/workflows/packages.yml publish-pypi: needs: - - test - packages runs-on: ubuntu-latest environment: release @@ -39,7 +35,7 @@ jobs: with: repository-url: https://upload.pypi.org/legacy/ - name: Upload pipy test - if: github.ref == 'refs/heads/main' + if: ! startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@2f6f737ca5f74c637829c0f5c3acd0e29ea5e8bf with: repository-url: https://test.pypi.org/legacy/ From 65e0e301f9f0e6a6c6eeb2896ac37b85dbccfb3d Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 8 Mar 2024 16:06:02 +0100 Subject: [PATCH 04/18] fix yaml --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 93de4b691..4b8065352 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,7 +35,7 @@ jobs: with: repository-url: https://upload.pypi.org/legacy/ - name: Upload pipy test - if: ! startsWith(github.ref, 'refs/tags') + if: ${{ ! startsWith(github.ref, 'refs/tags') }} uses: pypa/gh-action-pypi-publish@2f6f737ca5f74c637829c0f5c3acd0e29ea5e8bf with: repository-url: https://test.pypi.org/legacy/ From d89c5a344928093f13e0a69d96a20f397439701f Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 8 Mar 2024 16:48:49 +0100 Subject: [PATCH 05/18] enable test, slack message conditional and fix names --- .github/workflows/release.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4b8065352..4f039b3aa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,11 +13,17 @@ permissions: jobs: + test: + uses: ./.github/workflows/test.yml + with: + full-matrix: true + packages: uses: ./.github/workflows/packages.yml publish-pypi: needs: + - test - packages runs-on: ubuntu-latest environment: release @@ -29,12 +35,12 @@ jobs: with: name: packages path: dist - - name: Upload pipy + - name: Upload pypi.org if: startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@2f6f737ca5f74c637829c0f5c3acd0e29ea5e8bf with: repository-url: https://upload.pypi.org/legacy/ - - name: Upload pipy test + - name: Upload test.pypi.org if: ${{ ! startsWith(github.ref, 'refs/tags') }} uses: pypa/gh-action-pypi-publish@2f6f737ca5f74c637829c0f5c3acd0e29ea5e8bf with: @@ -147,7 +153,7 @@ jobs: notify: runs-on: ubuntu-latest - if: ${{ always() && startsWith(github.ref, 'refs/tags') }} + if: ${{ always() }} needs: - publish-lambda-layers - publish-pypi @@ -159,6 +165,7 @@ jobs: with: needs: ${{ toJSON(needs) }} - uses: elastic/apm-pipeline-library/.github/actions/notify-build-status@current + if: startsWith(github.ref, 'refs/tags') with: status: ${{ steps.check.outputs.status }} vaultUrl: ${{ secrets.VAULT_ADDR }} From 3497c213d2d5edf6973fc2bc7aed512a6c856658 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 8 Mar 2024 16:49:14 +0100 Subject: [PATCH 06/18] Update .github/workflows/release.yml --- .github/workflows/release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4f039b3aa..849494c03 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,7 +6,6 @@ on: - "v*.*.*" branches: - main - - test/release permissions: contents: read From 3f76529820cfa980914a391ce2c31b86df635bd1 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 8 Mar 2024 16:49:27 +0100 Subject: [PATCH 07/18] Update .github/workflows/release.yml --- .github/workflows/release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 849494c03..8650f1291 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,7 +11,6 @@ permissions: contents: read jobs: - test: uses: ./.github/workflows/test.yml with: From a79d6055f22e4d5f5d69a68d4bf8846877ec0390 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 8 Mar 2024 16:49:46 +0100 Subject: [PATCH 08/18] Update .github/workflows/release.yml --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8650f1291..97bc933ab 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -151,7 +151,7 @@ jobs: notify: runs-on: ubuntu-latest - if: ${{ always() }} + if: always() needs: - publish-lambda-layers - publish-pypi From eed2e825593caad9cda5ea05971c36fa50c08ad4 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 11 Mar 2024 09:22:27 +0100 Subject: [PATCH 09/18] skip test for branches in the release workflow --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 97bc933ab..0fee4e905 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,6 +12,7 @@ permissions: jobs: test: + if: startsWith(github.ref, 'refs/tags') uses: ./.github/workflows/test.yml with: full-matrix: true From 054de934800ac2dbc9e1d37c8c8793c9cf3e9dd6 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 11 Mar 2024 09:31:12 +0100 Subject: [PATCH 10/18] workaround --- .github/workflows/release.yml | 4 ++-- .github/workflows/test-release.yml | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/test-release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0fee4e905..b9c3553c0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,10 +12,10 @@ permissions: jobs: test: - if: startsWith(github.ref, 'refs/tags') - uses: ./.github/workflows/test.yml + uses: ./.github/workflows/test-release.yml with: full-matrix: true + enabled: ${{ startsWith(github.ref, 'refs/tags') }} packages: uses: ./.github/workflows/packages.yml diff --git a/.github/workflows/test-release.yml b/.github/workflows/test-release.yml new file mode 100644 index 000000000..dd6ffe05b --- /dev/null +++ b/.github/workflows/test-release.yml @@ -0,0 +1,26 @@ +name: test-release + +on: + workflow_dispatch: + inputs: + full-matrix: + description: "Run the full matrix" + required: true + type: boolean + enabled: + description: "Whether to run the workfow" + required: true + type: boolean + +jobs: + test: + if: ${{ inputs.enabled }} + uses: ./.github/workflows/test.yml + with: + full-matrix: ${{ inputs.full-matrix }} + + run-if-disabled: + if: ${{ ! inputs.enabled }} + runs-on: ubuntu-latest + steps: + - run: echo "do something to help with the reusable workflows with needs" From 79bbc4b860ac8112db97a5abc24ca5755add9328 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 11 Mar 2024 09:32:45 +0100 Subject: [PATCH 11/18] fix --- .github/workflows/test-release.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/test-release.yml b/.github/workflows/test-release.yml index dd6ffe05b..c873f9eb7 100644 --- a/.github/workflows/test-release.yml +++ b/.github/workflows/test-release.yml @@ -1,6 +1,20 @@ name: test-release on: + workflow_call: + inputs: + full-matrix: + description: "Run the full matrix" + required: true + type: boolean + ref: + description: "The git ref of elastic/apm-agent-python to run test workflow from." + required: false + type: string + enabled: + description: "Whether to run the workfow" + required: true + type: boolean workflow_dispatch: inputs: full-matrix: From b6e03ab8b1cd95cdef3c46d6b0afcb0223064cba Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 11 Mar 2024 16:42:02 +0100 Subject: [PATCH 12/18] Update .github/workflows/release.yml --- .github/workflows/release.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b9c3553c0..58b9446bd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,6 +10,10 @@ on: permissions: contents: read +# Override the version if there is no tag release. +env: + ELASTIC_CI_LOCAL_VERSION: ${{ startsWith(github.ref, 'refs/tags') && '' || github.run_id }} + jobs: test: uses: ./.github/workflows/test-release.yml From e98595e17c394fbe0d4bc4f50a53cc00a435941f Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 11 Mar 2024 17:21:11 +0100 Subject: [PATCH 13/18] support releases in test.pypi for commits on main --- setup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 23ebec33e..235bf4d81 100644 --- a/setup.py +++ b/setup.py @@ -100,7 +100,11 @@ def get_version(): for line in version_file: if line.startswith("__version__"): version_tuple = ast.literal_eval(line.split(" = ")[1]) - return ".".join(map(str, version_tuple)) + version_str = ".".join(map(str, version_tuple)) + local_version = os.getenv("ELASTIC_CI_LOCAL_VERSION") + if local_version: + return f"{version_str}+{local_version}" + return version_str return "unknown" From 59034cc224fd18caa620d6f33e86e905f5683879 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 11 Mar 2024 17:28:02 +0100 Subject: [PATCH 14/18] set environment variable in the reusable workflow instead env propagation between reusable workflow is not supported --- .github/workflows/packages.yml | 4 ++++ .github/workflows/release.yml | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index af485c455..70bdd3165 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -13,6 +13,10 @@ on: - '**/*.md' - '**/*.asciidoc' +# Override the version if there is no tag release. +env: + ELASTIC_CI_LOCAL_VERSION: ${{ startsWith(github.ref, 'refs/tags') && '' || github.run_id }} + jobs: build: runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 58b9446bd..b9c3553c0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,10 +10,6 @@ on: permissions: contents: read -# Override the version if there is no tag release. -env: - ELASTIC_CI_LOCAL_VERSION: ${{ startsWith(github.ref, 'refs/tags') && '' || github.run_id }} - jobs: test: uses: ./.github/workflows/test-release.yml From af5c2c9f4d917d48e762f461e01717c7ed39c20b Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 11 Mar 2024 17:39:31 +0100 Subject: [PATCH 15/18] see https://github.com/pypi/legacy/issues/731 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 235bf4d81..d1a8c8007 100644 --- a/setup.py +++ b/setup.py @@ -103,7 +103,7 @@ def get_version(): version_str = ".".join(map(str, version_tuple)) local_version = os.getenv("ELASTIC_CI_LOCAL_VERSION") if local_version: - return f"{version_str}+{local_version}" + return f"{version_str}-dev{local_version}" return version_str return "unknown" From f248801e93a609c772f69e91c3cd59fcd8776696 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Mon, 11 Mar 2024 17:03:42 +0100 Subject: [PATCH 16/18] setup: take a post version from environment variable Make it possible to add a post version (per PEP-440) to the current agent version at build time. This will be useful because we want to continuously build the agent on CI and avoid version conflicts when publishing to test pypi. So when returning the agent version add anything that has been passed in ELASTIC_CI_POST_VERSION environment variable as post version. Refs #1994 --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index d1a8c8007..a88cdeb5b 100644 --- a/setup.py +++ b/setup.py @@ -101,9 +101,9 @@ def get_version(): if line.startswith("__version__"): version_tuple = ast.literal_eval(line.split(" = ")[1]) version_str = ".".join(map(str, version_tuple)) - local_version = os.getenv("ELASTIC_CI_LOCAL_VERSION") - if local_version: - return f"{version_str}-dev{local_version}" + post_version = os.getenv("ELASTIC_CI_POST_VERSION") + if post_version: + return f"{version_str}.post{post_version}" return version_str return "unknown" From 86be9eeb2372d8a9e2a1cfef96d1147dc4582d54 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 11 Mar 2024 18:06:48 +0100 Subject: [PATCH 17/18] fix --- .github/workflows/packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 70bdd3165..f2d9a4f83 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -15,7 +15,7 @@ on: # Override the version if there is no tag release. env: - ELASTIC_CI_LOCAL_VERSION: ${{ startsWith(github.ref, 'refs/tags') && '' || github.run_id }} + ELASTIC_CI_POST_VERSION: ${{ startsWith(github.ref, 'refs/tags') && '' || github.run_id }} jobs: build: From a55eab2549abe49d3955fc8c2b2d17c5bd1a3c26 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 13 Mar 2024 14:47:00 +0100 Subject: [PATCH 18/18] Update .github/workflows/release.yml --- .github/workflows/release.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b9c3553c0..b1d0dff48 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -125,8 +125,7 @@ jobs: - name: Docker push if: startsWith(github.ref, 'refs/tags') run: |- - docker push ${{ env.DOCKER_IMAGE_NAME }}:${{ steps.setup-docker.outputs.tag }} - docker push ${{ env.DOCKER_IMAGE_NAME }}:latest + docker push --all-tags ${{ env.DOCKER_IMAGE_NAME }} github-draft: permissions: