From 0670fe98c93da390a2a27b48568e5e62b8bc3fd2 Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Mon, 5 May 2025 17:28:21 +0200 Subject: [PATCH 1/6] Add link-index-updater lambda build job And reuse it in the release workflow --- .../build-link-index-updater-lambda.yml | 51 +++++++++++++++++++ .github/workflows/release.yml | 40 ++++++++------- 2 files changed, 73 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/build-link-index-updater-lambda.yml diff --git a/.github/workflows/build-link-index-updater-lambda.yml b/.github/workflows/build-link-index-updater-lambda.yml new file mode 100644 index 000000000..21b073419 --- /dev/null +++ b/.github/workflows/build-link-index-updater-lambda.yml @@ -0,0 +1,51 @@ +--- +# This workflow is used to build the link index updater lambda function zip +# that can be deployed to AWS Lambda. +name: Build Link Index Updater Lambda + +on: + workflow_dispatch: + workflow_call: + inputs: + ref: + required: false + type: string + outputs: + artifact-id: + description: 'Artifact ID of the uploaded artifact. Can be used to download the artifact in other workflows.' + value: ${{ jobs.build.outputs.artifact-id }} + artifact-path: + description: 'Path to the zip file.' + value: ${{ jobs.build.outputs.zip-file }} + +jobs: + build: + outputs: + artifact-id: ${{ steps.upload-artifact.outputs.artifact-id }} + zip-file: ${{ steps.create-zip.outputs.zip-file }} + runs-on: ubuntu-latest + env: + ZIP_FILE: link-index-updater-lambda.zip + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + - name: Amazon Linux 2023 build + run: | + docker build . -t publish-links-index:latest -f src/infra/docs-lambda-index-publisher/lambda.DockerFile + - name: Get bootstrap binary + run: | + docker cp $(docker create --name tc publish-links-index:latest):/app/.artifacts/publish ./.artifacts && docker rm tc + - name: Create zip file + id: create-zip + run: | + zip -j "${ZIP_FILE}" .artifacts/docs-lambda-index-publisher/release_linux-x64/bootstrap + echo "zip-file=${ZIP_FILE}" >> "${GITHUB_OUTPUT}" + - name: Archive artifact + id: upload-artifact + uses: actions/upload-artifact@v4 + with: + name: link-index-updater-lambda + retention-days: 7 + if-no-files-found: error + path: ${{ env.ZIP_FILE }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 14712c920..9c90912a2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,6 +44,8 @@ jobs: }) containers: + needs: + - release-drafter runs-on: ubuntu-latest outputs: full-version: ${{ steps.bootstrap.outputs.full-version }} @@ -52,7 +54,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: ${{ steps.release-drafter.outputs.tag_name }} + ref: ${{ needs.release-drafter.outputs.tag_name }} - name: Bootstrap Action Workspace id: bootstrap uses: ./.github/actions/bootstrap @@ -65,38 +67,40 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Publish Containers - run: ./build.sh publishcontainers + run: ./build.sh publishcontainers\ + + build-lambda: + needs: + - release-drafter + uses: ./.github/workflows/build-link-index-updater-lambda.yml + with: + ref: ${{ needs.release-drafter.outputs.tag_name }} release-lambda: environment: name: link-index-updater-prod runs-on: ubuntu-latest needs: + - build-lambda - release-drafter permissions: contents: write id-token: write env: - ZIP_FILE: link-index-updater-lambda.zip + ZIP_FILE: ${{ needs.build-lambda.outputs.artifact-path }} steps: - - uses: actions/checkout@v4 - with: - ref: ${{ steps.release-drafter.outputs.tag_name }} - - name: Amazon Linux 2023 build - run: | - docker build . -t publish-links-index:latest -f src/infra/docs-lambda-index-publisher/lambda.DockerFile - - name: Get bootstrap binary - run: | - docker cp $(docker create --name tc publish-links-index:latest):/app/.artifacts/publish ./.artifacts && docker rm tc - + + - uses: actions/download-artifact@v4 + with: + artifact-ids: ${{ needs.build-lambda.outputs.artifact-id }} + - uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0 with: role-to-assume: arn:aws:iam::197730964718:role/elastic-docs-v3-link-index-updater-deployer aws-region: us-east-2 - + - name: Upload Lambda function - run: | - zip -j "${ZIP_FILE}" .artifacts/docs-lambda-index-publisher/release_linux-x64/bootstrap + run: | aws lambda update-function-code \ --function-name elastic-docs-v3-link-index-updater \ --zip-file "fileb://${ZIP_FILE}" @@ -104,7 +108,7 @@ jobs: - name: Attach Distribution to release env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: gh release upload ${{ needs.release-drafter.outputs.tag_name }} "${ZIP_FILE}" + run: gh release upload --repo ${{ github.repository }} ${{ needs.release-drafter.outputs.tag_name }} "${ZIP_FILE}" release: needs: @@ -126,7 +130,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: ${{ steps.release-drafter.outputs.tag_name }} + ref: ${{ needs.release-drafter.outputs.tag_name }} - name: Bootstrap Action Workspace id: bootstrap From 123614b52efd61aa556cfb7e0cdb5c545027f84b Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Mon, 5 May 2025 22:00:00 +0200 Subject: [PATCH 2/6] Also re-use in PR --- .github/workflows/build-link-index-updater-lambda.yml | 5 +++++ .github/workflows/pr.yml | 1 + 2 files changed, 6 insertions(+) diff --git a/.github/workflows/build-link-index-updater-lambda.yml b/.github/workflows/build-link-index-updater-lambda.yml index 21b073419..b5598d384 100644 --- a/.github/workflows/build-link-index-updater-lambda.yml +++ b/.github/workflows/build-link-index-updater-lambda.yml @@ -10,6 +10,7 @@ on: ref: required: false type: string + default: ${{ github.ref }} outputs: artifact-id: description: 'Artifact ID of the uploaded artifact. Can be used to download the artifact in other workflows.' @@ -36,6 +37,10 @@ jobs: - name: Get bootstrap binary run: | docker cp $(docker create --name tc publish-links-index:latest):/app/.artifacts/publish ./.artifacts && docker rm tc + - name: Validate bootstrap binary + run: | + tree .artifacts + stat .artifacts/docs-lambda-index-publisher/release_linux-x64/bootstrap - name: Create zip file id: create-zip run: | diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index c0a9b88af..19844c7e6 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -27,6 +27,7 @@ jobs: run: dotnet run --project src/tooling/docs-assembler -c release -- navigation validate build-lambda: + uses: ./.github/workflows/build-link-index-updater-lambda.yml runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From e7c57d72a85d7bd42103b095a0fec3b8ae837fc3 Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Mon, 5 May 2025 22:01:12 +0200 Subject: [PATCH 3/6] Fix --- .github/workflows/pr.yml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 19844c7e6..63730c37c 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -28,18 +28,7 @@ jobs: build-lambda: uses: ./.github/workflows/build-link-index-updater-lambda.yml - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Amazon Linux 2023 build - run: | - docker build . -t publish-links-index:latest -f src/infra/docs-lambda-index-publisher/lambda.DockerFile - - name: Get bootstrap binary - run: | - docker cp $(docker create --name tc publish-links-index:latest):/app/.artifacts/publish ./.artifacts && docker rm tc - tree .artifacts - stat .artifacts/docs-lambda-index-publisher/release_linux-x64/bootstrap - + lint: runs-on: ubuntu-latest defaults: From 7affbcc81dd572937701eca4f75f59a6e825a427 Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Mon, 5 May 2025 22:03:11 +0200 Subject: [PATCH 4/6] Decrease retention --- .github/workflows/build-link-index-updater-lambda.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-link-index-updater-lambda.yml b/.github/workflows/build-link-index-updater-lambda.yml index b5598d384..59edbd538 100644 --- a/.github/workflows/build-link-index-updater-lambda.yml +++ b/.github/workflows/build-link-index-updater-lambda.yml @@ -37,7 +37,7 @@ jobs: - name: Get bootstrap binary run: | docker cp $(docker create --name tc publish-links-index:latest):/app/.artifacts/publish ./.artifacts && docker rm tc - - name: Validate bootstrap binary + - name: Inspect bootstrap binary run: | tree .artifacts stat .artifacts/docs-lambda-index-publisher/release_linux-x64/bootstrap @@ -51,6 +51,6 @@ jobs: uses: actions/upload-artifact@v4 with: name: link-index-updater-lambda - retention-days: 7 + retention-days: 1 if-no-files-found: error path: ${{ env.ZIP_FILE }} From e86d2a68c2da1b26fefd3a081cbd398e275fd8bc Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Mon, 5 May 2025 22:39:40 +0200 Subject: [PATCH 5/6] Refactor --- .../build-link-index-updater-lambda.yml | 22 ++++++------------- .github/workflows/release.yml | 8 +++++-- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build-link-index-updater-lambda.yml b/.github/workflows/build-link-index-updater-lambda.yml index 59edbd538..ebc3ea51e 100644 --- a/.github/workflows/build-link-index-updater-lambda.yml +++ b/.github/workflows/build-link-index-updater-lambda.yml @@ -1,6 +1,6 @@ --- -# This workflow is used to build the link index updater lambda function zip -# that can be deployed to AWS Lambda. +# This workflow is used to build the link-index-updater +# lambda function bootstrap binary that can be deployed to AWS Lambda. name: Build Link Index Updater Lambda on: @@ -15,10 +15,7 @@ on: artifact-id: description: 'Artifact ID of the uploaded artifact. Can be used to download the artifact in other workflows.' value: ${{ jobs.build.outputs.artifact-id }} - artifact-path: - description: 'Path to the zip file.' - value: ${{ jobs.build.outputs.zip-file }} - + jobs: build: outputs: @@ -26,7 +23,7 @@ jobs: zip-file: ${{ steps.create-zip.outputs.zip-file }} runs-on: ubuntu-latest env: - ZIP_FILE: link-index-updater-lambda.zip + BINARY_PATH: .artifacts/docs-lambda-index-publisher/release_linux-x64/bootstrap steps: - uses: actions/checkout@v4 with: @@ -40,17 +37,12 @@ jobs: - name: Inspect bootstrap binary run: | tree .artifacts - stat .artifacts/docs-lambda-index-publisher/release_linux-x64/bootstrap - - name: Create zip file - id: create-zip - run: | - zip -j "${ZIP_FILE}" .artifacts/docs-lambda-index-publisher/release_linux-x64/bootstrap - echo "zip-file=${ZIP_FILE}" >> "${GITHUB_OUTPUT}" + stat "${BINARY_PATH}" - name: Archive artifact id: upload-artifact uses: actions/upload-artifact@v4 with: - name: link-index-updater-lambda + name: link-index-updater-lambda-binary retention-days: 1 if-no-files-found: error - path: ${{ env.ZIP_FILE }} + path: ${{ env.BINARY_PATH }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9c90912a2..cc360ab47 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -87,18 +87,22 @@ jobs: contents: write id-token: write env: - ZIP_FILE: ${{ needs.build-lambda.outputs.artifact-path }} + ZIP_FILE: link-index-updater-lambda.zip steps: - uses: actions/download-artifact@v4 with: artifact-ids: ${{ needs.build-lambda.outputs.artifact-id }} + - name: Create zip + run: | + zip -j "${ZIP_FILE}" ./bootstrap + - uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0 with: role-to-assume: arn:aws:iam::197730964718:role/elastic-docs-v3-link-index-updater-deployer aws-region: us-east-2 - + - name: Upload Lambda function run: | aws lambda update-function-code \ From 4a266cc535aaa0418bb1426e8d7bdd968439f291 Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Mon, 5 May 2025 22:51:28 +0200 Subject: [PATCH 6/6] Remove `_modified` and check It's not concurrency safe and doesn't work as expected. It's not worth the hassle --- src/infra/docs-lambda-index-publisher/LinkIndexProvider.cs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/infra/docs-lambda-index-publisher/LinkIndexProvider.cs b/src/infra/docs-lambda-index-publisher/LinkIndexProvider.cs index b52a473bd..d6965ebc7 100644 --- a/src/infra/docs-lambda-index-publisher/LinkIndexProvider.cs +++ b/src/infra/docs-lambda-index-publisher/LinkIndexProvider.cs @@ -19,7 +19,6 @@ public class LinkIndexProvider(IAmazonS3 s3Client, ILambdaLogger logger, string { private string? _etag; private LinkReferenceRegistry? _linkIndex; - private bool _modified; private async Task GetLinkIndex() { @@ -57,18 +56,12 @@ public async Task UpdateLinkIndexEntry(LinkRegistryEntry linkRegistryEntry) { { linkRegistryEntry.Branch, linkRegistryEntry } }); - _modified = true; logger.LogInformation("Added new entry for {repository}@{branch}", linkRegistryEntry.Repository, linkRegistryEntry.Branch); } } public async Task Save() { - if (!_modified) - { - logger.LogInformation("Skipping Save() because the link index was not modified"); - return; - } if (_etag == null || _linkIndex == null) throw new InvalidOperationException("You must call UpdateLinkIndexEntry() before Save()"); var json = LinkReferenceRegistry.Serialize(_linkIndex);