From 85e5e71adb108e5cfe5a18ca04030e95e5531304 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 24 Nov 2020 14:15:06 -0500 Subject: [PATCH 01/12] Convert lambda Python version to argument Switch lambda building to use the Python version given as a build argument. Change the build script to accept an argument to use the correct major.minor version from the lambci imagine being used. --- .github/workflows/build.yml | 20 ++++++++++++-------- .github/workflows/release.yml | 9 +++++---- Dockerfile | 5 +++-- build.sh | 25 ++++++++++++++++--------- 4 files changed, 36 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0899a15..75a8274 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -115,29 +115,34 @@ jobs: build_lambda: runs-on: ubuntu-latest needs: build + strategy: + matrix: + # Versions supported by AWS and lambci/lambda images + python-version: [3.6, 3.7, 3.8] steps: - uses: actions/checkout@v2 - - id: setup-python - uses: actions/setup-python@v2 + - uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: ${{ matrix.python-version }} - name: Cache lambda building environments uses: actions/cache@v2 with: path: ${{ env.PIP_CACHE_DIR }} key: "build_lambda-${{ runner.os }}-\ - py${{ steps.setup-python.outputs.python-version }}-\ + py${{ matrix.python-version }}-\ ${{ hashFiles('**/requirements.txt') }}" restore-keys: | build_lambda-${{ runner.os }}-\ - py${{ steps.setup-python.outputs.python-version }}- + py${{ matrix.python-version }}- build_lambda-${{ runner.os }}- - name: Install dependencies run: | python -m pip install --upgrade pip wheel pip install --upgrade --requirement requirements.txt - name: Build environment - run: docker-compose build + run: | + docker-compose build \ + --build-arg PY_VERSION=${{ matrix.python-version }} - name: Generate lambda zip run: docker-compose up # We have to unzip the produced lambda zip because of limitations with the @@ -152,6 +157,5 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v2 with: - name: "skeleton-aws-lambda-\ - py${{ steps.setup-python.outputs.python-version }}" + name: skeleton-aws-lambda-py${{ matrix.python-version }} path: lambda_zip_contents/ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 339898a..e0d08f5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,18 +6,19 @@ on: types: [prereleased, released] env: - ASSET_NAME: skeleton-aws-lambda.zip PIP_CACHE_DIR: ~/.cache/pip - + PY_VERSION: 3.8 jobs: release: runs-on: ubuntu-latest + env: + ASSET_NAME: skeleton-aws-lambda_${{ env.PY_VERSION }}.zip steps: - uses: actions/checkout@v2 - id: setup-python uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: ${{ env.PY_VERSION }} - name: Cache lambda building environments uses: actions/cache@v2 with: @@ -34,7 +35,7 @@ jobs: python -m pip install --upgrade pip wheel pip install --upgrade --requirement requirements.txt - name: Build environment - run: docker-compose build + run: docker-compose build --build-arg PY_VERSION=${{ env.PY_VERSION }} - name: Generate lambda zip run: docker-compose up - name: Upload lambda zip as release asset diff --git a/Dockerfile b/Dockerfile index 93c3756..567f1f6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM lambci/lambda:build-python3.8 +ARG PY_VERSION +FROM lambci/lambda:build-python$PY_VERSION LABEL maintainer="mark.feldhousen@trio.dhs.gov" LABEL vendor="Cyber and Infrastructure Security Agency" @@ -12,4 +13,4 @@ COPY eal ./eal COPY lambda_handler.py . -ENTRYPOINT ["./build.sh"] +ENTRYPOINT ["./build.sh", "$PY_VERSION"] diff --git a/build.sh b/build.sh index bb02ac1..934e3f2 100755 --- a/build.sh +++ b/build.sh @@ -4,6 +4,13 @@ set -o nounset set -o errexit set -o pipefail +if [ "$#" -ne 1 ] +then + exit 1 +else + PY_VERSION="$1" +fi + ### # Define the name of the Lambda zip file being produced. ### @@ -55,24 +62,24 @@ BUILD_DIR=/build # Copy all packages, including any hidden dotfiles. Also copy the # local eal package and the lambda handler. ### -cp -rT $VENV_DIR/lib/python3.8/site-packages/ $BUILD_DIR -cp -rT $VENV_DIR/lib64/python3.8/site-packages/ $BUILD_DIR -cp -r eal $BUILD_DIR -cp lambda_handler.py $BUILD_DIR +cp -rT "$VENV_DIR/lib/python$PY_VERSION/site-packages/" "$BUILD_DIR" +cp -rT "$VENV_DIR/lib64/python$PY_VERSION/site-packages/" "$BUILD_DIR" +cp -r eal "$BUILD_DIR" +cp lambda_handler.py "$BUILD_DIR" ### # Zip it all up. ### OUTPUT_DIR=/output -if [ ! -d $OUTPUT_DIR ] +if [ ! -d "$OUTPUT_DIR" ] then - mkdir $OUTPUT_DIR + mkdir "$OUTPUT_DIR" fi -if [ -e $OUTPUT_DIR/$ZIP_FILE ] +if [ -e "$OUTPUT_DIR/$ZIP_FILE" ] then - rm $OUTPUT_DIR/$ZIP_FILE + rm "$OUTPUT_DIR/$ZIP_FILE" fi cd $BUILD_DIR -zip -rq9 $OUTPUT_DIR/$ZIP_FILE . +zip -rq9 "$OUTPUT_DIR/$ZIP_FILE" . From a96fe7e081b01fdd6e7077ad25b71df46ba70819 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 24 Nov 2020 15:13:17 -0500 Subject: [PATCH 02/12] Fix up build script --- .github/workflows/build.yml | 4 +++- Dockerfile | 20 ++++++++++++----- build.sh | 44 +++++++++++++++++++++++++++++-------- 3 files changed, 52 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 75a8274..43d183f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -153,7 +153,9 @@ jobs: # most likely be able leverage the functionality of upload-artifact v2(+) # to upload a single file as an artifact. - name: Unzip produced zip - run: unzip skeleton-aws-lambda.zip -d lambda_zip_contents + run: | + unzip skeleton-aws-lambda_${{ matrix.python-version }}.zip \ + -d lambda_zip_contents - name: Upload artifacts uses: actions/upload-artifact@v2 with: diff --git a/Dockerfile b/Dockerfile index 567f1f6..4a8ceb0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,24 @@ -ARG PY_VERSION +ARG PY_VERSION=3.8 + FROM lambci/lambda:build-python$PY_VERSION LABEL maintainer="mark.feldhousen@trio.dhs.gov" LABEL vendor="Cyber and Infrastructure Security Agency" +# Declare it a second time so it's brought into this scope. +ARG PY_VERSION=3.8 +ARG IMAGE_NAME=skeleton-aws-lambda + +ENV BUILD_PY_VERSION=$PY_VERSION +ENV BUILD_IMAGE_NAME=$IMAGE_NAME + COPY build.sh . +COPY lambda_handler.py . + # Files needed to install local eal module -COPY setup.py . -COPY requirements.txt . COPY README.md . +COPY requirements.txt . +COPY setup.py . COPY eal ./eal -COPY lambda_handler.py . - -ENTRYPOINT ["./build.sh", "$PY_VERSION"] +ENTRYPOINT ["./build.sh"] diff --git a/build.sh b/build.sh index 934e3f2..4175759 100755 --- a/build.sh +++ b/build.sh @@ -4,17 +4,43 @@ set -o nounset set -o errexit set -o pipefail -if [ "$#" -ne 1 ] +# Check for required external programs. If any are missing output a list of all +# requirements and then exit. +function check_dependencies { + required_tools="pip python zip" + for tool in $required_tools + do + if [ -z "$(command -v "$tool")" ] + then + echo "This script requires the following tools to run:" + for item in $required_tools + do + echo "- $item" + done + exit 1 + fi + done +} + +PY_VERSION="3.8" +IMAGE_NAME="skeleton-aws-lambda" + +check_dependencies + +if [ -n "$BUILD_PY_VERSION" ] +then + PY_VERSION="$BUILD_PY_VERSION" +fi + +if [ -n "$BUILD_IMAGE_NAME" ] then - exit 1 -else - PY_VERSION="$1" + IMAGE_NAME="$BUILD_IMAGE_NAME" fi ### # Define the name of the Lambda zip file being produced. ### -ZIP_FILE=skeleton-aws-lambda.zip +ZIP_FILE="${IMAGE_NAME}_${PY_VERSION}.zip" ### # Set up the Python virtual environment. @@ -22,15 +48,15 @@ ZIP_FILE=skeleton-aws-lambda.zip # installed in the container to avoid duplicating what will be available in the # lambda environment on AWS. ### -VENV_DIR=/venv -python -m venv --system-site-packages $VENV_DIR +VENV_DIR="/venv" +python -m venv --system-site-packages "$VENV_DIR" # Here shellcheck complains because it can't follow the dynamic path. # The path doesn't even exist until runtime, so we must disable that # check. # # shellcheck disable=1090 -source $VENV_DIR/bin/activate +source "$VENV_DIR/bin/activate" ### # Upgrade pip. @@ -70,7 +96,7 @@ cp lambda_handler.py "$BUILD_DIR" ### # Zip it all up. ### -OUTPUT_DIR=/output +OUTPUT_DIR="/output" if [ ! -d "$OUTPUT_DIR" ] then mkdir "$OUTPUT_DIR" From 7cc9b1d64b01f7b7bde57ce637d7f21820ec22c0 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 25 Nov 2020 19:00:55 -0500 Subject: [PATCH 03/12] Remove unnecessary Python setup When building the lambda we only need Docker tools which are pre-installed on the GH Actions runners. Thus, we do not need to set up a Python environment and install Python requirements. --- .github/workflows/build.yml | 22 ++-------------------- .github/workflows/release.yml | 21 +-------------------- 2 files changed, 3 insertions(+), 40 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 43d183f..77817d0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -121,25 +121,7 @@ jobs: python-version: [3.6, 3.7, 3.8] steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Cache lambda building environments - uses: actions/cache@v2 - with: - path: ${{ env.PIP_CACHE_DIR }} - key: "build_lambda-${{ runner.os }}-\ - py${{ matrix.python-version }}-\ - ${{ hashFiles('**/requirements.txt') }}" - restore-keys: | - build_lambda-${{ runner.os }}-\ - py${{ matrix.python-version }}- - build_lambda-${{ runner.os }}- - - name: Install dependencies - run: | - python -m pip install --upgrade pip wheel - pip install --upgrade --requirement requirements.txt - - name: Build environment + - name: Build Docker image run: | docker-compose build \ --build-arg PY_VERSION=${{ matrix.python-version }} @@ -152,7 +134,7 @@ jobs: # is the best source for possible updates. Once the UI is updated we will # most likely be able leverage the functionality of upload-artifact v2(+) # to upload a single file as an artifact. - - name: Unzip produced zip + - name: Unzip generated zip run: | unzip skeleton-aws-lambda_${{ matrix.python-version }}.zip \ -d lambda_zip_contents diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e0d08f5..e506c21 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,26 +15,7 @@ jobs: ASSET_NAME: skeleton-aws-lambda_${{ env.PY_VERSION }}.zip steps: - uses: actions/checkout@v2 - - id: setup-python - uses: actions/setup-python@v2 - with: - python-version: ${{ env.PY_VERSION }} - - name: Cache lambda building environments - uses: actions/cache@v2 - with: - path: ${{ env.PIP_CACHE_DIR }} - key: "release-${{ runner.os }}-\ - py${{ steps.setup-python.outputs.python-version }}-\ - ${{ hashFiles('**/requirements.txt') }}" - restore-keys: | - release-${{ runner.os }}-\ - py${{ steps.setup-python.outputs.python-version }}- - release-${{ runner.os }}- - - name: Install dependencies - run: | - python -m pip install --upgrade pip wheel - pip install --upgrade --requirement requirements.txt - - name: Build environment + - name: Build Docker image run: docker-compose build --build-arg PY_VERSION=${{ env.PY_VERSION }} - name: Generate lambda zip run: docker-compose up From a2745eff327f01e2ec3b3f5d11e4dda8d3b6535a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 26 Nov 2020 00:02:00 -0500 Subject: [PATCH 04/12] Change output file names both locally and in Actions workflows --- .github/workflows/build.yml | 8 +++++--- .github/workflows/release.yml | 7 +++++-- .gitignore | 2 +- Dockerfile | 7 +++++-- build.sh | 15 +++++++++++---- 5 files changed, 27 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 77817d0..cb5b970 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -124,7 +124,8 @@ jobs: - name: Build Docker image run: | docker-compose build \ - --build-arg PY_VERSION=${{ matrix.python-version }} + --build-arg PY_VERSION=${{ matrix.python-version }} \ + --build-arg FILE_NAME=${{ github.event.repository.name }} - name: Generate lambda zip run: docker-compose up # We have to unzip the produced lambda zip because of limitations with the @@ -136,10 +137,11 @@ jobs: # to upload a single file as an artifact. - name: Unzip generated zip run: | - unzip skeleton-aws-lambda_${{ matrix.python-version }}.zip \ + unzip ${{ github.event.repository.name }}*.zip \ -d lambda_zip_contents - name: Upload artifacts uses: actions/upload-artifact@v2 with: - name: skeleton-aws-lambda-py${{ matrix.python-version }} + name: "${{ github.event.repository.name }}-${{ github.sha }}-\ + py${{ matrix.python-version }}" path: lambda_zip_contents/ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e506c21..5b9e2ce 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,11 +12,14 @@ jobs: release: runs-on: ubuntu-latest env: - ASSET_NAME: skeleton-aws-lambda_${{ env.PY_VERSION }}.zip + ASSET_NAME: ${{ github.event.repository.name }}.zip steps: - uses: actions/checkout@v2 - name: Build Docker image - run: docker-compose build --build-arg PY_VERSION=${{ env.PY_VERSION }} + run: "docker-compose build \ + --build-arg PY_VERSION=${{ env.PY_VERSION }} \ + --build-arg FILE_NAME=${{ github.event.repository.name }} \ + --build-arg IS_RELEASE=TRUE" - name: Generate lambda zip run: docker-compose up - name: Upload lambda zip as release asset diff --git a/.gitignore b/.gitignore index 3db7a19..4edad1c 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,4 @@ .pytest_cache .python-version __pycache__ -skeleton-aws-lambda.zip +skeleton-aws-lambda*.zip diff --git a/Dockerfile b/Dockerfile index 4a8ceb0..403908b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,12 +6,15 @@ LABEL vendor="Cyber and Infrastructure Security Agency" # Declare it a second time so it's brought into this scope. ARG PY_VERSION=3.8 -ARG IMAGE_NAME=skeleton-aws-lambda +ARG FILE_NAME=skeleton-aws-lambda +ARG IS_RELEASE=FALSE ENV BUILD_PY_VERSION=$PY_VERSION -ENV BUILD_IMAGE_NAME=$IMAGE_NAME +ENV BUILD_FILE_NAME=$FILE_NAME +ENV BUILD_IS_RELEASE=$IS_RELEASE COPY build.sh . +COPY bump_version.sh . COPY lambda_handler.py . diff --git a/build.sh b/build.sh index 4175759..570c2c6 100755 --- a/build.sh +++ b/build.sh @@ -23,7 +23,7 @@ function check_dependencies { } PY_VERSION="3.8" -IMAGE_NAME="skeleton-aws-lambda" +FILE_NAME="skeleton-aws-lambda" check_dependencies @@ -32,15 +32,22 @@ then PY_VERSION="$BUILD_PY_VERSION" fi -if [ -n "$BUILD_IMAGE_NAME" ] +if [ -n "$BUILD_FILE_NAME" ] then - IMAGE_NAME="$BUILD_IMAGE_NAME" + FILE_NAME="$BUILD_FILE_NAME" fi +LAMBDA_VERSION=$(./bump_version.sh show) + ### # Define the name of the Lambda zip file being produced. ### -ZIP_FILE="${IMAGE_NAME}_${PY_VERSION}.zip" +if [ -n "$BUILD_IS_RELEASE" ] && [ "$BUILD_IS_RELEASE" == "TRUE" ] +then + ZIP_FILE="${FILE_NAME}.zip" +else + ZIP_FILE="${FILE_NAME}_${LAMBDA_VERSION}_py${PY_VERSION}.zip" +fi ### # Set up the Python virtual environment. From f4295ad74515fc31d376aa5f70f2d598bbd31c94 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 30 Nov 2020 10:52:56 -0500 Subject: [PATCH 05/12] Restore simple build script output name Revert the complicated output name bits from the build script and Actions workflows. Instead the Actions workflows will handle any fancy asset naming. --- .github/workflows/build.yml | 10 +++++++--- .github/workflows/release.yml | 17 +++++++++-------- .gitignore | 2 +- Dockerfile | 2 -- build.sh | 24 ++++-------------------- 5 files changed, 21 insertions(+), 34 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cb5b970..a222e09 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -121,6 +121,9 @@ jobs: python-version: [3.6, 3.7, 3.8] steps: - uses: actions/checkout@v2 + - name: Get short SHA for the commit being used + run: | + echo "GH_SHORT_SHA=${GITHUB_SHA::7}" >> $GITHUB_ENV - name: Build Docker image run: | docker-compose build \ @@ -137,11 +140,12 @@ jobs: # to upload a single file as an artifact. - name: Unzip generated zip run: | - unzip ${{ github.event.repository.name }}*.zip \ + unzip ${{ github.event.repository.name }}.zip \ -d lambda_zip_contents - name: Upload artifacts uses: actions/upload-artifact@v2 with: - name: "${{ github.event.repository.name }}-${{ github.sha }}-\ - py${{ matrix.python-version }}" + name: "${{ github.event.repository.name }}-\ + py${{ matrix.python-version }}-\ + ${{ env.GH_SHORT_SHA }}" path: lambda_zip_contents/ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5b9e2ce..609afff 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,19 +7,19 @@ on: env: PIP_CACHE_DIR: ~/.cache/pip - PY_VERSION: 3.8 jobs: release: runs-on: ubuntu-latest - env: - ASSET_NAME: ${{ github.event.repository.name }}.zip + strategy: + matrix: + # Versions supported by AWS and lambci/lambda images + python-version: [3.6, 3.7, 3.8] steps: - uses: actions/checkout@v2 - name: Build Docker image run: "docker-compose build \ - --build-arg PY_VERSION=${{ env.PY_VERSION }} \ - --build-arg FILE_NAME=${{ github.event.repository.name }} \ - --build-arg IS_RELEASE=TRUE" + --build-arg PY_VERSION=${{ matrix.python-version }} \ + --build-arg FILE_NAME=${{ github.event.repository.name }}" - name: Generate lambda zip run: docker-compose up - name: Upload lambda zip as release asset @@ -28,6 +28,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ github.event.release.upload_url }} - asset_path: ${{ env.ASSET_NAME }} - asset_name: ${{ env.ASSET_NAME }} + asset_path: "${{ github.event.repository.name }}.zip" + asset_name: "${{ github.event.repository.name }}-\ + ${{ matrix.python-version }}.zip" asset_content_type: application/zip diff --git a/.gitignore b/.gitignore index 4edad1c..3db7a19 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,4 @@ .pytest_cache .python-version __pycache__ -skeleton-aws-lambda*.zip +skeleton-aws-lambda.zip diff --git a/Dockerfile b/Dockerfile index 403908b..dac6915 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,10 +11,8 @@ ARG IS_RELEASE=FALSE ENV BUILD_PY_VERSION=$PY_VERSION ENV BUILD_FILE_NAME=$FILE_NAME -ENV BUILD_IS_RELEASE=$IS_RELEASE COPY build.sh . -COPY bump_version.sh . COPY lambda_handler.py . diff --git a/build.sh b/build.sh index 570c2c6..f298e14 100755 --- a/build.sh +++ b/build.sh @@ -22,32 +22,16 @@ function check_dependencies { done } -PY_VERSION="3.8" -FILE_NAME="skeleton-aws-lambda" - check_dependencies -if [ -n "$BUILD_PY_VERSION" ] -then - PY_VERSION="$BUILD_PY_VERSION" -fi - -if [ -n "$BUILD_FILE_NAME" ] -then - FILE_NAME="$BUILD_FILE_NAME" -fi - -LAMBDA_VERSION=$(./bump_version.sh show) +PY_VERSION="${BUILD_PY_VERSION:-3.8}" +# Use the current directory name +FILE_NAME="${BUILD_FILE_NAME:-${PWD##*/}}" ### # Define the name of the Lambda zip file being produced. ### -if [ -n "$BUILD_IS_RELEASE" ] && [ "$BUILD_IS_RELEASE" == "TRUE" ] -then - ZIP_FILE="${FILE_NAME}.zip" -else - ZIP_FILE="${FILE_NAME}_${LAMBDA_VERSION}_py${PY_VERSION}.zip" -fi +ZIP_FILE="${FILE_NAME}.zip" ### # Set up the Python virtual environment. From a759190dd93304d3d19b3345cc0f82a7b2b3917a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 30 Nov 2020 11:42:18 -0500 Subject: [PATCH 06/12] Enable Docker BuildKit in Workflows --- .github/workflows/build.yml | 2 +- .github/workflows/release.yml | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a222e09..93a15da 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -126,7 +126,7 @@ jobs: echo "GH_SHORT_SHA=${GITHUB_SHA::7}" >> $GITHUB_ENV - name: Build Docker image run: | - docker-compose build \ + DOCKER_BUILDKIT=1 docker-compose build \ --build-arg PY_VERSION=${{ matrix.python-version }} \ --build-arg FILE_NAME=${{ github.event.repository.name }} - name: Generate lambda zip diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 609afff..3ce5242 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,9 +17,10 @@ jobs: steps: - uses: actions/checkout@v2 - name: Build Docker image - run: "docker-compose build \ - --build-arg PY_VERSION=${{ matrix.python-version }} \ - --build-arg FILE_NAME=${{ github.event.repository.name }}" + run: | + DOCKER_BUILDKIT=1 docker-compose build \ + --build-arg PY_VERSION=${{ matrix.python-version }} \ + --build-arg FILE_NAME=${{ github.event.repository.name }} - name: Generate lambda zip run: docker-compose up - name: Upload lambda zip as release asset From 8004177923a622b9c766abbbad52455267f34831 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 2 Dec 2020 11:38:21 -0500 Subject: [PATCH 07/12] Clean up Dockerfile Remove reverted IS_RELEASE argument. Add some comments explaining functionality. Update the LABELs being set to reflect the current best practices. --- Dockerfile | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index dac6915..e9d48ce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,21 @@ +# Get the Python version to use from the commandline if provided ARG PY_VERSION=3.8 FROM lambci/lambda:build-python$PY_VERSION -LABEL maintainer="mark.feldhousen@trio.dhs.gov" -LABEL vendor="Cyber and Infrastructure Security Agency" # Declare it a second time so it's brought into this scope. ARG PY_VERSION=3.8 +# Get the output file name base from the commandline if provided ARG FILE_NAME=skeleton-aws-lambda -ARG IS_RELEASE=FALSE +# For a list of pre-defined annotation keys and value types see: +# https://github.com/opencontainers/image-spec/blob/master/annotations.md +# Note: Additional labels are added by the build workflow. +LABEL org.opencontainers.image.authors="nicholas.mcdonnell@cisa.dhs.gov" +LABEL org.opencontainers.image.vendor="Cyber and Infrastructure Security Agency" + +# Bring the command line ARGs into the ENV so they are available in the +# generated image. ENV BUILD_PY_VERSION=$PY_VERSION ENV BUILD_FILE_NAME=$FILE_NAME From b7766d80ef925dc4eef8ad3fb72cc55742dec26e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 2 Dec 2020 11:50:32 -0500 Subject: [PATCH 08/12] Adjust asset name for release uploads --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3ce5242..79e886b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,5 +31,5 @@ jobs: upload_url: ${{ github.event.release.upload_url }} asset_path: "${{ github.event.repository.name }}.zip" asset_name: "${{ github.event.repository.name }}-\ - ${{ matrix.python-version }}.zip" + py${{ matrix.python-version }}.zip" asset_content_type: application/zip From 7286f2dc2a51c8986b5faf1ee86a90d7efc5908c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 2 Dec 2020 14:13:29 -0500 Subject: [PATCH 09/12] Switch cp options to their long form --- build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sh b/build.sh index f298e14..49c4ee4 100755 --- a/build.sh +++ b/build.sh @@ -79,9 +79,9 @@ BUILD_DIR=/build # Copy all packages, including any hidden dotfiles. Also copy the # local eal package and the lambda handler. ### -cp -rT "$VENV_DIR/lib/python$PY_VERSION/site-packages/" "$BUILD_DIR" -cp -rT "$VENV_DIR/lib64/python$PY_VERSION/site-packages/" "$BUILD_DIR" -cp -r eal "$BUILD_DIR" +cp --recursive --no-target-directory "$VENV_DIR/lib/python$PY_VERSION/site-packages/" "$BUILD_DIR" +cp --recursive --no-target-directory "$VENV_DIR/lib64/python$PY_VERSION/site-packages/" "$BUILD_DIR" +cp --recursive eal "$BUILD_DIR" cp lambda_handler.py "$BUILD_DIR" ### From 7d2718d228f06b3d7a618ac6c18774e6430dd7f1 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 2 Dec 2020 14:14:55 -0500 Subject: [PATCH 10/12] Add comment explaining options for zip in build script --- build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build.sh b/build.sh index 49c4ee4..0f2341a 100755 --- a/build.sh +++ b/build.sh @@ -99,4 +99,6 @@ then fi cd $BUILD_DIR +# Recursively (-r) add the current directory to the specified output filename +# using maximum compression (-9) without informational message (-q). zip -rq9 "$OUTPUT_DIR/$ZIP_FILE" . From ec77abe5822440491013b45fa6010156bfa5ae28 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 2 Dec 2020 14:20:40 -0500 Subject: [PATCH 11/12] Expand the comment for an Actions job step Expand the comment to reference what the command is doing since zip does not have long options. --- .github/workflows/build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 93a15da..027c64a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -137,7 +137,9 @@ jobs: # https://github.com/actions/upload-artifact/issues/14 # is the best source for possible updates. Once the UI is updated we will # most likely be able leverage the functionality of upload-artifact v2(+) - # to upload a single file as an artifact. + # to upload a single file as an artifact. Until then we extract the + # contents to the "lambda_zip_contents" directory to let + # actions/upload-artifacts repackage them for availability on GitHub. - name: Unzip generated zip run: | unzip ${{ github.event.repository.name }}.zip \ From f525714964e6d28184d712cf1e93d8445fd257dc Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 2 Dec 2020 14:23:31 -0500 Subject: [PATCH 12/12] Add comment explaining variable usage in GHA step command --- .github/workflows/build.yml | 3 +++ .github/workflows/release.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 027c64a..ce2cb38 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -125,6 +125,9 @@ jobs: run: | echo "GH_SHORT_SHA=${GITHUB_SHA::7}" >> $GITHUB_ENV - name: Build Docker image + # DOCKER_BUILDKIT=1 tells Docker to use BuildKit when building the + # image per: + # https://docs.docker.com/develop/develop-images/build_enhancements/ run: | DOCKER_BUILDKIT=1 docker-compose build \ --build-arg PY_VERSION=${{ matrix.python-version }} \ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 79e886b..417e78e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,6 +17,9 @@ jobs: steps: - uses: actions/checkout@v2 - name: Build Docker image + # DOCKER_BUILDKIT=1 tells Docker to use BuildKit when building the + # image per: + # https://docs.docker.com/develop/develop-images/build_enhancements/ run: | DOCKER_BUILDKIT=1 docker-compose build \ --build-arg PY_VERSION=${{ matrix.python-version }} \