⚠️ CONFLICT! Lineage pull request for: skeleton

.github/workflows/build.yml

@@ -20,22 +20,20 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: 3.9
-      - name: Cache linting environments
-        uses: actions/cache@v2
+      - uses: actions/cache@v2
+        env:
+          BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\
+            py${{ steps.setup-python.outputs.python-version }}-"
         with:
           path: |
             ${{ env.PIP_CACHE_DIR }}
             ${{ env.PRE_COMMIT_CACHE_DIR }}
-          key: |
-            lint-${{ runner.os }}-\
-            py${{ steps.setup-python.outputs.python-version }}-\
+          key: "${{ env.BASE_CACHE_KEY }}\
             ${{ hashFiles('**/requirements-test.txt') }}-\
             ${{ hashFiles('**/requirements.txt') }}-\
             ${{ hashFiles('**/.pre-commit-config.yaml') }}"
           restore-keys: |
-            lint-${{ runner.os }}-\
-            py${{ steps.setup-python.outputs.python-version }}-
-            lint-${{ runner.os }}-
+            ${{ env.BASE_CACHE_KEY }}
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -51,21 +49,21 @@ jobs:
         python-version: [3.6, 3.7, 3.8, 3.9]
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - id: setup-python
+        uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Cache testing environments
-        uses: actions/cache@v2
+      - uses: actions/cache@v2
+        env:
+          BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\
+            py${{ steps.setup-python.outputs.python-version }}-"
         with:
           path: ${{ env.PIP_CACHE_DIR }}
-          key: "test-${{ runner.os }}-\
-            py${{ matrix.python-version }}-\
+          key: "${{ env.BASE_CACHE_KEY }}\
             ${{ hashFiles('**/requirements-test.txt') }}-\
             ${{ hashFiles('**/requirements.txt') }}"
           restore-keys: |
-            test-${{ runner.os }}-\
-            py${{ matrix.python-version }}-
-            test-${{ runner.os }}-
+            ${{ env.BASE_CACHE_KEY }}
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -87,20 +85,20 @@ jobs:
         python-version: [3.6, 3.7, 3.8, 3.9]
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - id: setup-python
+        uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Cache building environments
-        uses: actions/cache@v2
+      - uses: actions/cache@v2
+        env:
+          BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\
+            py${{ steps.setup-python.outputs.python-version }}-"
         with:
           path: ${{ env.PIP_CACHE_DIR }}
-          key: "build-${{ runner.os }}-\
-            py${{ matrix.python-version }}-\
+          key: "${{ env.BASE_CACHE_KEY }}\
             ${{ hashFiles('**/requirements.txt') }}"
           restore-keys: |
-            build-${{ runner.os }}-\
-            py${{ matrix.python-version }}-
-            build-${{ runner.os }}-
+            ${{ env.BASE_CACHE_KEY }}
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip wheel
@@ -115,29 +113,23 @@ jobs:
   build_lambda:
     runs-on: ubuntu-latest
     needs: build
+    strategy:
+      matrix:
+        # Versions supported by AWS and lambci/lambda images
+        python-version: [3.6, 3.7, 3.8]
     steps:
       - uses: actions/checkout@v2
-      - id: setup-python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.8
-      - name: Cache lambda building environments
-        uses: actions/cache@v2
-        with:
-          path: ${{ env.PIP_CACHE_DIR }}
-          key: "build_lambda-${{ runner.os }}-\
-            py${{ steps.setup-python.outputs.python-version }}-\
-            ${{ hashFiles('**/requirements.txt') }}"
-          restore-keys: |
-            build_lambda-${{ runner.os }}-\
-            py${{ steps.setup-python.outputs.python-version }}-
-            build_lambda-${{ runner.os }}-
-      - name: Install dependencies
+      - name: Get short SHA for the commit being used
         run: |
-          python -m pip install --upgrade pip wheel
-          pip install --upgrade --requirement requirements.txt
-      - name: Build environment
-        run: docker-compose build
+          echo "GH_SHORT_SHA=${GITHUB_SHA::7}" >> $GITHUB_ENV
+      - name: Build Docker image
+        # DOCKER_BUILDKIT=1 tells Docker to use BuildKit when building the
+        # image per:
+        # https://docs.docker.com/develop/develop-images/build_enhancements/
+        run: |
+          DOCKER_BUILDKIT=1 docker-compose build \
+            --build-arg PY_VERSION=${{ matrix.python-version }} \
+            --build-arg FILE_NAME=${{ github.event.repository.name }}
       - name: Generate lambda zip
         run: docker-compose up
       # We have to unzip the produced lambda zip because of limitations with the
@@ -146,12 +138,17 @@ jobs:
       # https://github.com/actions/upload-artifact/issues/14
       # is the best source for possible updates. Once the UI is updated we will
       # most likely be able leverage the functionality of upload-artifact v2(+)
-      # to upload a single file as an artifact.
-      - name: Unzip produced zip
-        run: unzip findings-data-import.zip -d lambda_zip_contents
+      # to upload a single file as an artifact. Until then we extract the
+      # contents to the "lambda_zip_contents" directory to let
+      # actions/upload-artifacts repackage them for availability on GitHub.
+      - name: Unzip generated zip
+        run: |
+          unzip ${{ github.event.repository.name }}.zip \
+            -d lambda_zip_contents
       - name: Upload artifacts
         uses: actions/upload-artifact@v2
         with:
-          name: "findings-data-import-\
-            py${{ steps.setup-python.outputs.python-version }}"
+          name: "${{ github.event.repository.name }}-\
+            py${{ matrix.python-version }}-\
+            ${{ env.GH_SHORT_SHA }}"
           path: lambda_zip_contents/

.github/workflows/release.yml

@@ -6,35 +6,24 @@ on:
     types: [prereleased, released]
 
 env:
-  ASSET_NAME: findings-data-import-lambda.zip
   PIP_CACHE_DIR: ~/.cache/pip
-
 jobs:
   release:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # Versions supported by AWS and lambci/lambda images
+        python-version: [3.6, 3.7, 3.8]
     steps:
       - uses: actions/checkout@v2
-      - id: setup-python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.8
-      - name: Cache lambda building environments
-        uses: actions/cache@v2
-        with:
-          path: ${{ env.PIP_CACHE_DIR }}
-          key: "release-${{ runner.os }}-\
-            py${{ steps.setup-python.outputs.python-version }}-\
-            ${{ hashFiles('**/requirements.txt') }}"
-          restore-keys: |
-            release-${{ runner.os }}-\
-            py${{ steps.setup-python.outputs.python-version }}-
-            release-${{ runner.os }}-
-      - name: Install dependencies
+      - name: Build Docker image
+        # DOCKER_BUILDKIT=1 tells Docker to use BuildKit when building the
+        # image per:
+        # https://docs.docker.com/develop/develop-images/build_enhancements/
         run: |
-          python -m pip install --upgrade pip wheel
-          pip install --upgrade --requirement requirements.txt
-      - name: Build environment
-        run: docker-compose build
+          DOCKER_BUILDKIT=1 docker-compose build \
+            --build-arg PY_VERSION=${{ matrix.python-version }} \
+            --build-arg FILE_NAME=${{ github.event.repository.name }}
       - name: Generate lambda zip
         run: docker-compose up
       - name: Upload lambda zip as release asset
@@ -43,6 +32,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           upload_url: ${{ github.event.release.upload_url }}
-          asset_path: ${{ env.ASSET_NAME }}
-          asset_name: ${{ env.ASSET_NAME }}
+          asset_path: "${{ github.event.repository.name }}.zip"
+          asset_name: "${{ github.event.repository.name }}-\
+            py${{ matrix.python-version }}.zip"
           asset_content_type: application/zip

Dockerfile

@@ -1,15 +1,32 @@
-FROM lambci/lambda:build-python3.8
-LABEL maintainer="Matthew Zackschewski <matthew.zackschewski@trio.dhs.gov>"
-LABEL vendor="Cyber and Infrastructure Security Agency"
+# Get the Python version to use from the commandline if provided
+ARG PY_VERSION=3.8
+
+FROM lambci/lambda:build-python$PY_VERSION
+
+# Declare it a second time so it's brought into this scope.
+ARG PY_VERSION=3.8
+# Get the output file name base from the commandline if provided
+ARG FILE_NAME=findings-data-import
+
+# For a list of pre-defined annotation keys and value types see:
+# https://github.com/opencontainers/image-spec/blob/master/annotations.md
+# Note: Additional labels are added by the build workflow.
+LABEL org.opencontainers.image.authors="nicholas.mcdonnell@cisa.dhs.gov"
+LABEL org.opencontainers.image.vendor="Cyber and Infrastructure Security Agency"
+
+# Bring the command line ARGs into the ENV so they are available in the
+# generated image.
+ENV BUILD_PY_VERSION=$PY_VERSION
+ENV BUILD_FILE_NAME=$FILE_NAME
 
 COPY build.sh .
 
-# Files needed to install local fdi module.
-COPY setup.py .
-COPY requirements.txt .
+COPY lambda_handler.py .
+
+# Files needed to install local eal module
 COPY README.md .
+COPY requirements.txt .
+COPY setup.py .
 COPY fdi ./fdi
 
-COPY lambda_handler.py .
-
 ENTRYPOINT ["./build.sh"]

build.sh

@@ -4,26 +4,50 @@ set -o nounset
 set -o errexit
 set -o pipefail
 
+# Check for required external programs. If any are missing output a list of all
+# requirements and then exit.
+function check_dependencies {
+  required_tools="pip python zip"
+  for tool in $required_tools
+  do
+    if [ -z "$(command -v "$tool")" ]
+    then
+      echo "This script requires the following tools to run:"
+      for item in $required_tools
+      do
+        echo "- $item"
+      done
+      exit 1
+    fi
+  done
+}
+
+check_dependencies
+
+PY_VERSION="${BUILD_PY_VERSION:-3.8}"
+# Use the current directory name
+FILE_NAME="${BUILD_FILE_NAME:-${PWD##*/}}"
+
 ###
 # Define the name of the Lambda zip file being produced.
 ###
-ZIP_FILE=findings-data-import.zip
+ZIP_FILE="${FILE_NAME}.zip"
 
 ###
 # Set up the Python virtual environment.
 # We use --system-site-packages so the venv has access to the packages already
 # installed in the container to avoid duplicating what will be available in the
 # lambda environment on AWS.
 ###
-VENV_DIR=/venv
-python -m venv --system-site-packages $VENV_DIR
+VENV_DIR="/venv"
+python -m venv --system-site-packages "$VENV_DIR"
 
 # Here shellcheck complains because it can't follow the dynamic path.
 # The path doesn't even exist until runtime, so we must disable that
 # check.
 #
 # shellcheck disable=1090
-source $VENV_DIR/bin/activate
+source "$VENV_DIR/bin/activate"
 
 ###
 # Upgrade pip.
@@ -55,24 +79,26 @@ BUILD_DIR=/build
 # Copy all packages, including any hidden dotfiles. Also copy the
 # local fdi package and the lambda handler.
 ###
-cp -rT $VENV_DIR/lib/python3.8/site-packages/ $BUILD_DIR
-cp -rT $VENV_DIR/lib64/python3.8/site-packages/ $BUILD_DIR
-cp -r fdi $BUILD_DIR
-cp lambda_handler.py $BUILD_DIR
+cp --recursive --no-target-directory "$VENV_DIR/lib/python$PY_VERSION/site-packages/" "$BUILD_DIR"
+cp --recursive --no-target-directory "$VENV_DIR/lib64/python$PY_VERSION/site-packages/" "$BUILD_DIR"
+cp --recursive fdi "$BUILD_DIR"
+cp lambda_handler.py "$BUILD_DIR"
 
 ###
 # Zip it all up.
 ###
-OUTPUT_DIR=/output
-if [ ! -d $OUTPUT_DIR ]
+OUTPUT_DIR="/output"
+if [ ! -d "$OUTPUT_DIR" ]
 then
-    mkdir $OUTPUT_DIR
+    mkdir "$OUTPUT_DIR"
 fi
 
-if [ -e $OUTPUT_DIR/$ZIP_FILE ]
+if [ -e "$OUTPUT_DIR/$ZIP_FILE" ]
 then
-    rm $OUTPUT_DIR/$ZIP_FILE
+    rm "$OUTPUT_DIR/$ZIP_FILE"
 fi
 
 cd $BUILD_DIR
-zip -rq9 $OUTPUT_DIR/$ZIP_FILE .
+# Recursively (-r) add the current directory to the specified output filename
+# using maximum compression (-9) without informational message (-q).
+zip -rq9 "$OUTPUT_DIR/$ZIP_FILE" .