Merge pull request #335 from helxplatform/trivy-scan-sapbert

bump versions

.github/workflows/build-push-dev-image.yml

@@ -48,20 +48,20 @@ jobs:
     # https://github.com/marketplace/actions/build-and-push-docker-images
 
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
+      uses: docker/setup-buildx-action@v3
       with:
         driver-opts: |
           network=host
 
     - name: Login to DockerHub
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
         logout: true
 
     - name: Login to Container Registry
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         registry: containers.renci.org
         username: ${{ secrets.CONTAINERHUB_USERNAME }}
@@ -72,7 +72,7 @@ jobs:
     # Notes on Cache: 
     # https://docs.docker.com/build/ci/github-actions/examples/#inline-cache
     - name: Build Push Container
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         context: .
         push: true

.github/workflows/build-push-release.yml

@@ -63,20 +63,20 @@ jobs:
     # step
     # https://github.com/marketplace/actions/build-and-push-docker-images
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
+      uses: docker/setup-buildx-action@v3
       with:
         driver-opts: |
           network=host
 
     - name: Login to DockerHub
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
         logout: true
 
     - name: Login to Container Registry
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         registry: containers.renci.org
         username: ${{ secrets.CONTAINERHUB_USERNAME }}
@@ -86,7 +86,7 @@ jobs:
     # Notes on Cache: 
     # https://docs.docker.com/build/ci/github-actions/examples/#inline-cache
     - name: Build Push Container
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         push: true
         # Push to renci-registry and dockerhub here.

.github/workflows/code-checks.yml

@@ -1,23 +1,27 @@
 # Workflow responsible for core acceptance testing.
 # Tests Currently Run:
 #     - flake8-linter
-#     - image-build-test
-#    
-# This workflow only validates images can build
-# but does not push images to any repository.
-#
+#     - PYTest
+#     - Bandit
+# For PR Vulnerability Scanning a separate workflow will run.
 # The build-push-dev-image and build-push-release workflows 
 # handle the develop and release image storage respectively.
 #
 #
 
 name: Code-Checks
 on:
-  push:
-    branches-ignore:
-      - master
-      - main
+  # push:
+  #   branches-ignore:
+  #     - master
+  #     - main
+  #     - develop
+  pull_request:
+    branches:
       - develop
+      - master
+      - main 
+    types: [ opened, synchronize ]
     paths-ignore:
       - README.md
       - .old_cicd/*
@@ -27,13 +31,6 @@ on:
       - .gitignore
       - .dockerignore
       - .githooks
-  pull_request:
-    branches:
-      - develop
-      - master
-      - main 
-    types: [ opened, synchronize ]
-
 
 jobs:
 ############################## flake8-linter ##############################
@@ -45,7 +42,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: '3.10'
+        python-version: '3.12'
 
     # Currently actions/setup-python supports caching
     # but the cache is not as robust as cache action.
@@ -69,35 +66,45 @@ jobs:
       # flake8 --ignore=E,W --exit-zero . 
       continue-on-error: true
 
-############################## test-image-build ##############################
-  test-image-build:
-    # needs: flake8-linter
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
+# ############################## build-vuln-test ##############################
+  # build-vuln-test:
+  #   # needs: flake8-linter
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #   - uses: actions/checkout@v3
 
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
-      with:
-        driver-opts: |
-          network=host
+  #   - name: Set up Docker Buildx
+  #     uses: docker/setup-buildx-action@v3
+  #     with:
+  #       driver-opts: |
+  #         network=host
 
-    - name: Login to DockerHub
-      uses: docker/login-action@v2
-      with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-        logout: true
+  #   - name: Login to DockerHub
+  #     uses: docker/login-action@v3
+  #     with:
+  #       username: ${{ secrets.DOCKERHUB_USERNAME }}
+  #       password: ${{ secrets.DOCKERHUB_TOKEN }}
+  #       logout: true
+
+  #   # Notes on Cache: 
+  #   # https://docs.docker.com/build/ci/github-actions/examples/#inline-cache
+  #   - name: Build Container
+  #     uses: docker/build-push-action@v5
+  #     with:
+  #       context: .
+  #       push: false
+  #       load: true
+  #       tag: ${{ github.repository }}:vuln-test
+  #       cache-from: type=registry,ref=${{ github.repository }}:buildcache
+  #       cache-to: type=registry,ref=${{ github.repository }}:buildcache,mode=max
+  #   ####### Run for Fidelity ######
+  #   - name: Run Trivy vulnerability scanner
+  #     uses: aquasecurity/trivy-action@master
+  #     with:
+  #       image-ref: '${{ github.repository }}:vuln-test'
+  #       severity: 'CRITICAL,HIGH'
+  #       exit-code: '1'
 
-    # Notes on Cache: 
-    # https://docs.docker.com/build/ci/github-actions/examples/#inline-cache
-    - name: Build Container
-      uses: docker/build-push-action@v4
-      with:
-        context: .
-        push: false
-        cache-from: type=registry,ref=${{ github.repository }}:buildcache
-        cache-to: type=registry,ref=${{ github.repository }}:buildcache,mode=max
 ################################### PYTEST ###################################
   pytest:
     runs-on: ubuntu-latest
@@ -106,7 +113,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: '3.10'
+        python-version: '3.12'
 
     - name: Install Requirements
       run: |
@@ -126,7 +133,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: '3.10'
+        python-version: '3.12'
 
     - name: Install Requirements
       run: |

.github/workflows/trivy-pr-scan.yml

@@ -0,0 +1,67 @@
+
+name: trivy-pr-scan 
+on:
+  pull_request:
+    branches:
+      - develop
+      - master
+      - main 
+    types: [ opened, synchronize ]
+    paths-ignore:
+    - README.md
+    - .old_cicd/*
+    - .github/*
+    - .github/workflows/*
+    - LICENSE
+    - .gitignore
+    - .dockerignore
+    - .githooks
+
+jobs:
+ trivy-pr-scan:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+      with:
+        driver-opts: |
+          network=host
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v3
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+        logout: true
+
+    # Notes on Cache: 
+    # https://docs.docker.com/build/ci/github-actions/examples/#inline-cache
+    - name: Build Container
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        push: false
+        load: true
+        tags: ${{ github.repository }}:vuln-test
+        cache-from: type=registry,ref=${{ github.repository }}:buildcache
+        cache-to: type=registry,ref=${{ github.repository }}:buildcache,mode=max
+
+    # We will not be concerned with Medium and Low vulnerabilities
+    - name: Run Trivy vulnerability scanner
+      uses: aquasecurity/trivy-action@master
+      with:
+        image-ref: '${{ github.repository }}:vuln-test'
+        format: 'sarif'
+        severity: 'CRITICAL,HIGH'
+        output: 'trivy-results.sarif'
+        exit-code: '1'
+    # Scan results should be viewable in GitHub Security Dashboard
+    # We still fail the job if results are found, so below will always run
+    # unless manually canceled.
+    - name: Upload Trivy scan results to GitHub Security tab
+      uses: github/codeql-action/upload-sarif@v2
+      if: '!cancelled()'
+      with:
+        sarif_file: 'trivy-results.sarif'

Dockerfile

@@ -6,16 +6,15 @@
 FROM python:3.12.0-alpine3.18
 
 # Install required packages
-RUN apt-get update && \
-    apt-get install -y curl make vim && \
-    rm -rf /var/cache/apt/*
+RUN apk update && \
+    apk add g++ make    
 
 # Create a non-root user.
 ENV USER dug
 ENV HOME /home/$USER
 ENV UID 1000
 
-RUN adduser --disabled-login --home $HOME --shell /bin/bash --uid $UID $USER
+RUN adduser -D --home $HOME  --uid $UID $USER
 
 USER $USER
 WORKDIR $HOME

README.md

@@ -290,3 +290,8 @@ TOPMed phenotypic concept data is [here](https://github.com/helxplatform/dug/tre
 ## Release
 
 To release, commit the change and select feature.
+
+#### Fail on Vulnerability Detection
+
+During PR's several vulnerability scanners are run. If there are vulnerabilities detected, the pr checks will fail and a report will be sent to Github Security Dashboard for viewing. Please ensure the vulnerability is mitigated prior to continuing the merge to protected branches.
+

bin/vlmd_to_dbgap_xml.py

@@ -161,10 +161,12 @@ def vlmd_to_dbgap_xml(input_file, output, file_format, study_id, appl_id, study_
                 # description later if that is useful.
                 if row.get('constraints.pattern'):
                     counters['constraints.pattern'] += 1
-                    logging.warning(f"`constraints.pattern` of {row['constraints.pattern']} found in row {row_index}, skipped.")
+                    logging.warning(f"`constraints.pattern` of {row['constraints.pattern']} found in row {row_index}, "
+                                    f"but pattern constraints are not currently being written.")
                 if row.get('format'):
                     counters['format'] += 1
-                    logging.warning(f"Found `format` of {row['format']} found in row {row_index}, skipped.")
+                    logging.warning(f"Found `format` of {row['format']} found in row {row_index}, but format is not "
+                                    f"currently being written.")
 
                 # Process enumerated and encoded values.
                 encs = {}

requirements.txt

@@ -1,29 +1,29 @@
 aiohttp
 asyncio
-fastapi==0.95.0
-uvicorn==0.23.2
+fastapi
+uvicorn
 elasticsearch[async]==8.5.2
 gunicorn
 itsdangerous
 Jinja2
 jsonschema
 MarkupSafe
-ormar==0.12.1
-mistune==2.0.3
-pluggy==1.0.0
-pyrsistent==0.17.3
+ormar
+mistune
+pluggy
+pyrsistent
 pytest
-pytz==2021.1
-PyYAML==6.0
-requests==2.31.0
-# old redis==4.4.2
-redis==4.5.4
-requests-cache==0.9.8
-six==1.16.0
+pytz
+PyYAML
+requests
+redis
+requests-cache
+six
 
 # Click for command line arguments
 # We use Click 7.0 because that's what one of the pinned packages above use.
 click
-httpx>=0.24.1
+httpx
+linkml-runtime==1.6.0
 bmt==1.1.0
-urllib3>=1.26.17
+urllib3