SYS-576 updates for python 3.11.4 base image (#19)

.gitlab-ci.yml

@@ -3,16 +3,20 @@
 #   PYPI_PASSWORD, PYPI_USER, REGISTRY_URI
 
 variables:
-  PLATFORMS: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
+#  PLATFORMS: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
+  PLATFORMS: linux/amd64,linux/arm64
+  IMAGE: example-api
+  REGISTRY: $REGISTRY_URI/$CI_PROJECT_PATH
 
 stages:
   - Analyze and Unit Test
   - Images
   - Functional Tests
+  - Security Scan
   - Publish Packages
   - Clean
 
-image: registry.gitlab.com/instantlinux/docker-tools/python-builder:3.9.7-r4
+image: registry.gitlab.com/instantlinux/docker-tools/python-builder:3.11.4-r0
 
 before_script:
   - export TAG=bld_$CI_PIPELINE_IID_${CI_COMMIT_SHA:0:7}
@@ -51,6 +55,39 @@ test:
   stage: Functional Tests
   script: make test_functional
 
+security_scan_trivy:
+  services: [ "docker:dind" ]
+  image:
+    name: aquasec/trivy:latest
+    entrypoint: [""]
+  stage: Security Scan
+  variables:
+    GIT_STRATEGY: none
+    TRIVY_CACHE_DIR: .trivycache/
+    TRIVY_DEBUG: "true"
+    TRIVY_EXIT_CODE: 1
+    TRIVY_FORMAT: json
+    TRIVY_OUTPUT: gl-container-scanning-report.json
+    TRIVY_SEVERITY: HIGH,CRITICAL
+    TRIVY_VULN_TYPE: os,library
+  script:
+  - export TAG=bld_$CI_PIPELINE_IID_${CI_COMMIT_SHA:0:7}
+  - trivy image --clear-cache
+  - trivy image --download-db-only --no-progress
+  - trivy image "${REGISTRY}/${IMAGE}:${TAG}" --severity LOW,MEDIUM
+      --exit-code 0 --format table --output medium-vulns.txt
+  - cat medium-vulns.txt
+  - trivy image "${REGISTRY}/${IMAGE}:${TAG}"
+  cache:
+    paths: [ .trivycache ]
+  interruptible: true
+  timeout: 5m
+  artifacts:
+    reports:
+      container_scanning: gl-container-scanning-report.json
+    expire_in: 30 days
+    paths: [ medium-vulns.txt ]
+
 promote_images:
   stage: Publish Packages
   <<: *registry_login

Makefile

@@ -55,9 +55,14 @@ flake8: dev_requirements
 
 dev_requirements: python_env requirements-dev.txt
 
-requirements-dev.txt: python_env
-	@echo Updating Pipfile.lock and requirements-dev.txt
-	. $(VDIR)/bin/activate && pipenv lock --requirements --dev > $@ || rm $@
+Pipfile.lock: Pipfile
+	@echo Updating $@
+	. $(VDIR)/bin/activate && pipenv lock --dev || rm $@
+
+requirements-dev.txt: python_env Pipfile.lock
+	@echo Updating $@
+	pipenv requirements --dev > $@ || rm $@
+	sed -i '1s/^/# To update, edit Pipfile and then "make requirements-dev.txt"\n/' $@
 	@echo "Installing dev requirements"
 	. $(VDIR)/bin/activate && pip install -r $@
 # TODO remove this when doc build works with sphinx 4.x

Pipfile

@@ -2,57 +2,58 @@
 geocoder = "*"
 
 # alpine provides pre-built wheels for these
-Authlib = "==0.15.5"
-boto3 = "==1.18.49"
-botocore = "==1.21.49"
-cryptography = "==3.3.2"
-Flask = "==2.0.2"
-Flask-Babel = "==2.0.0"
-itsdangerous = "==2.0.1"
-pycryptodomex = "==3.10.1"
-requests = "==2.26.0"
+# loads of breaking changes in 1.2
+Authlib = "<1.0.0"
+boto3 = "==1.26.132"
+botocore = "==1.29.132"
+cachetools = "==5.3.0"
+# upgrade to 41.0.2 to resolve CVE
+cryptography = "==40.0.2"
+# published 2.2.3 has CVE, newer 2.3.1 has error
+# AttributeError: module 'flask.json' has no attribute 'JSONEncoder'
+#  (See https://github.com/spec-first/connexion/issues/1699, in v2.14.2)
+Flask = "<2.3.0"
+# AttributeError: 'Babel' object has no attribute 'localeselector'
+Flask-Babel = "<3.0.0"
+greenlet = "==2.0.2"
+itsdangerous = "==2.1.2"
+jinja2 = "==3.1.2"
+ldap3 = "==2.9.1"
+passlib = "==1.7.4"
+pycryptodomex = "==3.17"
+requests = "==2.31.0"
+werkzeug = "==2.2.3"
 
 # for faster builds, pin these versions in sync with images
 # instantlinux/python-builder and instantlinux/python-uwsgi
 alembic = "*"
 b2sdk = "*"
-cachetools = "*"
 celery = "*"
 connexion = "*"
-"connexion[swagger-ui]" = "==2.7.0"
+"connexion[swagger-ui]" = "*"
 dollar-ref = "*"
 Flask-Cors = "*"
 # GeoAlchemy2 = "*"
-greenlet = "==1.1.2"
-ldap3 = "*"
-passlib = "*"
 PyJWT = "*"
 PyMySQL = "*"
 pyotp = "*"
 pytz = "*"
-# fakeredis doesn't handle newest redis
-redis = "<4.1.0"
+redis = "*"
 # Held back due to SAWarnings about many-to-many tables
 SQLAlchemy = "<1.4.0"
 SQLAlchemy-Utils = "*"
 swagger-ui-bundle = "*"
 urllib3 = "*"
-# Held back due to
-# AttributeError: 'LocalStack' object has no attribute '__ident_func__'
-werkzeug = "==2.0.2"
-# Held back due to
-# AttributeError: module 'jinja2.ext' has no attribute 'autoescape'
-jinja2 = "==3.0.3"
 
 # Transitive dependencies pinned to versions of python-wsgi base image
-arrow = "==1.2.2"
-cffi = "==1.14.5"
+##arrow = "==1.2.2" no longer present
+cffi = "==1.15.1"
 six = "==1.16.0"
 
 [dev-packages]
-coverage = "==6.3.3"
+coverage = "==7.2.7"
 fakeredis = "*"
-flake8 = "==4.0.1"
+flake8 = "==6.1.0"
 httpretty = "*"
 pytest = "*"
 pytest-cov = "*"