Merge pull request #3 from Georepublic/dev-python3.11

Python3.12

.coveragerc

@@ -6,3 +6,6 @@ omit =
   */python?.?/*
   */site-packages/nose/*
 show_missing = true
+
+[html]
+directory = coverage_html_report

.github/workflows/ci.yaml

@@ -0,0 +1,29 @@
+name: CI
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.12'
+      - name: Create virtualenv
+        run: |
+          python -m venv venv
+          source venv/bin/activate
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+          pip install -r requirements-dev.txt
+      - name: Run pytest
+        run: |
+          pytest -v
+      - name: Run pre-commit
+        run: |
+          pip install pre-commit
+          pre-commit install-hooks
+          pre-commit run --all-files

.github/workflows/lambda.yaml

@@ -0,0 +1,63 @@
+name: Test with LocalStack
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install dependencies
+        run: sudo apt update && sudo apt install -y make wget
+      - name: build docker image
+        run: make archive
+      - name: Start LocalStack
+        uses: LocalStack/setup-localstack@main
+        with:
+          image-tag: 'latest'
+          install-awslocal: 'true'
+      - name: Run tests against localstack
+        run: |
+          awslocal s3 mb s3://antivirus-definitions
+          awslocal s3 mb s3://test-bucket
+          wget https://secure.eicar.org/eicar_com.zip
+          awslocal s3 cp eicar_com.zip s3://test-bucket/eicar_com.zip
+          awslocal lambda create-function \
+            --function-name update-clamav \
+            --runtime python3.12 \
+            --handler update.lambda_handler \
+            --role arn:aws:iam::123456789012:role/lambda-role \
+            --zip-file fileb://./build/lambda.zip \
+            --timeout 120 \
+            --environment "Variables={AV_DEFINITION_S3_BUCKET=antivirus-definitions}"
+          sleep 30
+          awslocal lambda invoke \
+            --function-name update-clamav \
+            --invocation-type RequestResponse \
+            --log-type Tail \
+            --payload '{}' \
+            response.json \
+            --query 'LogResult' | tr -d '"' | base64 -d
+          awslocal lambda create-function \
+            --function-name scan-clamav \
+            --runtime python3.12 \
+            --handler scan.lambda_handler \
+            --role arn:aws:iam::123456789012:role/lambda-role \
+            --zip-file fileb://./build/lambda.zip \
+            --timeout 120 \
+            --environment "Variables={AV_DEFINITION_S3_BUCKET=antivirus-definitions,AV_DELETE_INFECTED_FILES=True}"
+          sleep 30
+          awslocal lambda invoke \
+            --function-name scan-clamav \
+            --invocation-type RequestResponse \
+            --log-type Tail \
+            --payload '{"Records": [{"s3": {"bucket": {"name": "test-bucket"}, "object": {"key": "eicar_com.zip"}}}]}' \
+            response.json \
+            --query 'LogResult' | tr -d '"' | base64 -d
+          result=$(awslocal s3 ls s3://test-bucket)
+          if [ -z "$result" ]; then
+            echo "Bucket is empty"
+          else
+            echo "Bucket is not empty"
+            exit 1
+          fi

.github/workflows/pre-commit.yaml

@@ -0,0 +1,26 @@
+name: pre-commit
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.12'
+      - name: Create virtualenv
+        run: |
+          python -m venv venv
+          source venv/bin/activate
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+          pip install -r requirements-dev.txt
+      - name: Run pre-commit
+        run: |
+          pip install pre-commit
+          pre-commit install-hooks
+          pre-commit run --all-files

.gitignore

@@ -117,3 +117,9 @@ tmp/
 
 # EICAR Files
 *eicar*
+
+# response.json
+response.json
+
+# coverage report
+coverage_html_report/

.pre-commit-config.yaml

@@ -1,30 +1,41 @@
 repos:
 
   - repo: https://github.com/ambv/black
-    rev: stable
+    rev: 23.12.1
     hooks:
       - id: black
-        language_version: python3.7
+        language_version: python3.12
         exclude: >
           (?x)^(
             scripts/gen-docs-index|
           )$
 
-  - repo: git://github.com/pre-commit/pre-commit-hooks
-    rev: v2.2.3
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
     hooks:
       - id: check-ast
       - id: check-json
       - id: check-merge-conflict
       - id: check-yaml
+        exclude: deploy/
       - id: debug-statements
       - id: detect-private-key
       - id: fix-encoding-pragma
-      - id: flake8
       - id: trailing-whitespace
 
-  - repo: git://github.com/igorshubovych/markdownlint-cli
-    rev: v0.17.0
+  - repo: https://github.com/pycqa/flake8
+    rev: 7.0.0
+    hooks:
+      - id: flake8
+
+  - repo: https://github.com/igorshubovych/markdownlint-cli
+    rev: v0.38.0
     hooks:
       - id: markdownlint
         entry: markdownlint --ignore .github/*.md
+
+  - repo: https://github.com/aws-cloudformation/cfn-python-lint
+    rev: v0.84.0
+    hooks:
+      - id: cfn-python-lint
+        files: deploy/

Dockerfile

@@ -1,32 +1,51 @@
-FROM amazonlinux:2
+FROM amazonlinux:2023 as builder
+
+# Set up working directories
+RUN mkdir -p /opt/python
+
+# Install packages
+RUN dnf update -y
+RUN dnf install -y gcc openssl-devel bzip2-devel libffi-devel zlib-devel wget make tar xz
+
+# Download and install Python 3.12
+WORKDIR /opt
+RUN wget https://www.python.org/ftp/python/3.12.1/Python-3.12.1.tar.xz
+RUN tar xvf Python-3.12.1.tar.xz
+WORKDIR /opt/Python-3.12.1
+RUN ./configure --enable-optimizations --prefix=/opt/python
+RUN make -j
+RUN make install
+
+FROM amazonlinux:2023
 
 # Set up working directories
 RUN mkdir -p /opt/app
 RUN mkdir -p /opt/app/build
 RUN mkdir -p /opt/app/bin/
 
+# Copy over the python binaries
+COPY --from=builder /opt/python /opt/python
+
 # Copy in the lambda source
 WORKDIR /opt/app
 COPY ./*.py /opt/app/
 COPY requirements.txt /opt/app/requirements.txt
 
 # Install packages
-RUN yum update -y
-RUN yum install -y cpio python2-pip yum-utils zip unzip less
-RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+RUN dnf update -y
+RUN dnf install -y cpio openssl bzip2 libffi yum-utils zip unzip less
 
 # This had --no-cache-dir, tracing through multiple tickets led to a problem in wheel
-RUN pip install -r requirements.txt
+RUN /opt/python/bin/pip3 install -r requirements.txt
 RUN rm -rf /root/.cache/pip
 
 # Download libraries we need to run in lambda
 WORKDIR /tmp
-RUN yumdownloader -x \*i686 --archlist=x86_64 clamav clamav-lib clamav-update json-c pcre2
+RUN yumdownloader -x \*i686 --archlist=x86_64 clamav clamav-lib clamav-update libtool-ltdl
 RUN rpm2cpio clamav-0*.rpm | cpio -idmv
 RUN rpm2cpio clamav-lib*.rpm | cpio -idmv
 RUN rpm2cpio clamav-update*.rpm | cpio -idmv
-RUN rpm2cpio json-c*.rpm | cpio -idmv
-RUN rpm2cpio pcre*.rpm | cpio -idmv
+RUN rpm2cpio libtool-ltdl* | cpio -idmv
 
 # Copy over the binaries and libraries
 RUN cp /tmp/usr/bin/clamscan /tmp/usr/bin/freshclam /tmp/usr/lib64/* /opt/app/bin/
@@ -39,7 +58,7 @@ RUN echo "CompressLocalDatabase yes" >> /opt/app/bin/freshclam.conf
 WORKDIR /opt/app
 RUN zip -r9 --exclude="*test*" /opt/app/build/lambda.zip *.py bin
 
-WORKDIR /usr/lib/python2.7/site-packages
+WORKDIR /opt/python/lib/python3.12/site-packages
 RUN zip -r9 /opt/app/build/lambda.zip *
 
 WORKDIR /opt/app

Makefile

@@ -29,7 +29,8 @@ clean:  ## Clean build artifacts
 	rm -rf build/
 	rm -rf tmp/
 	rm -f .coverage
-	find ./ -type d -name '__pycache__' -delete
+	rm -rf .pytest_cache/
+	find ./ -type d -name '__pycache__' | xargs rm -rf
 	find ./ -type f -name '*.pyc' -delete
 
 .PHONY: archive
@@ -40,8 +41,8 @@ archive: clean  ## Create the archive for AWS lambda
 
 .PHONY: pre_commit_install  ## Ensure that pre-commit hook is installed and kept up to date
 pre_commit_install: .git/hooks/pre-commit ## Ensure pre-commit is installed
-.git/hooks/pre-commit: /usr/local/bin/pre-commit
-	pip install pre-commit==1.18.3
+.git/hooks/pre-commit: venv ## Ensure venv is created first
+	pip install pre-commit
 	pre-commit install
 	pre-commit install-hooks
 
@@ -51,16 +52,8 @@ pre_commit_tests: ## Run pre-commit tests
 
 .PHONY: test
 test: clean  ## Run python tests
-	nosetests
+	pytest --no-cov
 
 .PHONY: coverage
 coverage: clean  ## Run python tests with coverage
-	nosetests --with-coverage
-
-.PHONY: scan
-scan: ./build/lambda.zip ## Run scan function locally
-	scripts/run-scan-lambda $(TEST_BUCKET) $(TEST_KEY)
-
-.PHONY: update
-update: ./build/lambda.zip ## Run update function locally
-	scripts/run-update-lambda
+	pytest --cov=. --cov-report html