Use poetry and Docker cache

.dockerignore

@@ -7,4 +7,5 @@
 !setup.py
 !setup.cfg
 !requirements.txt
-!requirements-dev.txt
+!poetry.lock
+!pyproject.toml

.github/workflows/main.yaml

@@ -51,15 +51,13 @@ jobs:
         run: c2cciutils-checks
 
       # the if the generated files are up to date
-      - run: ./run-jsonschema-gentypes
-      - run: jsonschema2md scan_to_paperless/config_schema.json config.md
-      - run: jsonschema2md scan_to_paperless/process_schema.json process.md
+      - run: make jsonschema
       - run: git diff --quiet
       - run: git diff
         if: failure()
 
-      - run: docker build --target=base --tag=sbrunner/scan-to-paperless .
-      - run: docker build --target=tests --tag=tests .
+      - run: make build
+      - run: make build-tests
       - run: docker run --rm tests prospector --output=pylint
       - name: pytest
         run: >
@@ -85,7 +83,7 @@ jobs:
           if-no-files-found: ignore
           retention-days: 5
 
-      - run: docker build --target=all --tag=sbrunner/scan-to-paperless:latest-all .
+      - run: make build-all
       - run: docker login --username=${{ secrets.DOCKER_USERNAME }} --password=${{ secrets.DOCKER_PASSWORD }}
 
       - name: Init pypi

.github/workflows/rebuild.yaml

@@ -30,15 +30,13 @@ jobs:
         run: c2cciutils-checks
 
       # the if the generated files are up to date
-      - run: ./run-jsonschema-gentypes
-      - run: jsonschema2md scan_to_paperless/config_schema.json config.md
-      - run: jsonschema2md scan_to_paperless/process_schema.json process.md
+      - run: make jsonschema
       - run: git diff --quiet
       - run: git diff
         if: failure()
 
-      - run: docker build --target=base --tag=sbrunner/scan-to-paperless .
-      - run: docker build --target=tests --tag=tests .
+      - run: make build
+      - run: make build-tests
       - run: docker run --rm tests prospector --output=pylint
       - name: Init Docker volumes
         run: |
@@ -74,7 +72,7 @@ jobs:
           retention-days: 5
         if: failure()
 
-      - run: docker build --target=all --tag=sbrunner/scan-to-paperless:latest-all .
+      - run: make build-all
       - run: docker login --username=${{ secrets.DOCKER_USERNAME }} --password=${{ secrets.DOCKER_PASSWORD }}
 
       - name: Init pypi

.hadolint.yaml

@@ -1,3 +1,4 @@
 ignored:
-  - DL3027 # warning: Do not use apt as it is meant to be a end-user tool, use apt-get or apt-cache instead
   - DL3003 # warning: Use WORKDIR to switch to a directory
+  - DL3008 # warning: Pin versions in apt get install.
+  - DL3042 # warning: Avoid cache directory with pip install --no-cache-dir <package>

Dockerfile

@@ -1,18 +1,37 @@
-FROM ubuntu:20.04 as base-dist
+FROM ubuntu:20.04 as base-all
+
+RUN --mount=type=cache,target=/var/lib/apt/lists \
+    apt-get update
 
 ENV DEBIAN_FRONTEND=noninteractive
-RUN \
-    apt update && \
-    apt install --assume-yes --no-install-recommends \
+RUN --mount=type=cache,target=/var/lib/apt/lists \
+    --mount=type=cache,target=/var/cache,sharing=locked \
+    apt-get install --assume-yes --no-install-recommends python3-pip
+
+FROM base-all as poetry
+
+WORKDIR /tmp
+COPY requirements.txt ./
+RUN --mount=type=cache,target=/root/.cache \
+    python3 -m pip install --disable-pip-version-check --requirement=requirements.txt && \
+    rm requirements.txt
+
+COPY poetry.lock pyproject.toml ./
+RUN poetry export --without-hashes --output=requirements.txt && \
+    poetry export --dev --without-hashes --output=requirements-dev.txt
+
+FROM base-all as base-dist
+
+RUN --mount=type=cache,target=/var/lib/apt/lists \
+    --mount=type=cache,target=/var/cache,sharing=locked \
+    apt-get install --assume-yes --no-install-recommends \
     graphicsmagick pdftk-java \
     tesseract-ocr tesseract-ocr-fra tesseract-ocr-deu tesseract-ocr-eng \
-    libimage-exiftool-perl software-properties-common python3-pip ghostscript && \
-    apt clean && \
-    rm --recursive --force /var/lib/apt/lists/* /root/.cache /var/cache/*
+    libimage-exiftool-perl software-properties-common ghostscript
 
-COPY requirements.txt /tmp/
-RUN python3 -m pip install --disable-pip-version-check --no-cache-dir --requirement=/tmp/requirements.txt && \
-    rm --recursive --force /tmp/*
+RUN --mount=type=cache,target=/root/.cache \
+    --mount=type=bind,from=poetry,source=/tmp,target=/tmp \
+    python3 -m pip install --disable-pip-version-check --requirement=/tmp/requirements.txt
 
 VOLUME /source \
     /destination
@@ -23,36 +42,33 @@ WORKDIR /opt
 
 FROM base-dist as tests-dist
 
-RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt install --assume-yes --no-install-recommends \
-    poppler-utils ghostscript graphviz && \
-    apt-get clean && \
-    rm --recursive --force /var/lib/apt/lists/* /root/.cache /var/cache/*
+RUN --mount=type=cache,target=/var/lib/apt/lists \
+    --mount=type=cache,target=/var/cache,sharing=locked \
+    apt-get install --assume-yes --no-install-recommends poppler-utils ghostscript graphviz
 
-COPY requirements-dev.txt /tmp/
-RUN python3 -m pip install --disable-pip-version-check --no-cache-dir --requirement=/tmp/requirements-dev.txt && \
-    rm --recursive --force /tmp/*
+RUN --mount=type=cache,target=/root/.cache \
+    --mount=type=bind,from=poetry,source=/tmp,target=/tmp \
+    python3 -m pip install --disable-pip-version-check --requirement=/tmp/requirements-dev.txt
 
 FROM base-dist as base
 
 COPY scan_to_paperless scan_to_paperless/
 COPY setup.py README.md ./
-RUN python3 -m pip install --no-cache-dir --editable .
+RUN --mount=type=cache,target=/root/.cache \
+    python3 -m pip install --editable .
 
 CMD ["scan-process"]
 
 
 FROM tests-dist as tests
 
 COPY . ./
-RUN python3 -m pip install --no-cache-dir --editable .
+RUN --mount=type=cache,target=/root/.cache \
+    python3 -m pip install --editable .
 
 
 FROM base as all
 
-RUN \
-    apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt install --assume-yes --no-install-recommends \
-    tesseract-ocr-all && \
-    apt-get clean && \
-    rm --recursive --force /var/lib/apt/lists/* /var/cache/*
+RUN --mount=type=cache,target=/var/lib/apt/lists \
+    --mount=type=cache,target=/var/cache,sharing=locked \
+    apt-get install --assume-yes --no-install-recommends tesseract-ocr-all

Makefile

@@ -1,12 +1,21 @@
+export DOCKER_BUILDKIT=1
 
 .PHONY: jsonschema
 jsonschema:
 	jsonschema2md scan_to_paperless/config_schema.json config.md
 	jsonschema2md scan_to_paperless/process_schema.json process.md
 	jsonschema-gentypes
 
-.PHONY: build-test
-build-test:
+.PHONY: build
+build:
+	docker build --target=base --tag=sbrunner/scan-to-paperless .
+
+.PHONY: build-all
+build-all:
+	docker build --target=all --tag=sbrunner/scan-to-paperless:latest-all .
+
+.PHONY: build-tests
+build-tests:
 	docker build --target=tests --tag=tests .
 
 .PHONY: prospector

ci/config.yaml

@@ -4,6 +4,9 @@ checks:
     backport.yaml: False
     audit.yaml: False
     main.yaml: False
+  codespell:
+    ignore_re:
+      - poetry.lock
 
 publish:
   docker:

ci/requirements.txt

@@ -1,2 +1,3 @@
 c2cciutils==1.1.dev20220105162844
 jsonschema2md==0.3.0
+jsonschema-gentypes==0.9.4