Merge pull request #39 from sbrunner/test

Test

.circleci/config.yml

@@ -33,7 +33,7 @@ jobs:
       - run:
           name: Init Docker volumes
           command: |
-            docker run --rm --detach --name volumes --volume=tests:/tests --volume=results:/results alpine sleep 9999
+            docker run --rm --detach --name volumes --volume=tests:/tests --volume=results:/results alpine tail -f /dev/null
             docker cp tests/ volumes:/tests/
           when: always
       - run:
@@ -62,26 +62,22 @@ jobs:
           path: results/test-reports
 
       - run: docker build --target=all --tag=sbrunner/scan-to-paperless:all .
-      - run: docker build --target=experimental --tag=sbrunner/scan-to-paperless:experimental .
       - run: docker login --username $DOCKER_USER --password $DOCKER_PASS
       - deploy:
           command: |
             if [ "${CIRCLE_BRANCH}" == "master" ]
             then
               docker push sbrunner/scan-to-paperless
               docker push sbrunner/scan-to-paperless:all
-              docker push sbrunner/scan-to-paperless:experimental
             fi
       - deploy:
           command: |
             if [ "${CIRCLE_BRANCH}" == "test" ]
             then
               docker tag sbrunner/scan-to-paperless sbrunner/scan-to-paperless:test
               docker tag sbrunner/scan-to-paperless:all sbrunner/scan-to-paperless:test-all
-              docker tag sbrunner/scan-to-paperless:experimental sbrunner/scan-to-paperless:test-experimental
               docker push sbrunner/scan-to-paperless:test
               docker push sbrunner/scan-to-paperless:test-all
-              docker push sbrunner/scan-to-paperless:test-experimental
             fi
       - run:
           name: Init Git

Dockerfile

@@ -1,36 +1,15 @@
-FROM ubuntu:cosmic as builder
-
-RUN apt-get update
-RUN DEBIAN_FRONTEND=noninteractive apt-get install --assume-yes --no-install-recommends \
-    python3-dev python3-wheel python3-pip python3-setuptools \
-    curl unzip
-
-RUN curl http://galfar.vevb.net/store/deskew-125.zip > /tmp/deskew-125.zip && \
-  unzip /tmp/deskew-125.zip -d /opt && \
-  chmod +x /opt/Deskew/Bin/deskew &&\
-  rm /tmp/deskew-125.zip
-
-
 FROM ubuntu:cosmic as base-dist
 
+ENV DEBIAN_FRONTEND=noninteractive
 RUN \
-  . /etc/os-release && \
   apt-get update && \
-  DEBIAN_FRONTEND=noninteractive apt-get install --assume-yes --no-install-recommends \
+  apt-get install --assume-yes --no-install-recommends \
       python3 graphicsmagick pdftk-java vim \
       tesseract-ocr tesseract-ocr-fra tesseract-ocr-deu tesseract-ocr-eng \
-      libimage-exiftool-perl software-properties-common && \
-  apt-get clean && \
-  rm --recursive --force /var/lib/apt/lists/* /var/cache/*
-
-COPY --from=builder /opt/Deskew /opt/Deskew
-RUN \
-  . /etc/os-release && \
-  apt-get update && \
-  DEBIAN_FRONTEND=noninteractive apt-get install --assume-yes --no-install-recommends \
+      libimage-exiftool-perl software-properties-common \
       python3-pip python3-setuptools && \
-  python3 -m pip install PyYaml numpy scipy scikit-image opencv-python-headless && \
-  DEBIAN_FRONTEND=noninteractive apt-get auto-remove --assume-yes python3-pip python3-setuptools && \
+  python3 -m pip install PyYaml numpy scipy scikit-image opencv-python-headless deskew && \
+  apt-get auto-remove --assume-yes python3-pip python3-setuptools && \
   apt-get clean && \
   rm --recursive --force /var/lib/apt/lists/* /root/.cache /var/cache/*
 
@@ -75,18 +54,3 @@ RUN \
       tesseract-ocr-all && \
   apt-get clean && \
   rm --recursive --force /var/lib/apt/lists/* /var/cache/*
-
-
-FROM all as experimental
-
-RUN \
-  . /etc/os-release && \
-  apt-get update && \
-  apt-get install --assume-yes --no-install-recommends unpaper && \
-  add-apt-repository ppa:stephane-brunner/cosmic && \
-  apt-get update && \
-  apt-get install --assume-yes --no-install-recommends scantailor && \
-  (apt-get install --assume-yes --no-install-recommends scantailor-advanced || true) && \
-  (apt-get install --assume-yes --no-install-recommends scantailor-universal || true) && \
-  apt-get clean && \
-  rm --recursive --force /var/lib/apt/lists/* /var/cache/*

mypy.ini

@@ -3,23 +3,12 @@ python_version = 3.6
 
 disallow_untyped_calls = False
 
-disallow_untyped_defs=True
-check_untyped_defs=True
+disallow_untyped_defs = True
+check_untyped_defs = True
 strict_optional = True
 
 warn_no_return = True
-warn_redundant_casts=True
-warn_unused_ignores=True
-warn_unused_configs=True
-
-[mypy-numpy.*]
-ignore_missing_imports = True
-
-[mypy-scipy.*]
-ignore_missing_imports = True
-
-[mypy-skimage.*]
-ignore_missing_imports = True
-
-[mypy-cv2.*]
+warn_redundant_casts = True
+warn_unused_ignores = True
+warn_unused_configs = True
 ignore_missing_imports = True

process

@@ -14,11 +14,15 @@ import traceback
 from typing import cast, List, Dict, Any, Tuple, Union, Optional, Callable
 import yaml
 from skimage.measure import compare_ssim
+from skimage.color import rgb2gray
 import numpy as np
+# read, write, rotate, crop, sharpen, draw_line, find_line, find_contour
 import cv2
 from scipy.signal import find_peaks
+from deskew import determine_skew
 
 
+# dither, crop, append, repage
 CONVERT = ['gm', 'convert']
 
 
@@ -275,53 +279,56 @@ def crop(context: Context, margin_horizontal: int = 25, margin_vertical: int = 2
 
 
 @Process('level')
-def level(context: Context, config: Dict[str, Any]) -> np.ndarray:
+def level(context: Context) -> np.ndarray:
     img_yuv = cv2.cvtColor(context.image, cv2.COLOR_BGR2YUV)
 
-    if config['args']['level']:
-        min_p100 = 15
-        max_p100 = 85
+    level_ = context.config['args'].get('level')
+    if level_ is True:
+        min_p100 = 15.0
+        max_p100 = 85.0
+    elif level_ is False:
+        min_p100 = 0.0
+        max_p100 = 100.0
+    elif isinstance(level_, (float, int)):
+        min_p100 = 0.0 + level_
+        max_p100 = 100.0 - level_
+    elif context.config['args'].get('auto_level'):
+        img_yuv[:, :, 0] = cv2.equalizeHist(img_yuv[:, :, 0])
+        return cv2.cvtColor(img_yuv, cv2.COLOR_YUV2BGR)
+    else:
+        min_p100 = context.config['args'].get('min_level', 0.0)
+        max_p100 = context.config['args'].get('max_level', 100.0)
 
-        min_ = min_p100 / 100 * 255
-        max_ = max_p100 / 100 * 255
+    min_ = min_p100 / 100.0 * 255.0
+    max_ = max_p100 / 100.0 * 255.0
 
-        chanel_y = img_yuv[:, :, 0]
-        mins = np.zeros(chanel_y.shape)
-        maxs = np.zeros(chanel_y.shape) + 255
-
-        values = (chanel_y - min_) / (max_ - min_) * 255
-        img_yuv[:, :, 0] = np.minimum(maxs, np.maximum(mins, values))
-    else:
-        img_yuv[:, :, 0] = cv2.equalizeHist(img_yuv[:, :, 0])
+    chanel_y = img_yuv[:, :, 0]
+    mins = np.zeros(chanel_y.shape)
+    maxs = np.zeros(chanel_y.shape) + 255
 
+    values = (chanel_y - min_) / (max_ - min_) * 255
+    img_yuv[:, :, 0] = np.minimum(maxs, np.maximum(mins, values))
     return cv2.cvtColor(img_yuv, cv2.COLOR_YUV2BGR)
 
 
 @Process('deskew')
 def deskew(context: Context) -> None:
     image = context.get_masked()
-    contours = find_contours(image, 200)
-    if contours:
-        x, y, width, height = get_contour_to_crop(contours)
-        image = crop_image(image, x, y, width, height, (255, 255, 255))
-    source = tempfile.NamedTemporaryFile(suffix='.png')
-    cv2.imwrite(source.name, image)
-    out = output(['/opt/Deskew/Bin/deskew', source.name]).split('\n')
-    text = 'Skew angle found [deg]: '
-    out = [e for e in out if e.startswith(text)]
-    angle = float(out[0][len(text):])
+    grayscale = rgb2gray(image)
+    angle = determine_skew(grayscale)
     context.rotate(angle)
 
 
 @Process('docrop')
 def docrop(context: Context) -> None:
     # Margin in mm
-    marging_horizontal = 9
-    maring_vertical = 6
+    marging_horizontal = context.config['args'].get('marging_horizontal', 9)
+    maring_vertical = context.config['args'].get('maring_vertical', 6)
+    dpi = context.config['args'].get('dpi', 300)
     crop(
         context,
-        int(round(marging_horizontal / 10 / 2.51 * 300)),
-        int(round(maring_vertical / 10 / 2.51 * 300))
+        int(round(marging_horizontal / 10 / 2.51 * dpi)),
+        int(round(maring_vertical / 10 / 2.51 * dpi))
     )
 
 
@@ -575,15 +582,15 @@ def transform(
         if os.path.exists(mask_file):
             context.mask = cv2.imread(mask_file)
             context.init_mask()
-        level(context, config)
+        level(context)
         deskew(context)
         docrop(context)
         sharpen(context)
         dither(context)
         autorotate(context)
 
         # Is empty ?
-        contours = find_contours(context.get_masked())
+        contours = find_contours(context.get_masked(), 70)
         if not contours:
             print("Ignore image with no content: {}".format(img))
             continue
@@ -728,13 +735,14 @@ def split(config: Dict[str, Any], step: Dict[str, Any], root_folder: str) -> Dic
                     page_pos = 0
 
                 save(root_folder, img2, '{}-split'.format(context.get_process_count()))
-                marging_horizontal = 10
-                maring_vertical = 7
+                marging_horizontal = context.config['args'].get('marging_horizontal', 9)
+                maring_vertical = context.config['args'].get('maring_vertical', 6)
+                dpi = context.config['args'].get('dpi', 300)
                 context.image = cv2.imread(img2)
                 crop(
                     context,
-                    int(round(marging_horizontal / 10 / 2.51 * 300)),
-                    int(round(maring_vertical / 10 / 2.51 * 300)),
+                    int(round(marging_horizontal / 10 / 2.51 * dpi)),
+                    int(round(maring_vertical / 10 / 2.51 * dpi)),
                 )
                 img3_file = tempfile.NamedTemporaryFile(suffix='.png')
                 img3 = img3_file.name

scan_to_paperless/__init__.py

@@ -0,0 +1,19 @@
+import os.path
+import yaml
+
+CONFIG_FILENAME = 'scan-to-paperless.yaml'
+
+if 'APPDATA' in os.environ:
+    CONFIG_FOLDER = os.environ['APPDATA']
+elif 'XDG_CONFIG_HOME' in os.environ:
+    CONFIG_FOLDER = os.environ['XDG_CONFIG_HOME']
+else:
+    CONFIG_FOLDER = os.path.expanduser('~/.config')
+
+CONFIG_PATH = os.path.join(CONFIG_FOLDER, CONFIG_FILENAME)
+
+def get_config():
+    if os.path.exists(CONFIG_PATH):
+        with open(CONFIG_PATH, encoding='utf-8') as f:
+            return yaml.safe_load(f.read())
+    return {}