process

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# TODO
# - Do blur only one times
# - other layouts
# - fix content detection for romande energie
# - add http://www.graphicsmagick.org/GraphicsMagick.html#details-sharpen

import datetime
import glob
import os
import random
import re
import shutil
import subprocess
import time
import traceback
import yaml


mogrify = ['gm', 'mogrify']
convert = ['gm', 'convert']


def add_intermediate_error(config, config_file_name, error, tb):
    if 'intermediate_error' not in config:
        config['intermediate_error'] = []
    config['intermediate_error'].append({
        'error': error,
        'traceback': tb,
    })
    with open(config_file_name, 'w') as config_file:
        config_file.write(yaml.safe_dump(config, default_flow_style=False))


def call(cmd, cmd2=None, **kwargs):
    print(' '.join(cmd) if isinstance(cmd, list) else cmd)
    subprocess.check_output(cmd, stderr=subprocess.PIPE, **kwargs)


def output(cmd, cmd2=None, **kwargs):
    print(' '.join(cmd) if isinstance(cmd, list) else cmd)
    return subprocess.check_output(cmd, stderr=subprocess.PIPE, **kwargs)


def crop(img, marging_horizontal=25, maring_vertical=25, rm_top=25, rm_left=15, noise='7', blur='0x3', fuzz='25%'):
    """
    Margin in px
    """
    o_w, o_h = [int(e) for e in output(convert + [img, '-format', '%w %h', 'info:-']).strip().split(b' ')]
    w, h, x, y = [int(e) for e in re.split(b'[x+]', output(convert + [
        img, '-crop', '{}x{}+{}+{}'.format(o_w - rm_left, o_h - rm_top, rm_left, rm_top),
        '+repage', '-virtual-pixel', 'edge', '-noise', noise, '-blur', blur, '-fuzz', fuzz, '-format', '%@', 'info:-'
    ]).strip())]
    if h <= o_h and w <= o_w:
        call(convert + [img, '-background', '#ffffff', '-crop', '{}x{}+{}+{}'.format(
            w + 2 * marging_horizontal, h + 2 * maring_vertical, x - marging_horizontal + rm_left, y - maring_vertical + rm_top
        ), '+repage', img])


def save(root_folder, img, folder):
    if os.environ.get("PROGRESS") == "TRUE":
        if not os.path.exists(os.path.join(root_folder, folder)):
            os.makedirs(os.path.join(root_folder, folder))
        shutil.copyfile(img, os.path.join(root_folder, folder, os.path.basename(img)))


def transform(config, root_folder):
    del config['intermediate_error']

    images = list()
    if not os.path.exists(os.path.join(root_folder, 'process')):
        os.makedirs(os.path.join(root_folder, 'process'))
#    if not os.path.exists(os.path.join(root_folder, 'scantailor')):
#        os.makedirs(os.path.join(root_folder, 'scantailor'))
#    if not os.path.exists(os.path.join(root_folder, 'scantailor1200')):
#        os.makedirs(os.path.join(root_folder, 'scantailor1200'))
#    if not os.path.exists(os.path.join(root_folder, 'unpaper')):
#        os.makedirs(os.path.join(root_folder, 'unpaper'))
#    if not os.path.exists(os.path.join(root_folder, 'deskew')):
#        os.makedirs(os.path.join(root_folder, 'deskew'))
    for img in config['images']:
        shutil.copyfile(os.path.join(root_folder, img), os.path.join(root_folder, 'process', os.path.basename(img)))
        img = os.path.join(root_folder, 'process', os.path.basename(img))
        # force-cleanup
        if config['args']['level']:
            call(mogrify + ['-level', '15%,1,85%', img])
            save(root_folder, img, '1-cleanup')

        marging_horizontal=100
        maring_vertical=100
        crop(img, round(marging_horizontal/10/2.51*300), round(maring_vertical/10/2.51*300))
        save(root_folder, img, '2-precrop')

        try:
            call(['/opt/Deskew/Bin/deskew', '-b', 'FFFFFF', '-o', img + '.d.png', img])
            shutil.move(img + '.d.png', img)
            save(root_folder, img, '3-deskew')
        except subprocess.CalledProcessError as e:
            print(e)
            add_intermediate_error(
                config, config_file_name, e, traceback.format_exc().split('\n')
            )

        # Margin in mm
        marging_horizontal=10
        maring_vertical=7
        crop(img, round(marging_horizontal/10/2.51*300), round(maring_vertical/10/2.51*300))
        save(root_folder, img, '4-crop')

        # sharpen
        call(mogrify + ['-sharpen', '0x2', img])
        save(root_folder, img, '5-sharpen')
        # dither
        #call(mogrify + ['+dither', img])
        # auto-rotate
        try:
            orientation = 0
            orientation = output(['tesseract', img, '-', '--psm', '0', '-l', 'osd']).decode('utf-8')
#            orientation = output(['tesseract', img, '-', '--psm', '0', '-l', 'fra+eng']).decode('utf-8')
            orientation = [e for e in orientation.splitlines() if 'Orientation in degrees' in e]
            orientation = '-' + orientation[0].split()[3]

            call(mogrify + ['-rotate', orientation, img])
            save(root_folder, img, '6-autorotate')
        except subprocess.CalledProcessError as e:
            add_intermediate_error(
                config, config_file_name, e, traceback.format_exc().split('\n')
            )

        # is empty ?
        w, h = [int(e) for e in output(convert + [img, '-format', '%w %h', 'info:-']).split(b' ')]
        if output(
            convert + [img, '-noise', '7', '-blur', '0x5', '-level', '20%,1,80%', '-draw', 'point {},{}'.format(w/2, h/2), '-format', '%@', 'info:-'])[0:4] == b'1x1+':
            print("Ignore image with no content: {}".format(img))
            continue

 #       call([
 #           'scantailor-cli', '--dpi=300', '--content-detection=normal',
 #           '--output-dpi=300', '--color-mode=color_grayscale',
 #           img, os.path.join(root_folder, 'scantailor')
 #       ])
 #       call([
 #           'scantailor-cli', '--dpi=300', '--output-dpi=1200',
 #           img, os.path.join(root_folder, 'scantailor1200')
 #       ])
 #       call([
 #           'unpaper', img, os.path.join(root_folder, 'unpaper', os.path.basename(img))
 #       ])
 #       call([
 #           '/opt/Deskew/Bin/deskew', '-o', os.path.join(root_folder, 'deskew', os.path.basename(img)), img
 #       ])
        img2 = os.path.join(root_folder, os.path.basename(img))
        call(convert + [img, img2])
        images.append(img2)

    return images

def finalise(config, root_folder):

    full_name = config['full_name']
    destination = config['destination']

    if os.path.exists(destination):
        return

    images = config['transformed_images']

    if config['args']['append_credit_card']:
        images2 = []
        for img in images:
            if os.path.exists(img):
                    images2.append(img)

        call(convert + images2 + ['-background', '#ffffff', '-append', os.path.join(root_folder, 'final.png')])
        images = [os.path.join(root_folder, 'final.png')]

    pdf = []
    for img in images:
        if os.path.exists(img):
            name = os.path.splitext(os.path.basename(img))[0]
            #call('tesseract -l fra+eng {} stdout pdf > {}'.format(
            #    img, os.path.join(root_folder, '{}.pdf'.format(name))
            #), shell=True)
            call(convert + [img, os.path.join(root_folder, '{}.pdf'.format(name))])
            pdf.append(os.path.join(root_folder, '{}.pdf'.format(name)))

    call(['pdftk'] + pdf + ['output', destination, 'compress'])
    call(['exiftool', '-overwrite_original_in_place', '-Title=' + full_name, destination])

    shutil.rmtree(root_folder)

#    call(['chmod', '-R', 'go+rw', '/destination'])

mtime = time.ctime(os.path.getmtime('/opt/process'))

print('Welcome to scaned images document to paperless.')
print('Script time: ' + mtime)
while True:
    print("Scan")
    for config_file_name in glob.glob('/source/*/config.yaml'):
        if not os.path.exists(config_file_name):
            continue

        print(re.sub(r'.', '-', config_file_name))
        print(config_file_name)
        root_folder = os.path.dirname(config_file_name)

        with open(config_file_name) as config_file:
            config = yaml.safe_load(config_file.read())

        if os.path.exists(os.path.join(root_folder, 'error.yaml')) and config.get('mtime') == mtime:
            print("On error")
            continue

        if os.path.exists(os.path.join(root_folder, 'error.yaml')):
            os.apth.remove(os.path.join(root_folder, 'error.yaml'))

        try:
            allready_proceed = True
            if 'transformed_images' not in config:
                allready_proceed = False
            else:
                for img in config['transformed_images']:
                    if not os.path.exists(img):
                        allready_proceed = False

            if allready_proceed:
                if not os.path.exists(os.path.join(root_folder, 'REMOVE_TO_CONTINUE')):
                    print("Finalise")

                    config['mtime'] = mtime
                    with open(config_file_name, 'w') as config_file:
                        config_file.write(yaml.safe_dump(config, default_flow_style=False))

                    finalise(config, root_folder)
                else:
                    print("Still waiting")

            else:
                ok = True
                for img in config['images']:
                    if not os.path.exists(os.path.join(root_folder, img)):
                        ok = False

                if ok:
                    print("Transform")

                    config['mtime'] = mtime
                    with open(config_file_name, 'w') as config_file:
                        config_file.write(yaml.safe_dump(config, default_flow_style=False))

                    config['transformed_images'] = transform(config, root_folder)

                    with open(config_file_name, 'w') as config_file:
                        config_file.write(yaml.safe_dump(config, default_flow_style=False))

                    with open(os.path.join(root_folder, 'REMOVE_TO_CONTINUE'), 'w') as waiting_file:
                        pass

        except Exception as e:
            print(e)
            with open(os.path.join(root_folder, 'error.yaml'), 'w') as error_file:
                error_file.write(yaml.safe_dump({
                    'error': e,
                    'traceback': traceback.format_exc().split('\n'),
                }, default_flow_style=False))

    print("Sleep")
    time.sleep(30)