Low-level WSGI handling now performed by Werkzeug (and also Jinja2 fo…

…r templating needs)

Dockerfile

@@ -1,6 +1,6 @@
 FROM python:3.8-slim-buster
-
-RUN apt -qq update && \
+RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections && \
+    apt -qq update && \
     apt -qq -y --no-install-recommends install \
         apt-transport-https \
 	gnupg \
@@ -9,29 +9,38 @@ RUN apt -qq update && \
 	wget && \
     add-apt-repository "deb http://ftp.us.debian.org/debian stretch main" && \
     wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add - && \
-    add-apt-repository "deb https://artifacts.elastic.co/packages/5.x/apt stable main" && \
+    add-apt-repository "deb https://artifacts.elastic.co/packages/7.x/apt stable main" && \
     apt -qq update && \
     mkdir /usr/share/man/man1 && \
     apt -q -y --no-install-recommends install \
        antiword \
+       bzip2 \
        cron \
        curl \
-       mariadb-client \
-       mariadb-server \
-       mediawiki \
        elasticsearch \
        emacs24-nox \
+       file \
+       g++ \
        gcc \
        git \
        imagemagick \
+       libfontconfig1-dev \
+       libjpeg-dev \
+       liblcms2-dev \
+       libopenjp2-7-dev \
        libreoffice \
+       libtiff-dev \
        libtiff-tools \
        libxml2-dev \
        libxslt1-dev \
+       locales \    
+       make \
+       mariadb-client \
+       mariadb-server \
        mediawiki \
        nginx \
        openjdk-8-jre-headless \
-       poppler-utils \
+       pkg-config \
        procps \
        python3-dev \
        python3-venv \
@@ -41,7 +50,19 @@ RUN apt -qq update && \
        tesseract-ocr-swe \
        uwsgi \
        uwsgi-plugin-python3 \
+       xz-utils \
        zlib1g-dev && \
+   wget https://poppler.freedesktop.org/poppler-0.56.0.tar.xz && \
+       xz -d poppler-0.56.0.tar.xz && \
+       tar xvf poppler-0.56.0.tar && \
+       cd poppler-0.56.0 && \
+       ./configure && \
+       make install && \
+       cd .. && \
+       rm -r poppler-0.56.0 && \
+       ldconfig && \
+    wget https://github.com/htacg/tidy-html5/releases/download/5.4.0/tidy-5.4.0-64bit.deb && \
+       dpkg -i tidy-5.4.0-64bit.deb && \
     mkdir /opt/fuseki && \
        cd /opt/fuseki && \
        (curl -s http://www-eu.apache.org/dist/jena/binaries/apache-jena-fuseki-3.13.1.tar.gz | tar -xvz --strip-components=1 ) && \
@@ -50,16 +71,19 @@ RUN apt -qq update && \
 WORKDIR /usr/share/ferenda
 COPY requirements.txt . 
 RUN python3.7 -m venv .virtualenv && \
+    ./.virtualenv/bin/pip install wheel && \
     ./.virtualenv/bin/pip install -r requirements.txt
 
-EXPOSE 80 3330 9001 9200 
+EXPOSE 80 8000 3030 9001 9200 
 COPY docker /tmp/docker
 RUN mv /tmp/docker/supervisord.conf /etc/supervisor/conf.d/supervisord.conf && \
     mv /tmp/docker/nginx.conf /etc/nginx/sites-enabled/default && \
-    mv /tmp/docker/ferenda.ttl /opt/fuseki/run/configuration/
+    mv /tmp/docker/ferenda.ttl /opt/fuseki/run/configuration/ && \
+    mv /tmp/docker/locale.gen /etc/locale.gen && locale-gen
 COPY . .
+# mv /tmp/docker/elasticsearch-jvm.options /etc/elasticsearch/jvm.options && \
 
 ENTRYPOINT ["/bin/bash", "/tmp/docker/setup.sh"]
 CMD ["/usr/bin/supervisord"] # starts nginx, elasticsearch, fuseki, cron etc
 
-# then: docker run -d -v ferendafiles:/usr/share/ferenda  -p 80:80 -p 3330:3330 -p 9001:9001 -p 9200:9200 <imageid>
+# then: docker run --name ferenda -d -v c:/docker/ferenda:/usr/share/ferenda/site  -p 81:80 -p 3030:3030 -p 9001:9001 -p 9200:9200 -p 8000:8000 <imageid>

docker/elasticsearch-jvm.options

@@ -0,0 +1,76 @@
+## JVM configuration
+
+################################################################
+## IMPORTANT: JVM heap size
+################################################################
+##
+## You should always set the min and max JVM heap
+## size to the same value. For example, to set
+## the heap to 4 GB, set:
+##
+## -Xms4g
+## -Xmx4g
+##
+## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html
+## for more information
+##
+################################################################
+
+# Xms represents the initial size of total heap space
+# Xmx represents the maximum size of total heap space
+
+-Xms4g
+-Xmx4g
+
+################################################################
+## Expert settings
+################################################################
+##
+## All settings below this section are considered
+## expert settings. Don't tamper with them unless
+## you understand what you are doing
+##
+################################################################
+
+## GC configuration
+-XX:+UseConcMarkSweepGC
+-XX:CMSInitiatingOccupancyFraction=75
+-XX:+UseCMSInitiatingOccupancyOnly
+
+## G1GC Configuration
+# NOTE: G1GC is only supported on JDK version 10 or later.
+# To use G1GC uncomment the lines below.
+# 10-:-XX:-UseConcMarkSweepGC
+# 10-:-XX:-UseCMSInitiatingOccupancyOnly
+# 10-:-XX:+UseG1GC
+# 10-:-XX:G1ReservePercent=25
+# 10-:-XX:InitiatingHeapOccupancyPercent=30
+
+## JVM temporary directory
+-Djava.io.tmpdir=${ES_TMPDIR}
+
+## heap dumps
+
+# generate a heap dump when an allocation from the Java heap fails
+# heap dumps are created in the working directory of the JVM
+-XX:+HeapDumpOnOutOfMemoryError
+
+# specify an alternative path for heap dumps; ensure the directory exists and
+# has sufficient space
+-XX:HeapDumpPath=/var/lib/elasticsearch
+
+# specify an alternative path for JVM fatal error logs
+-XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log
+
+## JDK 8 GC logging
+8:-XX:+PrintGCDetails
+8:-XX:+PrintGCDateStamps
+8:-XX:+PrintTenuringDistribution
+8:-XX:+PrintGCApplicationStoppedTime
+8:-Xloggc:/var/log/elasticsearch/gc.log
+8:-XX:+UseGCLogFileRotation
+8:-XX:NumberOfGCLogFiles=32
+8:-XX:GCLogFileSize=64m
+
+# JDK 9+ GC logging
+9-:-Xlog:gc*,gc+age=trace,safepoint:file=/var/log/elasticsearch/gc.log:utctime,pid,tags:filecount=32,filesize=64m

docker/locale.gen

@@ -0,0 +1,4 @@
+# this should be copied to /etc/locale.gen, and then locale-gen should
+# run to create the following locales (not setting anyone to default)
+en_US.UTF-8 UTF-8
+sv_SE.UTF-8 UTF-8

docker/setup.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 set -e
 
+cd /usr/share/ferenda
+
 if [ -f site/ferenda.ini ]; then
     echo "site/ferenda.ini exists, not setting up a new site"
 fi 

docker/supervisord.conf

@@ -21,8 +21,7 @@ command=/opt/fuseki/fuseki-server
 
 [program:elasticsearch]
 # port 9200
-env=ES_JAVA_OPTS="-Xms2g -Xmx2g"
-command=/usr/share/elasticsearch/bin/elasticsearch -Edefault.path.conf=/etc/elasticsearch -Edefault.path.data=/var/lib/elasticsearch -Edefault.path.logs=/var/log/elasticsearch
+command=/usr/share/elasticsearch/bin/elasticsearch
 user=elasticsearch
 
 [program:uwsgi]

ferenda-setup.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
 from ferenda import manager
-manager.runsetup()
+if __name__ == '__main__':
+    manager.runsetup()