Merge pull request #126 from Skaiste/server-docker

Server docker

docs/rest_server.rst

@@ -45,8 +45,13 @@ Loading voices
 --------------
 
 In order for the users to get speech of any voice, the database must
-know about the voices and their location. To set a voice up, the server
-must be installed and in the ``idlak-server`` directory a command has to
+know about the voices and their location. 
+The current voice configuration file ``voiceconfig.json`` has prebuilt 
+voice information. Other voices can be added to this file to be loaded 
+to the database on startup of the server.
+
+To set a voice up in a different way, the server
+must be installed and in the ``idlak-server`` directory and a command has to
 be run:
 
 ::
@@ -69,7 +74,12 @@ information and get processed speech of these voices.
 
 Deployment
 ----------
-Follow `this link <http://flask.pocoo.org/docs/1.0/deploying/>`_ for
+For deployment with docker, docker images with and without cuda can be 
+found in ``idlak-server/docker`` directory.
+The server is run on port 80.
+
+
+Follow `this link <http://flask.pocoo.org/docs/1.0/deploying/>`_ for 
 information on how to deploy a Flask application.
 
 

idlak-server/app/__init__.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import sys
 import subprocess
 import shutil
 from logging.handlers import RotatingFileHandler
@@ -44,6 +45,7 @@ def create_app(config_name):
     with app.app_context():
         from app import models, endpoints, reqlogging       # noqa
         from app.models.user import User                    # noqa
+        from app.models.voice import Voice                    # noqa
 
         # if database is not created
         if not os.path.isfile(app.config['DATABASE_NAME']+'.db'):
@@ -54,6 +56,16 @@ def create_app(config_name):
             admin_user = User.new_user_full('admin', 'admin', True)
             app.logger.info("An initial admin user has been created: {}"
                             .format(admin_user))
+
+        # load all voices from config
+        if 'VOICE_CONFIG' in app.config:
+            for v in app.config['VOICE_CONFIG']['voices']:
+                voice = Voice.new_voice(v['vid'], v['name'], v['lang'], v['acc'],
+                                        v['gender'], v['dir'])
+                if type(voice) == dict and 'error' in voice and 'Voice already exists' not in voice['error']:
+                    app.logger.error(voice['error'])
+                    sys.exit()
+            app.logger.info("Voices from configuration file have been loaded")
 
     # url endpoints
     from app.endpoints.auth import Auth, Auth_Expire

idlak-server/app/endpoints/speech.py

@@ -15,8 +15,9 @@
 from flask_jwt_simple import jwt_required
 from app.models.voice import Voice
 
+idlakdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
 
-sys.path.append('../src/')
+sys.path.append(os.path.join(idlakdir, 'src'))
 from pyIdlak import TangleVoice         # noqa
 
 spch_parser = reqparser.RequestParser()

idlak-server/app/reqparser.py

@@ -23,17 +23,17 @@ class RequestParser():
     def __init__(self):
         self.reqschema = {
             # content-type: application/json
-            "json": RequestSchema(strict=True),
+            "json": RequestSchema(),
             # content-type: multipart/form-data
-            "form": RequestSchema(strict=True),
+            "form": RequestSchema(),
             # arguments from url
-            "args": RequestSchema(strict=True),
+            "args": RequestSchema(),
             # application/x-www-form-urlencoded, form-data, arguments from url
-            "values": RequestSchema(strict=True),
+            "values": RequestSchema(),
             # content-type: text/plain
-            "text": RequestSchema(strict=True),
-            "headers": RequestSchema(strict=True),
-            "cookies": RequestSchema(strict=True)
+            "text": RequestSchema(),
+            "headers": RequestSchema(),
+            "cookies": RequestSchema()
             }
         self.help = {}
 

idlak-server/config.ini

@@ -4,6 +4,9 @@ LOGGING = INFO
 LOG_FILE = idlak-server.log
 AUTHENTICATION = True
 DATABASE_NAME = app
+HOST = localhost
+PORT = 5000
+VOICE_CONFIG = voiceconfig.json
 
 [JWT]
 TOKEN_EXPIRATION_DELTA = 30

idlak-server/config.py

@@ -1,6 +1,7 @@
 import os
 import configparser
 import uuid
+import json
 from datetime import timedelta
 
 basedir = os.path.abspath(os.path.dirname(__file__))
@@ -26,6 +27,14 @@ def load_config_file(conf, config_name):
     # correct database value
     conf['SQLALCHEMY_DATABASE_URI'] = (conf['SQLALCHEMY_DATABASE_URI']
                                        .format(conf['DATABASE_NAME']))
+    # voice configuration
+    if 'VOICE_CONFIG' in conf:
+        vcfn = os.path.realpath(os.path.join(basedir, conf['VOICE_CONFIG']))
+        if os.path.isfile(vcfn):
+            with open(vcfn, "r") as vcf:
+               conf['VOICE_CONFIG'] = json.loads(vcf.read())
+            for v in conf['VOICE_CONFIG']['voices']:
+                v['dir'] = os.path.realpath(os.path.join(basedir, v['dir']))
     # correct AUTHENTICATION value
     if 'AUTHORIZATION' in conf:
         auth = conf['AUTHORIZATION']

idlak-server/docker/centos-cuda/Dockerfile

@@ -0,0 +1,76 @@
+FROM centos:7
+
+ENV CPU_CORE 4
+
+# Activate systemd. Taken from https://hub.docker.com/_/centos
+ENV container docker
+RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == \
+systemd-tmpfiles-setup.service ] || rm -f $i; done); \
+rm -f /lib/systemd/system/multi-user.target.wants/*;\
+rm -f /etc/systemd/system/*.wants/*;\
+rm -f /lib/systemd/system/local-fs.target.wants/*; \
+rm -f /lib/systemd/system/sockets.target.wants/*udev*; \
+rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \
+rm -f /lib/systemd/system/basic.target.wants/*;\
+rm -f /lib/systemd/system/anaconda.target.wants/*;
+VOLUME [ "/sys/fs/cgroup" ]
+
+RUN yum update -y
+RUN yum groupinstall -y "Development Tools" "System Tools"
+RUN yum install -y \
+    git bzip2 wget subversion which sox \
+    gcc-c++ make automake autoconf zlib-devel atlas-static \
+     python python3-devel cmake tcsh pcre-devel
+RUN yum remove -y swig
+RUN yum install -y http://mirror.centos.org/centos/7/extras/x86_64/Packages/swig3-3.0.12-17.el7.x86_64.rpm
+
+# install cuda
+RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm dkms
+RUN wget https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-repo-rhel7-10.0.130-1.x86_64.rpm
+RUN rpm -i cuda-repo-*.rpm
+RUN yum install -y cuda
+
+WORKDIR /usr/local/
+# Use the newest idlak version
+RUN git clone https://github.com/Idlak/idlak.git
+WORKDIR /usr/local/idlak
+RUN git checkout server-docker
+RUN git pull
+
+WORKDIR /usr/local/idlak/tools
+RUN mkdir python
+RUN touch python/.use_default_python
+RUN extras/check_dependencies.sh
+RUN make -j $CPU_CORE
+RUN ./install_idlak.sh
+
+WORKDIR /usr/local/idlak/src
+ENV SWIG=/usr/bin/swig
+RUN ./configure --shared --cudatk-dir=/usr/local/cuda-10.2/
+RUN make clean
+RUN make depend -j $CPU_CORE
+RUN make -j $CPU_CORE
+RUN make ext -j $CPU_CORE
+
+WORKDIR /usr/local/idlak/idlak-server
+RUN python3 -m pip install -r requirements.txt
+
+EXPOSE 80
+
+# setup nginx
+RUN yum install -y epel-release
+RUN yum install -y nginx
+
+# setup idlak service
+RUN cp docker/wsgi.ini ./.
+RUN cp docker/idlak.service /etc/systemd/system/.
+RUN cp docker/idlak.conf /etc/nginx/conf.d/idlak.conf
+RUN systemctl enable idlak
+
+# continue setting up nginx
+RUN usermod -a -G root nginx
+RUN chmod 710 /usr/local/idlak
+RUN systemctl enable nginx
+
+CMD ["/usr/sbin/init"]
+

idlak-server/docker/centos/Dockerfile

@@ -0,0 +1,70 @@
+FROM centos:7
+
+ENV CPU_CORE 4
+
+# Activate systemd. Taken from https://hub.docker.com/_/centos
+ENV container docker
+RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == \
+systemd-tmpfiles-setup.service ] || rm -f $i; done); \
+rm -f /lib/systemd/system/multi-user.target.wants/*;\
+rm -f /etc/systemd/system/*.wants/*;\
+rm -f /lib/systemd/system/local-fs.target.wants/*; \
+rm -f /lib/systemd/system/sockets.target.wants/*udev*; \
+rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \
+rm -f /lib/systemd/system/basic.target.wants/*;\
+rm -f /lib/systemd/system/anaconda.target.wants/*;
+VOLUME [ "/sys/fs/cgroup" ]
+
+RUN yum update -y
+RUN yum groupinstall -y "Development Tools" "System Tools"
+RUN yum install -y \
+    git bzip2 wget subversion which sox \
+    gcc-c++ make automake autoconf zlib-devel atlas-static \
+     python python3-devel cmake tcsh pcre-devel
+RUN yum remove -y swig
+RUN yum install -y http://mirror.centos.org/centos/7/extras/x86_64/Packages/swig3-3.0.12-17.el7.x86_64.rpm
+
+WORKDIR /usr/local/
+# Use the newest idlak version
+RUN git clone https://github.com/Idlak/idlak.git
+WORKDIR /usr/local/idlak
+RUN git checkout server-docker
+RUN git pull
+
+WORKDIR /usr/local/idlak/tools
+RUN mkdir python
+RUN touch python/.use_default_python
+RUN extras/check_dependencies.sh
+RUN make -j $CPU_CORE
+RUN ./install_idlak.sh
+
+WORKDIR /usr/local/idlak/src
+ENV SWIG=/usr/bin/swig
+RUN ./configure --shared
+RUN make clean
+RUN make depend -j $CPU_CORE
+RUN make -j $CPU_CORE
+RUN make ext -j $CPU_CORE
+
+WORKDIR /usr/local/idlak/idlak-server
+RUN python3 -m pip install -r requirements.txt
+
+EXPOSE 80
+
+# setup nginx
+RUN yum install -y epel-release
+RUN yum install -y nginx
+
+# setup idlak service
+RUN cp docker/wsgi.ini ./.
+RUN cp docker/idlak.service /etc/systemd/system/.
+RUN cp docker/idlak.conf /etc/nginx/conf.d/idlak.conf
+RUN systemctl enable idlak
+
+# continue setting up nginx
+RUN usermod -a -G root nginx
+RUN chmod 710 /usr/local/idlak
+RUN systemctl enable nginx
+
+CMD ["/usr/sbin/init"]
+

idlak-server/docker/idlak.conf

@@ -0,0 +1,9 @@
+server {
+    listen 80;
+    server_name localhost;
+
+    location / {
+        include uwsgi_params;
+        uwsgi_pass unix:/usr/local/idlak/idlak-server/idlak-wsgi.sock;
+    }
+}

idlak-server/docker/idlak.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=uWSGI instance to serve Idlak REST API server
+After=network.target
+
+[Service]
+User=root
+Group=nginx
+WorkingDirectory=/usr/local/idlak/idlak-server
+ExecStart=/usr/local/bin/uwsgi --ini wsgi.ini
+
+[Install]
+WantedBy=multi-user.target

idlak-server/docker/wsgi.ini

@@ -0,0 +1,15 @@
+[uwsgi]
+module = runserver
+callable = app
+
+master = true
+processes = 5
+
+socket = idlak-wsgi.sock
+chmod-socket = 660
+vacuum = true
+
+die-on-term = true
+
+req-logger = file:/usr/local/idlak/idlak-server/access.log
+logger = file:/usr/local/idlak/idlak-server/error.log

idlak-server/requirements.txt

@@ -11,3 +11,4 @@ python-pcre==0.7
 requests==2.21.0
 SQLAlchemy==1.2.10
 Werkzeug==0.14.1
+uwsgi==2.0.18

idlak-server/runserver.py

@@ -1,6 +1,13 @@
+import os
 from app import create_app
 
-app = create_app('config.ini')
+cdir = os.path.split(os.path.abspath(__file__))[0]
+
+config_fn = os.path.join(cdir, 'config.ini')
+
+app = create_app(config_fn)
 
 if __name__ == '__main__':
-    app.run()
+    app.run(host=app.config["HOST"], port=app.config["PORT"])
+else:
+    application = app

idlak-server/setup-server.sh

@@ -1,14 +1,11 @@
 #!/bin/bash
+set -v
 TOREPLACE="from pip import main"
 REPLACETO="from pip._internal import main"
 
-echo "virtualenv venv"
-virtualenv venv
-echo "source venv/bin/activate"
+virtualenv -p python3 venv
 source venv/bin/activate
-echo "replace \"$TOREPLACE\" \"$REPLACETO\" -- venv/bin/pip3"
 replace "$TOREPLACE" "$REPLACETO" -- venv/bin/pip3
-echo "venv/bin/pip3 install -r requirements.txt"
-venv/bin/pip3 install -r requirements.txt
-echo "export FLASK_APP=runserver.py"
+venv/bin/python3 -m pip install -r requirements.txt
 export FLASK_APP=runserver.py
+set +v

idlak-server/voiceconfig.json

@@ -0,0 +1,20 @@
+{
+    "voices": [
+        {
+            "name": "en_ga_slt",
+            "gender": "female",
+            "lang": "en",
+            "acc": "ga",
+            "vid": "slt",
+            "dir": "../idlak-voices/tangle/en/ga/slt"
+        },
+        {
+            "name": "ro_ro_bas",
+            "gender": "female",
+            "lang": "ro",
+            "acc": "ro",
+            "vid": "bas",
+            "dir": "../idlak-voices/tangle/ro/ro/bas"
+        }
+    ]
+}