Rest api config to /etc/jasmin/rest-api.cfg

docker/Dockerfile.restapi

@@ -6,16 +6,18 @@ MAINTAINER Jookies LTD <jasmin@jookies.net>
 RUN groupadd -r jasmin && useradd -r -g jasmin jasmin
 
 # Install requirements
-RUN apt-get update && apt-get install -y \
-    libffi-dev \
-    libssl-dev \
-    # Run python with jemalloc
-    # More on this:
-    # - https://zapier.com/engineering/celery-python-jemalloc/
-    # - https://paste.pics/581cc286226407ab0be400b94951a7d9
-    libjemalloc2
-
-RUN apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN apt-get update; \
+    apt-get install -y --no-install-recommends \
+        libffi-dev \
+        libssl-dev \
+        # Run python with jemalloc
+        # More on this:
+        # - https://zapier.com/engineering/celery-python-jemalloc/
+        # - https://paste.pics/581cc286226407ab0be400b94951a7d9
+        libjemalloc2 \
+    ; \
+    apt-get clean; \
+    rm -rf /var/lib/apt/lists/*
 
 # Run python with jemalloc
 ENV LD_PRELOAD /usr/lib/x86_64-linux-gnu/libjemalloc.so.2
@@ -26,15 +28,20 @@ ENV RESOURCE_PATH ${CONFIG_PATH}/resource
 ENV STORE_PATH ${CONFIG_PATH}/store
 ENV LOG_PATH /var/log/jasmin
 
-RUN mkdir -p ${RESOURCE_PATH} ${STORE_PATH} ${LOG_PATH}
+RUN mkdir -p ${CONFIG_PATH} ${RESOURCE_PATH} ${STORE_PATH} ${LOG_PATH}
 RUN chown jasmin:jasmin ${CONFIG_PATH} ${RESOURCE_PATH} ${STORE_PATH} ${LOG_PATH}
 
 WORKDIR /build
 
+RUN pip install --upgrade pip
+
 COPY . .
 
 RUN pip install .
 
+# Clean build dir
+RUN rm -rf /build
+
 # For RestAPI MODE
 RUN pip install gunicorn
 
@@ -45,26 +52,17 @@ COPY misc/config/resource ${RESOURCE_PATH}
 
 WORKDIR /etc/jasmin
 
-# Default Redis and RabbitMQ connections
+# Default Jasmin, Redis, and RabbitMQ connections
+ENV JCLI_BIND '0.0.0.0'
 ENV AMQP_BROKER_HOST 'rabbitmq'
 ENV AMQP_BROKER_PORT 5672
 ENV REDIS_CLIENT_HOST 'redis'
 ENV REDIS_CLIENT_PORT 6379
 
-# For RestAPI MODE
-ENV RESTAPI_MODE 0
-ENV RESTAPI_OLD_HTTP_HOST '127.0.0.1'
-
-ENV ENABLE_PUBLISH_SUBMIT_SM_RESP 0
-
-# Change binding host for jcli
-RUN sed -i '/\[jcli\]/a bind=0.0.0.0' ${CONFIG_PATH}/jasmin.cfg
-# Change binding port for redis, and amqp
-RUN sed -i "/\[redis-client\]/a port=$REDIS_CLIENT_PORT" ${CONFIG_PATH}/jasmin.cfg
-RUN sed -i "/\[amqp-broker\]/a port=$AMQP_BROKER_PORT" ${CONFIG_PATH}/jasmin.cfg
-# Change binding host for redis, and amqp
-RUN sed -i "/\[redis-client\]/a host=$REDIS_CLIENT_HOST" ${CONFIG_PATH}/jasmin.cfg
-RUN sed -i "/\[amqp-broker\]/a host=$AMQP_BROKER_HOST" ${CONFIG_PATH}/jasmin.cfg
+# For RestAPI http mode
+ENV RESTAPI_HTTP_MODE 0
+# For RestAPI Celery worker mode
+ENV RESTAPI_WORKER_MODE 0
 
 EXPOSE 2775 8990 1401 8080
 VOLUME ["/var/log/jasmin", "/etc/jasmin", "/etc/jasmin/store"]

docker/Dockerfile.restapi.alpine

@@ -6,60 +6,54 @@ MAINTAINER Jookies LTD <jasmin@jookies.net>
 RUN addgroup -S jasmin && adduser -S -g jasmin jasmin
 
 # Install requirements
-RUN apk --update add \
-    gcc \
-    musl-dev \
-    libffi-dev \
-    openssl-dev \
-    python3-dev \
-    py3-pip \
-    git \
-    bash 
-
-# For RestAPI MODE
-ENV RESTAPI_MODE 0
-ENV RESTAPI_OLD_HTTP_HOST '127.0.0.1'
-
-ENV ENABLE_PUBLISH_SUBMIT_SM_RESP 0
+RUN apk --no-cache add \
+        libffi-dev \
+        openssl-dev \
+        bash \
+    ; \
+    rm -vrf /var/cache/apk/*
 
 ENV ROOT_PATH /
 ENV CONFIG_PATH /etc/jasmin
-ENV RESOURCE_PATH /etc/jasmin/resource
-ENV STORE_PATH /etc/jasmin/store
+ENV RESOURCE_PATH ${CONFIG_PATH}/resource
+ENV STORE_PATH ${CONFIG_PATH}/store
 ENV LOG_PATH /var/log/jasmin
 
 RUN mkdir -p ${CONFIG_PATH} ${RESOURCE_PATH} ${STORE_PATH} ${LOG_PATH}
 RUN chown jasmin:jasmin ${CONFIG_PATH} ${RESOURCE_PATH} ${STORE_PATH} ${LOG_PATH}
 
 WORKDIR /build
 
-RUN pip install -e git+https://github.com/jookies/txamqp.git@master#egg=txamqp3
-RUN pip install -e git+https://github.com/jookies/python-messaging.git@master#egg=python-messaging
-RUN pip install -e git+https://github.com/jookies/smpp.pdu.git@master#egg=smpp.pdu3
-RUN pip install -e git+https://github.com/jookies/smpp.twisted.git@master#egg=smpp.twisted3
+RUN pip install --upgrade pip
 
 COPY . .
 
 RUN pip install .
 
+# Clean build dir
+RUN rm -rf /build
+
 # For RestAPI MODE
 RUN pip install gunicorn
 
 ENV UNICODEMAP_JP unicode-ascii
 
 COPY misc/config/*.cfg ${CONFIG_PATH}/
-COPY misc/config/resource/*.xml ${RESOURCE_PATH}/
+COPY misc/config/resource ${RESOURCE_PATH}
 
 WORKDIR /etc/jasmin
 
-# Change binding host for jcli
-RUN sed -i '/\[jcli\]/a bind=0.0.0.0' ${CONFIG_PATH}/jasmin.cfg
-# Change binding port for redis, and amqp
-RUN sed -i "/\[redis-client\]/a port=$REDIS_CLIENT_PORT" ${CONFIG_PATH}/jasmin.cfg
-RUN sed -i "/\[amqp-broker\]/a port=$AMQP_BROKER_PORT" ${CONFIG_PATH}/jasmin.cfg
-# Change binding host for redis, and amqp
-RUN sed -i "/\[redis-client\]/a host=$REDIS_CLIENT_HOST" ${CONFIG_PATH}/jasmin.cfg
-RUN sed -i "/\[amqp-broker\]/a host=$AMQP_BROKER_HOST" ${CONFIG_PATH}/jasmin.cfg
+# Default Jasmin, Redis, and RabbitMQ connections
+ENV JCLI_BIND '0.0.0.0'
+ENV AMQP_BROKER_HOST 'rabbitmq'
+ENV AMQP_BROKER_PORT 5672
+ENV REDIS_CLIENT_HOST 'redis'
+ENV REDIS_CLIENT_PORT 6379
+
+# For RestAPI http mode
+ENV RESTAPI_HTTP_MODE 0
+# For RestAPI Celery worker mode
+ENV RESTAPI_WORKER_MODE 0
 
 EXPOSE 2775 8990 1401 8080
 VOLUME ["/var/log/jasmin", "/etc/jasmin", "/etc/jasmin/store"]

docker/restapi-docker-entrypoint.sh

@@ -1,40 +1,26 @@
 #!/bin/bash
 set -e
 
-# Change binding host:port for redis, and amqp
-sed -i "/\[redis-client\]/,/host=/  s/host=.*/host=$REDIS_CLIENT_HOST/" ${CONFIG_PATH}/jasmin.cfg
-sed -i "/\[redis-client\]/,/port=/  s/port=.*/port=$REDIS_CLIENT_PORT/" ${CONFIG_PATH}/jasmin.cfg
-sed -i "/\[amqp-broker\]/,/host=/  s/host=.*/host=$AMQP_BROKER_HOST/" ${CONFIG_PATH}/jasmin.cfg
-sed -i "/\[amqp-broker\]/,/port=/  s/port=.*/port=$AMQP_BROKER_PORT/" ${CONFIG_PATH}/jasmin.cfg
-
+# Clean lock files
 echo 'Cleaning lock files'
 rm -f /tmp/*.lock
 
-# RestAPI
-if [ "$RESTAPI_MODE" = 1 ]; then
-  # find jasmin installation directory
-  jasminRoot=$(python -c "import jasmin as _; print(_.__path__[0])")
-  # update jasmin-restAPI config
-  sed -i "/# RESTAPI/,/old_api_uri/  s/old_api_uri.*/old_api_uri = 'http:\/\/$RESTAPI_OLD_HTTP_HOST:1401'/" ${jasminRoot}/protocols/rest/config.py 
-  sed -i "/# CELERY/,/broker_url/  s/broker_url.*/broker_url = 'amqp:\/\/guest:guest@$AMQP_BROKER_HOST:$AMQP_BROKER_PORT\/\/'/" ${jasminRoot}/protocols/rest/config.py 
-  sed -i "/# CELERY/,/result_backend/  s/result_backend.*/result_backend = 'redis:\/\/:@$REDIS_CLIENT_HOST:$REDIS_CLIENT_PORT\/1'/" ${jasminRoot}/protocols/rest/config.py 
+# If RestAPI http Mode, start Guicorn
+if [ "$RESTAPI_HTTP_MODE" = 1 ]; then
+  # start restapi
   exec gunicorn -b 0.0.0.0:8080 jasmin.protocols.rest:api --access-logfile /var/log/jasmin/rest-api.access.log --disable-redirect-access-to-syslog
-else
-  if [ "$ENABLE_PUBLISH_SUBMIT_SM_RESP" = 1 ]; then
-    # Enable publish_submit_sm_resp
-    echo 'Enabling publish_submit_sm_resp'
-    sed -i "s/.*publish_submit_sm_resp\s*=.*/publish_submit_sm_resp=True/g" ${CONFIG_PATH}/jasmin.cfg
-  else
-    # Disable publish_submit_sm_resp
-    echo 'Disabling publish_submit_sm_resp'
-    sed -i "s/.*publish_submit_sm_resp\s*=.*/publish_submit_sm_resp=False/g" ${CONFIG_PATH}/jasmin.cfg
-  fi
 
+# If Celery Worker is enabled, start Celery worker
+elif [ "$RESTAPI_WORKER_MODE" = 1 ]; then
+  echo 'Starting Celery worker'
+  exec celery -A jasmin.protocols.rest.tasks worker -l INFO -c 4 --autoscale=10,3
+
+# Else start jasmind
+else
   if [ "$2" = "--enable-interceptor-client" ]; then
     echo 'Starting interceptord'
     interceptord.py &
   fi
-
   echo 'Starting jasmind'
   exec "$@"
 fi

jasmin/protocols/rest/__init__.py

@@ -5,19 +5,13 @@
 import sys
 
 from .api import PingResource, BalanceResource, RateResource, SendResource, SendBatchResource
-from .config import *
+from .config import RestAPIForJasminConfig
 
 sys.path.append("%s/vendor" % os.path.dirname(os.path.abspath(jasmin.__file__)))
 from falcon import HTTPUnauthorized, HTTPUnsupportedMediaType, API
 
-# @TODO: make configuration loadable from /etc/jasmin/restapi.conf
-logger = logging.getLogger('jasmin-restapi')
-if len(logger.handlers) == 0:
-    logger.setLevel(log_level)
-    handler = logging.handlers.TimedRotatingFileHandler(filename=log_file, when=log_rotate)
-    handler.setFormatter(logging.Formatter(log_format, log_date_format))
-    logger.addHandler(handler)
-
+RestAPIForJasminConfigInstance = RestAPIForJasminConfig()
+logger = RestAPIForJasminConfigInstance.logger
 
 class TokenNotFound(Exception):
     """Raised when authentication token is not found"""
@@ -36,7 +30,7 @@ def process_response(self, request, response, resource, req_succeeded):
             response.body = json.dumps(response.body)
 
         # Add Jasmin signature
-        if show_jasmin_version:
+        if RestAPIForJasminConfigInstance.show_jasmin_version:
             response.set_header('Powered-By', 'Jasmin %s' % jasmin.get_release())
 
 

jasmin/protocols/rest/api.py

@@ -6,24 +6,25 @@
 import requests
 
 import jasmin
-from .config import *
+from .config import RestAPIForJasminConfig
 from .tasks import httpapi_send
 from datetime import datetime
 from falcon import HTTPInternalServerError, HTTPPreconditionFailed
 import falcon
 
 sys.path.append("%s/vendor" % os.path.dirname(os.path.abspath(jasmin.__file__)))
 
+RestAPIForJasminConfigInstance = RestAPIForJasminConfig()
 
 class JasminHttpApiProxy:
     """Provides a WS caller for old Jasmin http api"""
 
     def call_jasmin(self, url, params=None):
         try:
-            r = requests.get('%s/%s' % (old_api_uri, url), params=params)
+            r = requests.get('%s/%s' % (RestAPIForJasminConfigInstance.http_api_uri, url), params=params)
         except requests.exceptions.ConnectionError as e:
             raise HTTPInternalServerError('Jasmin httpapi connection error',
-                                          'Could not connect to Jasmin http api (%s): %s' % (old_api_uri, e))
+                                          'Could not connect to Jasmin http api (%s): %s' % (RestAPIForJasminConfigInstance.http_api_uri, e))
         except Exception as e:
             raise HTTPInternalServerError('Jasmin httpapi unknown error', str(e))
         else:
@@ -202,7 +203,7 @@ def on_post(self, request, response):
 
         batch_id = uuid.uuid4()
         params = self.decode_request_data(request)
-        config = {'throughput': http_throughput_per_worker, 'smart_qos': smart_qos}
+        config = {'throughput': RestAPIForJasminConfigInstance.http_throughput_per_worker, 'smart_qos': RestAPIForJasminConfigInstance.smart_qos}
 
         # Batch scheduling
         countdown = self.parse_schedule_at(params.get('batch_config', {}).get('schedule_at', None))

jasmin/protocols/rest/config.py

@@ -1,33 +1,73 @@
 """This is the jasmin-celery and jasmin-restapi configurations"""
 
 import logging
-from jasmin.config import LOG_PATH
-
-# RESTAPI
-old_api_uri = 'http://127.0.0.1:1401'
-show_jasmin_version = True
-auth_cache_seconds = 10
-auth_cache_max_keys = 500
-
-log_level = logging.getLevelName('INFO')
-log_file = '%s/restapi.log' % LOG_PATH
-log_rotate = 'W6'
-log_format = '%(asctime)s %(levelname)-8s %(process)d %(message)s'
-log_date_format = '%Y-%m-%d %H:%M:%S'
-
-# CELERY
-broker_url = 'amqp://guest:guest@127.0.0.1:5672//'
-result_backend = 'redis://:@127.0.0.1:6379/1'
-task_serializer = 'json'
-result_serializer = 'json'
-accept_content = ['json']
-timezone = 'UTC'
-enable_utc = True
-# Will push messages back to Jasmin's http api with a maximum throughput *per worker*
-# Please consider this is a *per worker* throughput !
-# Set to zero (0) to disable throughput control
-http_throughput_per_worker = 8
-# When set to True, the batch executor will consider Jasmin's response time to
-# control the batch throughput, slower response time will slow down the throughput
-# and vice-versa
-smart_qos = True
+import os
+from jasmin.config import ConfigFile, ROOT_PATH, LOG_PATH
+
+CONFIG_PATH = os.getenv('CONFIG_PATH', '%s/etc/jasmin/' % ROOT_PATH)
+REST_API_CONFIG = os.getenv('REST_API_CONFIG', '%s/rest-api.cfg' % CONFIG_PATH)
+
+class RestAPIForJasminConfig(ConfigFile):
+    """Config handler for 'rest-api' section"""
+
+    def __init__(self, config_file=REST_API_CONFIG):
+        ConfigFile.__init__(self, config_file)
+
+        # Celery global configuration
+        self.timezone = self._get('celery', 'timezone', 'UTC')
+        self.enable_utc = self._getbool('celery', 'enable_utc', True)
+        self.accept_content = self._get('celery', 'accept_content', 'json').split(',')
+
+        # AMQP Broker Connection configuration
+        self.broker_url = 'amqp://%s:%s@%s:%s/%s' % (
+            self._get('amqp-broker', 'username', 'guest'),
+            self._get('amqp-broker', 'password', 'guest'),
+            self._get('amqp-broker', 'host', '127.0.0.1'),
+            self._getint('amqp-broker', 'port', 5672),
+            self._get('amqp-broker', 'vhost', '/'))
+
+        self.broker_heartbeat = self._getint('amqp-broker', 'broker_heartbeat', 120)
+        self.broker_heartbeat_checkrate = self._getint('amqp-broker', 'broker_heartbeat_checkrate', 2)
+        self.broker_connection_timeout = self._getint('amqp-broker', 'broker_connection_timeout', 4)
+        self.broker_connection_retry = self._getbool('amqp-broker', 'broker_connection_retry', True)
+        self.broker_connection_startup_retry = self._getbool('amqp-broker', 'broker_connection_startup_retry', True)
+        self.broker_connection_max_retries = self._getint('amqp-broker', 'broker_connection_max_retries', 100)
+        self.task_serializer = self._get('amqp-broker', 'task_serializer', 'json')
+
+        # Redis configuration
+        redis_password = self._get('redis-client', 'password', None)
+        self.result_backend = 'redis://:%s@%s:%s/%s' % (
+            redis_password if redis_password is not None else '',
+            self._get('redis-client', 'host', '127.0.0.1'),
+            self._getint('redis-client', 'port', 6379),
+            self._getint('redis-client', 'result_dbid', 1))
+
+        self.redis_socket_timeout = self._getint('redis-client', 'redis_socket_timeout', 120)
+        self.result_expires = self._getint('redis-client', 'result_expires', 86400)
+        self.result_serializer = self._get('redis-client', 'result_serializer', 'json')
+
+
+        self.http_api_uri = 'http://%s:%s' % (
+            self._get('rest-api', 'http_api_host', '127.0.0.1'), 
+            self._get('rest-api', 'http_api_port', '1401'))
+
+        self.show_jasmin_version = self._getbool('rest-api', 'show_jasmin_version', True)
+        self.auth_cache_seconds = self._getint('rest-api', 'auth_cache_seconds', 10)
+        self.auth_cache_max_keys = self._getint('rest-api', 'auth_cache_max_keys', 500)
+        self.http_throughput_per_worker = self._getint('rest-api', 'http_throughput_per_worker', 8)
+        self.smart_qos = self._getbool('rest-api', 'smart_qos', True)
+
+        self.log_level = logging.getLevelName(self._get('rest-api', 'log_level', 'INFO'))
+        self.log_file = self._get('rest-api', 'log_file', '%s/rest-api.log' % LOG_PATH)
+        self.log_rotate = self._get('rest-api', 'log_rotate', 'W6')
+        self.log_format = self._get('rest-api', 'log_format', '%(asctime)s %(levelname)-8s %(process)d %(message)s')
+        self.log_date_format = self._get('rest-api', 'log_date_format', '%Y-%m-%d %H:%M:%S')
+
+        self.logger = logging.getLogger('jasmin-restapi')
+        if len(self.logger.handlers) == 0:
+            self.logger.setLevel(self.log_level)
+            handler = logging.handlers.TimedRotatingFileHandler(filename=self.log_file, when=self.log_rotate)
+            handler.setFormatter(logging.Formatter(self.log_format, self.log_date_format))
+            self.logger.addHandler(handler)
+
+