Permalink
Browse files

Use uWSGI for python runtime

Make python runtime more production-ready. Actually, the docker image
has been built and used in Qinling gate job.

Change-Id: Ie7a0ae19042acecc1cfcbb2ac13fa019ca5735a9
  • Loading branch information...
lingxiankong committed Mar 1, 2018
1 parent c52b9db commit 5615dd55dae23b0a9b6a4b5b39e06d4edbcc39a2
View
@@ -4,15 +4,15 @@
pbr!=2.1.0,>=2.0.0 # Apache-2.0
Babel!=2.4.0,>=2.3.4 # BSD
keystoneauth1>=3.3.0 # Apache-2.0
keystoneauth1>=3.4.0 # Apache-2.0
keystonemiddleware>=4.17.0 # Apache-2.0
oslo.concurrency>=3.20.0 # Apache-2.0
oslo.concurrency>=3.25.0 # Apache-2.0
oslo.config>=5.1.0 # Apache-2.0
oslo.db>=4.27.0 # Apache-2.0
oslo.messaging>=5.29.0 # Apache-2.0
oslo.policy>=1.30.0 # Apache-2.0
oslo.utils>=3.33.0 # Apache-2.0
oslo.log>=3.30.0 # Apache-2.0
oslo.log>=3.36.0 # Apache-2.0
oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0
oslo.service!=1.28.1,>=1.24.0 # Apache-2.0
pecan!=1.0.2,!=1.0.3,!=1.0.4,!=1.2,>=1.0.0 # BSD
@@ -22,12 +22,12 @@ SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,>=1.0.10 # MIT
sqlalchemy-migrate>=0.11.0 # Apache-2.0
stevedore>=1.20.0 # Apache-2.0
WSME>=0.8.0 # MIT
kubernetes>=4.0.0 # Apache-2.0
kubernetes>=4.0.0 # Apache-2.0
PyYAML>=3.10 # MIT
python-swiftclient>=3.2.0 # Apache-2.0
croniter>=0.3.4 # MIT License
python-dateutil>=2.4.2 # BSD
tenacity>=3.2.1 # Apache-2.0
PyMySQL>=0.7.6 # MIT License
etcd3gw>=0.2.0 # Apache-2.0
cotyledon>=1.3.0 # Apache-2.0
cotyledon>=1.3.0 # Apache-2.0
@@ -6,12 +6,14 @@ RUN useradd -Ms /bin/bash qinling
RUN apt-get update && \
apt-get -y install python-dev python-setuptools libffi-dev libxslt1-dev libxml2-dev libyaml-dev libssl-dev python-pip && \
pip install -U pip setuptools
pip install -U pip setuptools uwsgi
COPY . /app
WORKDIR /app
RUN pip install -r requirements.txt && \
chmod 0750 custom-entrypoint.sh && \
chown -R qinling:qinling /app
mkdir -p /var/lock/qinling && \
chown -R qinling:qinling /app /var/lock/qinling
CMD ["/bin/bash", "custom-entrypoint.sh"]
# uwsgi --http :9090 --uid qinling --wsgi-file server.py --callable app --master --processes 5 --threads 1
CMD ["/usr/local/bin/uwsgi", "--http", ":9090", "--uid", "qinling", "--wsgi-file", "server.py", "--callable", "app", "--master", "--processes", "5", "--threads", "1"]
View

This file was deleted.

Oops, something went wrong.

This file was deleted.

Oops, something went wrong.
@@ -1,4 +1,5 @@
Flask>=0.10,!=0.11,<1.0 # BSD
oslo.concurrency>=3.25.0 # Apache-2.0
python-openstackclient>=3.3.0,!=3.10.0 # Apache-2.0
python-neutronclient>=6.3.0 # Apache-2.0
python-swiftclient>=3.2.0 # Apache-2.0
View
@@ -14,10 +14,10 @@
import importlib
import json
import logging
from multiprocessing import Manager
from multiprocessing import Process
import os
import resource
import sys
import time
import traceback
@@ -27,35 +27,37 @@
from flask import Response
from keystoneauth1.identity import generic
from keystoneauth1 import session
from oslo_concurrency import lockutils
import requests
app = Flask(__name__)
downloaded = False
downloading = False
DOWNLOAD_ERROR = "Failed to download function package from %s, error: %s"
INVOKE_ERROR = "Function execution failed because of too much resource " \
"consumption"
def setup_logger(loglevel):
global app
root = logging.getLogger()
root.setLevel(loglevel)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(loglevel)
ch.setFormatter(
logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
)
app.logger.addHandler(ch)
def _print_trace():
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
print(''.join(line for line in lines))
def _set_ulimit():
"""Limit resources usage for the current process and/or its children.
Refer to https://docs.python.org/2.7/library/resource.html
"""
customized_limits = {
resource.RLIMIT_NOFILE: 1024,
resource.RLIMIT_NPROC: 128,
resource.RLIMIT_FSIZE: 61440
}
for t, soft in customized_limits.items():
_, hard = resource.getrlimit(t)
resource.setrlimit(t, (soft, hard))
def _get_responce(output, duration, logs, success, code):
return Response(
response=json.dumps(
@@ -71,8 +73,13 @@ def _get_responce(output, duration, logs, success, code):
)
@lockutils.synchronized('download_function', external=True,
lock_path='/var/lock/qinling')
def _download_package(url, zip_file, token=None):
app.logger.info('Downloading function, download_url:%s' % url)
if os.path.isfile(zip_file):
return True, None
print('Downloading function, download_url:%s' % url)
headers = {}
if token:
@@ -94,8 +101,7 @@ def _download_package(url, zip_file, token=None):
DOWNLOAD_ERROR % (url, str(e)), 0, '', False, 500
)
app.logger.info('Downloaded function package to %s' % zip_file)
print('Downloaded function package to %s' % zip_file)
return True, None
@@ -135,9 +141,6 @@ def execute():
reason, e.g. unlimited memory allocation)
- Deal with os error for process (e.g. Resource temporarily unavailable)
"""
global downloading
global downloaded
params = request.get_json() or {}
input = params.get('input') or {}
execution_id = params['execution_id']
@@ -155,25 +158,20 @@ def execute():
if entry:
function_module, function_method = tuple(entry.rsplit('.', 1))
app.logger.info(
print(
'Request received, request_id: %s, execution_id: %s, input: %s, '
'auth_url: %s' %
(request_id, execution_id, input, auth_url)
)
while downloading:
time.sleep(3)
if not downloading and not downloaded:
downloading = True
ret, resp = _download_package(download_url, zip_file,
params.get('token'))
if not ret:
return resp
downloading = False
downloaded = True
# Download function package if needed.
ret, resp = _download_package(
download_url,
zip_file,
params.get('token')
)
if not ret:
return resp
# Provide an openstack session to user's function
os_session = None
@@ -188,6 +186,9 @@ def execute():
os_session = session.Session(auth=auth, verify=False)
input.update({'context': {'os_session': os_session}})
# Set resource limit
_set_ulimit()
manager = Manager()
return_dict = manager.dict()
return_dict['success'] = False
@@ -223,10 +224,3 @@ def execute():
@app.route('/ping')
def ping():
return 'pong'
setup_logger(logging.DEBUG)
app.logger.info("Starting server")
# Just for testing purpose
app.run(host='0.0.0.0', port=9090, threaded=True)
View
@@ -25,5 +25,5 @@
pass
setuptools.setup(
setup_requires=['pbr'],
setup_requires=['pbr>=2.0.0'],
pbr=True)
View
@@ -4,14 +4,14 @@
hacking<0.13,>=0.12.0 # Apache-2.0
coverage!=4.4,>=4.0 # Apache-2.0
sphinx>=1.6.2 # BSD
oslotest>=1.10.0 # Apache-2.0
sphinx!=1.6.6,>=1.6.2 # BSD
oslotest>=3.2.0 # Apache-2.0
testrepository>=0.0.18 # Apache-2.0/BSD
testscenarios>=0.4 # Apache-2.0/BSD
testtools>=2.2.0 # MIT
tempest>=17.1.0 # Apache-2.0
futurist>=1.2.0 # Apache-2.0
openstackdocstheme>=1.17.0 # Apache-2.0
openstackdocstheme>=1.18.1 # Apache-2.0
reno>=2.5.0 # Apache-2.0
kubernetes>=4.0.0 # Apache-2.0
kubernetes>=4.0.0 # Apache-2.0

0 comments on commit 5615dd5

Please sign in to comment.