Merge pull request #7 from marshyski/master

Overhaul

Dockerfile

@@ -1,23 +1,18 @@
-FROM fedora:21
+FROM centos:7
 MAINTAINER Tim Marcinowski <marshyski@gmail.com>
 
 USER root
 
 RUN yum -y update
-RUN yum install -y epel-release python-pip supervisor gcc nginx createrepo python-setuptools
-RUN rm -rf /usr/share/nginx
+RUN yum install -y epel-release gcc createrepo python-setuptools python-devel
+RUN yum install -y python-pip supervisor
+RUN pip install --upgrade pip
 ADD ./requirements.txt /tmp/requirements.txt
 RUN pip install -r /tmp/requirements.txt
 RUN yum remove -y gcc && yum clean all
-RUN mkdir -p /opt/repos/pre-release
-ADD nginx/nginx.conf /etc/nginx/nginx.conf
-ADD nginx/mime.types /etc/nginx/mime.types
-ADD yumapi /opt/yumapi
-ADD supervisor/yumapi.conf /etc/supervisord.d/yumapi.conf
-ADD supervisor/supervisord.conf /etc/supervisord.conf
-ADD ./scripts/settings.sh /tmp/settings.sh
-RUN /bin/bash /tmp/settings.sh
+ADD . /opt/yum-nginx-api
+ADD supervisor/yumapi.ini /etc/supervisord.d/yumapi.ini
 
-EXPOSE 80
+EXPOSE 8888
 
 CMD ["supervisord", "-n", "-c", "/etc/supervisord.conf"]

README.md

@@ -15,40 +15,47 @@ It is a deployable solution with Docker or any existing web server with WSGI sup
 
 
 
-**Technologies Needed** [see install](#install) :
+**Requirements**:
 
  1.  Server (Bare-metal/Cloud)
  2.  [NGINX Web Server][2]
  3.  [Python Flask][3] (Optional)
  4.  [Python Gunicorn][4] (Optional)
  5.  [Python Supervisor][5] (Optional)
- 6.  [Docker][6] (Optional) 
+ 6.  [Docker Engine][6] >=1.10 (Optional) 
+ 7.  [Docker Compose][7] >=1.7.1 (Optional)
 
 
-## Pull Docker image from the Docker Registry <a name="install"></a>
+## Pull image from Docker Hub
+
     docker pull finraos/yum-nginx-api
-    docker run -d -p 80:80 finraos/yum-nginx-api
+
+## Run only API
+
+    mkdir -p /opt/repos/pre-release
+    docker run -d -p 8888:8888 -v /opt/repos:/opt/repos finraos/yum-nginx-api
+
+## Run Docker Compose
+
+	docker-compose up
 
 ## How to build yum-nginx-api (Docker)
 
     git clone https://github.com/FINRAOS/yum-nginx-api.git
-    cd yum-nginx-api && docker build -t finraos/yum-nginx-api .
-    docker run -d -p 80:80 finraos/yum-nginx-api
-    sleep 10 && docker logs `docker ps | grep 'yum-nginx-api' | awk '{ print $1 }' | head -n1` 
+    cd yum-nginx-api && docker build -t finraos/yum-nginx-api . 
 
 ## How to Install yum-nginx-api (Non-Docker)
 
     # Need EPEL repo installed
     git clone https://github.com/FINRAOS/yum-nginx-api.git
-    yum install -y python-pip supervisor gcc nginx createrepo python-setuptools
+    yum install -y python-pip supervisor gcc nginx createrepo python-setuptools python-devel
     cd yum-nginx-api && pip install -r requirements.txt
     mkdir -p /opt/repos/pre-release
     bash scripts/settings.sh
-    cp -f supervisor/supervisord.conf /etc/
-    cp -f supervisor/yumapi.conf /etc/supervisord.d/
-    cp -rf yumapi /opt/
+    cp -f supervisor/yumapi.ini /etc/supervisord.d/
     cp -rf nginx/* /etc/nginx/
-    supervisord -n -c /etc/supervisord.conf nohup &
+    service nginx restart
+    service supervisord restart
 
 ## Configuration File `config.yaml`
 
@@ -144,5 +151,5 @@ yum-nginx-api project is licensed under [Apache License Version 2.0](http://www.
   [3]: http://flask.pocoo.org
   [4]: http://gunicorn.org
   [5]: http://supervisord.org
-  [6]: https://docker.io
-  [7]: http://www.finra.org/sites/default/files/p075334_0.gif
+  [6]: https://docs.docker.com/engine/installation/
+  [7]: https://docs.docker.com/compose/

docker-compose.yml

@@ -0,0 +1,22 @@
+version: "2"
+services:
+  nginx:
+    image: nginx:latest
+    volumes:
+      - ./nginx/nginx.conf:/etc/nginx/nginx.conf
+      - ./nginx/mime.types:/etc/nginx/mime.types
+      - /opt/repos:/opt/repos
+    ports:
+      - "80:80"
+    network_mode: host
+    restart: always
+  yumapi:
+    image: finraos/yum-nginx-api:latest
+    network_mode: host
+    restart: always
+    volumes:
+      - /opt/repos:/opt/repos
+    ports:
+      - "8888:8888"
+    depends_on:
+      - nginx

nginx/mime.types

@@ -1,5 +1,3 @@
-
-
 types {
     text/html                             html htm shtml;
     text/css                              css;

nginx/nginx.conf

@@ -1,6 +1,5 @@
-daemon off;
 user nginx;
-worker_processes 2;
+worker_processes auto;
 worker_rlimit_nofile 65536;
 
 error_log  /var/log/nginx/error.log;
@@ -10,6 +9,7 @@ pid        /var/run/nginx.pid;
 events {
     worker_connections  8192;
     multi_accept on;
+    use epoll;
 }
 
 http {
@@ -22,15 +22,23 @@ http {
 
     access_log  /var/log/nginx/access.log  main;
 
+    server_tokens      off;
     sendfile           off;
     tcp_nopush         on;
-    keepalive_timeout  80;
+    keepalive_timeout  300;
+    keepalive_requests 100000;
     tcp_nodelay        off;
+    reset_timedout_connection on;
+
+    open_file_cache max=200000 inactive=5m;
+    open_file_cache_valid 5m;
+    open_file_cache_min_uses 2;
+    open_file_cache_errors off;
 
     gzip  on;
     gzip_http_version  1.1;
     gzip_comp_level    5;
-    gzip_min_length    1000;
+    gzip_min_length    10000;
     gzip_proxied       any;
     gzip_vary          on;
     gzip_types
@@ -40,11 +48,6 @@ http {
     text/plain
     text/x-component;
 
-    # Load modular configuration files from the /etc/nginx/conf.d directory.
-    # See http://nginx.org/en/docs/ngx_core_module.html#include
-    # for more information.
-    #include /etc/nginx/conf.d/*.conf;
-
     server {
         listen       80;
         server_name  localhost;
@@ -99,7 +102,5 @@ http {
             proxy_set_header Upgrade $http_upgrade;
             proxy_set_header Connection "Upgrade";
         }
-
     }
-
 }

requirements.txt

@@ -1,7 +1,8 @@
-Flask==0.10.1
-Flask-Limiter==0.7.9
-Werkzeug==0.10.4
-gunicorn==19.3.0
-python-magic==0.4.6
-SQLAlchemy==1.0.6
+Flask-Limiter==0.9.3
+Flask==0.11
 PyYAML==3.11
+SQLAlchemy==1.0.13
+Werkzeug==0.11.10
+gunicorn==19.6.0
+psutil==4.2.0
+python-magic==0.4.11

scripts/settings.sh

@@ -24,8 +24,8 @@ fs.file-max = 70000
 net.ipv4.ip_local_port_range = 2000 65000
 net.ipv4.tcp_window_scaling = 1
 net.ipv4.tcp_max_syn_backlog = 250000
-net.core.netdev_max_backlog = 4000
-net.core.somaxconn = 4000
+net.core.netdev_max_backlog = 4096
+net.core.somaxconn = 65535
 net.ipv4.tcp_max_tw_buckets = 1440000
 net.core.rmem_default = 8388608
 net.core.rmem_max = 16777216
@@ -42,5 +42,8 @@ net.ipv4.tcp_timestamps = 1
 net.ipv4.tcp_tw_reuse = 1
 net.ipv4.tcp_sack = 0
 net.ipv4.tcp_dsack = 0
+net.ipv4.tcp_syn_retries= 3
 net.ipv4.tcp_synack_retries = 3
+net.ipv4.tcp_slow_start_after_idle = 0
+net.ipv4.tcp_no_metrics_save = 0
 SYSCTL

supervisor/supervisord.conf → supervisor/yumapi.ini

@@ -3,12 +3,14 @@ logfile=/var/log/supervisor/supervisor.log ; (main log file;default $CWD/supervi
 logfile_maxbytes=10MB        ; (max main logfile bytes b4 rotation;default 50MB)
 loglevel=info                ; (log level;default info; others: debug,warn,trace)
 pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
+nodaemon=true
 
-[rpcinterface:supervisor]
-supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
-
-[supervisorctl]
-serverurl=unix:///var/run/supervisor.sock ; use a unix:// URL  for a unix socket
-
-[include]
-files = /etc/supervisord.d/*.conf
+[program:yumapi]
+command = /opt/yum-nginx-api/yumapi/yumapi.sh
+priority= 1
+user = root
+stdout_logfile = /var/log/yumapi.log
+redirect_stderr = true
+autostart = true
+autorestart = true
+startretries = 20

yumapi/__init__.py

@@ -14,6 +14,7 @@
 from subprocess import call
 from flask import Flask, request, jsonify, abort
 from flask_limiter import Limiter
+from flask_limiter.util import get_remote_address
 from werkzeug.utils import secure_filename
 from werkzeug.contrib.fixers import ProxyFix
 import os
@@ -42,7 +43,7 @@
     request_limit = '1 per second'
 
 if config_yaml['createrepo_workers']:
-    createrepo_workers = config_yaml['createrepo_workers']
+    createrepo_workers = str(config_yaml['createrepo_workers'])
 else:
     createrepo_workers = '2'
 
@@ -56,7 +57,7 @@
 """900MB default limit set below"""
 app.config['MAX_CONTENT_LENGTH'] = max_content_length
 
-limiter = Limiter(app)
+limiter = Limiter(app, key_func=get_remote_address)
 
 def uptime():
     """Return uptime about nginxify for health check"""
@@ -116,6 +117,10 @@ def method_not_allowed(error):
 def unsupported_media(error):
    return jsonify(message='File not RPM', status=415), 415
 
+@app.errorhandler(413)
+def payload_too_large(error):
+    return jsonify(message='Exceeds content length', status=413), 413
+
 app.wsgi_app = ProxyFix(app.wsgi_app)
 
 if __name__ == '__main__':

yumapi/config.yaml

@@ -1,4 +1,4 @@
 createrepo_workers:
 max_content_length:
 request_limit:
-upload_dir:
+upload_dir: /opt/repos/pre-release

yumapi/yumapi.sh

@@ -13,7 +13,7 @@
 NAME=yumapi
 USER=root
 GROUP=root
-WORKERS=2
+WORKERS=`lscpu | grep ^'CPU(s)' | awk '{ print $2 }'`
 DEPLOY_DIR=/opt/yum-nginx-api
 
 if [[ $1 = "" ]]; then