diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..94143827ed --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +Dockerfile diff --git a/Dockerfile b/Dockerfile index a33318966b..4a0793e68c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,170 @@ # To run: docker run -d -v /path/to/fence-config.yaml:/var/www/fence/fence-config.yaml --name=fence -p 80:80 fence # To check running container: docker exec -it fence /bin/bash -FROM quay.io/cdis/python-nginx:pybase3-1.0.0 +FROM python:3.7-buster + +LABEL maintainer="Sebastian Ramirez " + +# Standard set up Nginx +ENV NGINX_VERSION 1.17.4-1~buster +ENV NJS_VERSION 1.17.4.0.3.5-1~buster + +RUN set -x \ + && apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y gnupg1 apt-transport-https ca-certificates \ + && \ + NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \ + found=''; \ + for server in \ + ha.pool.sks-keyservers.net \ + hkp://keyserver.ubuntu.com:80 \ + hkp://p80.pool.sks-keyservers.net:80 \ + pgp.mit.edu \ + ; do \ + echo "Fetching GPG key $NGINX_GPGKEY from $server"; \ + apt-key adv --no-tty --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \ + done; \ + test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \ + apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \ + && dpkgArch="$(dpkg --print-architecture)" \ + && nginxPackages=" \ + nginx=${NGINX_VERSION} \ + nginx-module-xslt=${NGINX_VERSION} \ + nginx-module-geoip=${NGINX_VERSION} \ + nginx-module-image-filter=${NGINX_VERSION} \ + nginx-module-njs=${NJS_VERSION} \ + " \ + && case "$dpkgArch" in \ + amd64|i386) \ +# arches officialy built by upstream + echo "deb https://nginx.org/packages/mainline/debian/ buster nginx" >> /etc/apt/sources.list.d/nginx.list \ + && apt-get update \ + ;; \ + *) \ +# we're on an architecture upstream doesn't officially build for +# let's build binaries from the published source packages + echo "deb-src https://nginx.org/packages/mainline/debian/ buster nginx" >> /etc/apt/sources.list.d/nginx.list \ + \ +# new directory for storing sources and .deb files + && tempDir="$(mktemp -d)" \ + && chmod 777 "$tempDir" \ +# (777 to ensure APT's "_apt" user can access it too) + \ +# save list of currently-installed packages so build dependencies can be cleanly removed later + && savedAptMark="$(apt-mark showmanual)" \ + \ +# build .deb files from upstream's source packages (which are verified by apt-get) + && apt-get update \ + && apt-get build-dep -y $nginxPackages \ + && ( \ + cd "$tempDir" \ + && DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \ + apt-get source --compile $nginxPackages \ + ) \ +# we don't remove APT lists here because they get re-downloaded and removed later + \ +# reset apt-mark's "manual" list so that "purge --auto-remove" will remove all build dependencies +# (which is done after we install the built packages so we don't have to redownload any overlapping dependencies) + && apt-mark showmanual | xargs apt-mark auto > /dev/null \ + && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \ + \ +# create a temporary local APT repo to install from (so that dependency resolution can be handled by APT, as it should be) + && ls -lAFh "$tempDir" \ + && ( cd "$tempDir" && dpkg-scanpackages . > Packages ) \ + && grep '^Package: ' "$tempDir/Packages" \ + && echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list \ +# work around the following APT issue by using "Acquire::GzipIndexes=false" (overriding "/etc/apt/apt.conf.d/docker-gzip-indexes") +# Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) +# ... +# E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) + && apt-get -o Acquire::GzipIndexes=false update \ + ;; \ + esac \ + \ + && apt-get install --no-install-recommends --no-install-suggests -y \ + $nginxPackages \ + gettext-base \ + && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx.list \ + \ +# if we have leftovers from building, let's purge them (including extra, unnecessary build deps) + && if [ -n "$tempDir" ]; then \ + apt-get purge -y --auto-remove \ + && rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \ + fi + +# forward request and error logs to docker log collector +RUN ln -sf /dev/stdout /var/log/nginx/access.log \ + && ln -sf /dev/stderr /var/log/nginx/error.log +EXPOSE 80 +# Removed the section that breaks pip installations +# && apt-get remove --purge --auto-remove -y apt-transport-https ca-certificates +# added --no-tty to apt-key adv as without it it breaks (even though it works in official Nginx) +# apt-key adv --no-tty +# Standard set up Nginx finished + +# Expose 443, in case of LTS / HTTPS +EXPOSE 443 + +# Install uWSGI +# RUN pip install uwsgi +RUN apt-get update \ + && apt-get -y install mcrypt uwsgi uwsgi-plugin-python3 + +# Remove default configuration from Nginx +RUN rm /etc/nginx/conf.d/default.conf +# Copy the base uWSGI ini file to enable default dynamic uwsgi process number +COPY uwsgi.ini /etc/uwsgi/ + +# Install Supervisord +RUN apt-get update && apt-get install -y supervisor \ +&& rm -rf /var/lib/apt/lists/* +# Custom Supervisord config +COPY supervisord.ini /etc/supervisor.d/supervisord.ini + +# Which uWSGI .ini file should be used, to make it customizable +ENV UWSGI_INI /app/uwsgi.ini + +# By default, run 2 processes +ENV UWSGI_CHEAPER 2 + +# By default, when on demand, run up to 16 processes +ENV UWSGI_PROCESSES 16 + +# By default, allow unlimited file sizes, modify it to limit the file sizes +# To have a maximum of 1 MB (Nginx's default) change the line to: +# ENV NGINX_MAX_UPLOAD 1m +ENV NGINX_MAX_UPLOAD 0 + +# By default, Nginx will run a single worker process, setting it to auto +# will create a worker for each CPU core +ENV NGINX_WORKER_PROCESSES 1 + +# By default, Nginx listens on port 80. +# To modify this, change LISTEN_PORT environment variable. +# (in a Dockerfile or with an option for `docker run`) +ENV LISTEN_PORT 80 + +# Copy the entrypoint that will generate Nginx additional configs +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +COPY dockerrun.sh /dockerrun.sh +RUN chmod +x /dockerrun.sh + +ENTRYPOINT ["/entrypoint.sh"] + +# Add demo app +COPY ./app /app +WORKDIR /app + +# CMD ["/usr/bin/supervisord"] ENV appname=fence -RUN apk update \ - && apk add postgresql-libs postgresql-dev libffi-dev libressl-dev \ - && apk add linux-headers musl-dev gcc \ - && apk add curl bash git vim make +# RUN apk update \ +# && apk add postgresql-libs postgresql-dev libffi-dev libressl-dev \ +# && apk add linux-headers musl-dev gcc \ +# && apk add curl bash git vim make COPY . /$appname COPY ./deployment/uwsgi/uwsgi.ini /etc/uwsgi/uwsgi.ini @@ -27,33 +183,37 @@ RUN mkdir -p /var/www/$appname \ && chown nginx -R /var/www/.cache/Python-Eggs/ \ && chown nginx /var/www/$appname -RUN apk update && apk add openssh && apk add libmcrypt-dev +# RUN apk update && apk add openssh && apk add libmcrypt-dev # # libmhash is required by mcrypt - below - no apk package available # -RUN (cd /tmp \ - && wget -O mhash.tar.gz https://sourceforge.net/projects/mhash/files/mhash/0.9.9.9/mhash-0.9.9.9.tar.gz/download \ - && tar xvfz mhash.tar.gz \ - && cd mhash-0.9.9.9 \ - && ./configure && make && make install \ - && /bin/rm -rf /tmp/*) +# RUN (cd /tmp \ +# && wget -O mhash.tar.gz https://sourceforge.net/projects/mhash/files/mhash/0.9.9.9/mhash-0.9.9.9.tar.gz/download \ +# && tar xvfz mhash.tar.gz \ +# && cd mhash-0.9.9.9 \ +# && ./configure && make && make install \ +# && /bin/rm -rf /tmp/*) -# -# mcrypt is required to decrypt dbgap user files - see fence/sync/sync_users.py -# -RUN (cd /tmp \ - && wget -O mcrypt.tar.gz https://sourceforge.net/projects/mcrypt/files/MCrypt/Production/mcrypt-2.6.4.tar.gz/download \ - && tar xvfz mcrypt.tar.gz \ - && cd mcrypt-2.6.4 \ - && ./configure && make && make install \ - && /bin/rm -rf /tmp/*) -EXPOSE 80 +# # +# # mcrypt is required to decrypt dbgap user files - see fence/sync/sync_users.py +# # +# RUN (cd /tmp \ +# && wget -O mcrypt.tar.gz https://sourceforge.net/projects/mcrypt/files/MCrypt/Production/mcrypt-2.6.4.tar.gz/download \ +# && tar xvfz mcrypt.tar.gz \ +# && cd mcrypt-2.6.4 \ +# && ./configure && make && make install \ +# && /bin/rm -rf /tmp/*) +# EXPOSE 80 RUN COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" >$appname/version_data.py \ && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >>$appname/version_data.py \ && python setup.py develop +# RUN apt-get update \ +# && apt-get install +# RUN pip uninstall -y cffi && pip install pycrypto cffi + WORKDIR /var/www/$appname CMD ["sh","-c","bash /fence/dockerrun.bash && /dockerrun.sh"] diff --git a/app/main.py b/app/main.py new file mode 100755 index 0000000000..216c258f4e --- /dev/null +++ b/app/main.py @@ -0,0 +1,4 @@ +def application(env, start_response): + start_response('200 OK', [('Content-Type', 'text/html')]) + return [b"Hello World from a default Nginx uWSGI Python 3.6 app in a\ + Docker container (default)"] diff --git a/app/uwsgi.ini b/app/uwsgi.ini new file mode 100755 index 0000000000..8a29f73509 --- /dev/null +++ b/app/uwsgi.ini @@ -0,0 +1,2 @@ +[uwsgi] +wsgi-file=/app/main.py diff --git a/deployment/uwsgi/uwsgi.ini b/deployment/uwsgi/uwsgi.ini index 72a00fdaa1..f327e48fa1 100644 --- a/deployment/uwsgi/uwsgi.ini +++ b/deployment/uwsgi/uwsgi.ini @@ -22,7 +22,7 @@ plugins = python3 vacuum = true pythonpath = /var/www/fence/ pythonpath = /fence/ -pythonpath = /usr/local/lib/python3.6/site-packages/ +pythonpath = /usr/local/lib/python3.7/site-packages/ # Initialize application in worker processes, not master. This prevents the # workers from all trying to open the same database connections at startup. diff --git a/dockerrun.sh b/dockerrun.sh new file mode 100644 index 0000000000..b75e663db5 --- /dev/null +++ b/dockerrun.sh @@ -0,0 +1,103 @@ +#!/bin/sh +# +# Note: base alpine Linux image may not include bash shell, +# and we probably want to move to that for service images, +# so just use bourn shell ... + +# +# Update certificate authority index - +# environment may have mounted more authorities +# - ex: /usr/local/share/ca-certificates/cdis-ca.crt into system bundle +# + +GEN3_DEBUG="${GEN3_DEBUG:-False}" +GEN3_DRYRUN="${GEN3_DRYRUN:-False}" +GEN3_UWSGI_TIMEOUT="${GEN3_UWSGI_TIMEOUT:-45s}" + +run() { + if [ "$GEN3_DRYRUN" = True ]; then + echo "DRY RUN - not running: $@" + else + echo "Running $@" + "$@" + fi +} + +help() { + cat - <> ./wsgi.py +fi + +( + # Wait for nginx to create uwsgi.sock in a sub-process + count=0 + while [ ! -e /var/run/gen3/uwsgi.sock ] && [ $count -lt 10 ]; do + echo "... waiting for /var/run/gen3/uwsgi.sock to appear" + sleep 2 + count="$(($count+1))" + done + if [ ! -e /var/run/gen3/uwsgi.sock ]; then + echo "WARNING: /var/run/gen3/uwsgi.sock does not exist!!!" + fi + run uwsgi --ini /etc/uwsgi/uwsgi.ini +) & +run nginx -g 'daemon off;' +wait \ No newline at end of file diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000000..9982f1999a --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env sh +set -e + +# Get the maximum upload file size for Nginx, default to 0: unlimited +USE_NGINX_MAX_UPLOAD=${NGINX_MAX_UPLOAD:-0} +# Generate Nginx config for maximum upload file size +echo "client_max_body_size $USE_NGINX_MAX_UPLOAD;" > /etc/nginx/conf.d/upload.conf + +# Explicitly add installed Python packages and uWSGI Python packages to PYTHONPATH +# Otherwise uWSGI can't import Flask +export PYTHONPATH=$PYTHONPATH:/usr/local/lib/python3.6/site-packages:/usr/lib/python3.6/site-packages + +# Get the number of workers for Nginx, default to 1 +USE_NGINX_WORKER_PROCESSES=${NGINX_WORKER_PROCESSES:-1} +# Modify the number of worker processes in Nginx config +sed -i "/worker_processes\s/c\worker_processes ${USE_NGINX_WORKER_PROCESSES};" /etc/nginx/nginx.conf + +# Set the max number of connections per worker for Nginx, if requested +# Cannot exceed worker_rlimit_nofile, see NGINX_WORKER_OPEN_FILES below +if [ -n "$NGINX_WORKER_CONNECTIONS" ] ; then + sed -i "/worker_connections\s/c\ worker_connections ${NGINX_WORKER_CONNECTIONS};" /etc/nginx/nginx.conf +fi + +# Set the max number of open file descriptors for Nginx workers, if requested +if [ -n "$NGINX_WORKER_OPEN_FILES" ] ; then + echo "worker_rlimit_nofile ${NGINX_WORKER_OPEN_FILES};" >> /etc/nginx/nginx.conf +fi + +# Get the listen port for Nginx, default to 80 +USE_LISTEN_PORT=${LISTEN_PORT:-80} +# Modify Nignx config for listen port +if ! grep -q "listen ${USE_LISTEN_PORT};" /etc/nginx/nginx.conf ; then + sed -i -e "/server {/a\ listen ${USE_LISTEN_PORT};" /etc/nginx/nginx.conf +fi +exec "$@" diff --git a/nginx.conf b/nginx.conf new file mode 100755 index 0000000000..4334c0ea36 --- /dev/null +++ b/nginx.conf @@ -0,0 +1,51 @@ +user nginx; +worker_processes 1; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + ## + # Logging Settings + ## + log_format json '{"gen3log": "nginx", ' + '"date_access": "$time_iso8601", ' + '"user_id": "$http_x_userid", ' + '"request_id": "$http_x_reqid", ' + '"session_id": "$http_x_sessionid", ' + '"visitor_id": "$http_x_visitorid", ' + '"network_client_ip": "$http_x_forwarded_for", ' + '"network_bytes_write": $body_bytes_sent, ' + '"response_secs": $request_time, ' + '"http_status_code": $status, ' + '"http_request": "$request_uri", ' + '"http_verb": "$request_method", ' + '"http_referer": "$http_referer", ' + '"http_useragent": "$http_user_agent", ' + '"message": "$request"}'; + + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log json; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/*.conf; +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 528c445650..ed8364489e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,42 +1 @@ -Authlib==0.11 -addict==2.1.1 -authutils>=4.0.0<5.0.0 -boto>=2.36.0<3.0.0 -botocore>=1.7<1.10.39 -boto3>=1.5<1.6 -cached_property==1.5.1 -cdislogging>=1.0.0<2.0.0 -cdiserrors==0.1.2 -cdispyutils==1.0.2 -cryptography>=2.1.2<3.0 -datamodelutils==0.4.5 -Flask==1.1.1 -Flask-CORS==3.0.3 -Flask_OAuthlib==0.9.4 -flask-restful==0.3.6 -Flask_SQLAlchemy_Session==1.1 -gen3authz==0.2.3 -gen3config==0.1.7 -gen3cirrus==1.1.1 -gen3users -httplib2==0.10.3 -markdown==3.1.1 -python-jose==2.0.2 -oauthlib==3.0.0 -requests_oauthlib==1.2.0 -psycopg2==2.8.3 -pysftp==0.2.9 -pytest-flask==0.10.0 -pytest==3.2.3 -python_dateutil==2.6.1 -PyJWT==1.5.3 -requests>=2.18.0<3.0.0 -setuptools==36.6.0 -six==1.11.0 -SQLAlchemy==1.3.3 -temps==0.3.0 -userdatamodel==2.1.1 -Werkzeug==0.16.0 -pyyaml==5.1 -retry==0.9.2 git+https://github.com/uc-cdis/storage-client.git@1.0.0#egg=storageclient diff --git a/setup.py b/setup.py index 5c7cd89116..4c844e6da4 100644 --- a/setup.py +++ b/setup.py @@ -4,8 +4,9 @@ name="fence", version="0.2.0", install_requires=[ - "Authlib", + "Authlib==0.12.1", "oauth2client<4.0dev,>=2.0.0", + "authutils>=4.0.0<5.0.0", "addict>=2.1.1, <3.0.0", "boto>=2.36.0,<3.0.0", "botocore>=1.7,<1.9.0", @@ -24,7 +25,7 @@ "httplib2>=0.10.3,<1.0.0", "markdown>=3.1.1,<4.0.0", "python-jose>=2.0.0,<3.0.0", - "oauthlib>=3.0.0,<4.0.0", + "oauthlib==2.1.0", "psycopg2>=2.7.3.2,<3.0.0.0", "pysftp>=0.2.9,<1.0.0", "pytest-flask>=0.10.0,<1.0.0", @@ -33,6 +34,7 @@ "python_dateutil>=2.6.1,<3.0.0", "PyJWT>=1.5.3,<2.0.0", "requests>=2.18.0,<3.0.0", + "requests-oauthlib==1.1.0", "six>=1.11.0,<2.0.0", "SQLAlchemy>=1.3.3,<1.4.0", "temps>=0.3.0,<1.0.0", diff --git a/supervisord.ini b/supervisord.ini new file mode 100755 index 0000000000..3a6323227a --- /dev/null +++ b/supervisord.ini @@ -0,0 +1,18 @@ +[supervisord] +nodaemon=true + +[program:uwsgi] +command=/usr/sbin/uwsgi --ini /etc/uwsgi/uwsgi.ini --die-on-term --need-app --plugin python3 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +[program:nginx] +command=/usr/sbin/nginx +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 +# Graceful stop, see http://nginx.org/en/docs/control.html +stopsignal=QUIT diff --git a/uwsgi.conf b/uwsgi.conf new file mode 100644 index 0000000000..cac90f36a0 --- /dev/null +++ b/uwsgi.conf @@ -0,0 +1,38 @@ +server { + listen 80; + + location / { + uwsgi_param REMOTE_ADDR $http_x_forwarded_for if_not_empty; + uwsgi_param REMOTE_USER $http_x_userid if_not_empty; + uwsgi_param REMOTE_REQID $http_x_reqid if_not_empty; + uwsgi_param REMOTE_SESSIONID $http_x_sessionid if_not_empty; + uwsgi_param REMOTE_VISITORID $http_x_visitorid if_not_empty; + uwsgi_param GEN3_REQUEST_TIMESTAMP $msec; + uwsgi_param GEN3_TIMEOUT_SECONDS GEN3_UWSGI_TIMEOUT; + + include uwsgi_params; + uwsgi_pass unix:/var/run/gen3/uwsgi.sock; + uwsgi_read_timeout GEN3_UWSGI_TIMEOUT; + uwsgi_send_timeout GEN3_UWSGI_TIMEOUT; + } + + location /_status { + include uwsgi_params; + uwsgi_pass unix:/var/run/gen3/uwsgi.sock; + uwsgi_param GEN3_REQUEST_TIMESTAMP $msec; + uwsgi_param GEN3_TIMEOUT_SECONDS GEN3_UWSGI_TIMEOUT; + uwsgi_read_timeout GEN3_UWSGI_TIMEOUT; + uwsgi_ignore_client_abort on; + access_log off; + } + + error_page 502 /502.html; + location /502.html { + return 504 '{"error": "Request Timeout or Service Unavailable"}'; + } + + error_page 504 /504.html; + location /504.html { + return 504 '{"error": "Request Timeout"}'; + } +} \ No newline at end of file diff --git a/uwsgi.ini b/uwsgi.ini new file mode 100755 index 0000000000..e1d9f37a0c --- /dev/null +++ b/uwsgi.ini @@ -0,0 +1,6 @@ +[uwsgi] +socket = /tmp/uwsgi.sock +chown-socket = nginx:nginx +chmod-socket = 664 +# Graceful shutdown on SIGTERM, see https://github.com/unbit/uwsgi/issues/849#issuecomment-118869386 +hook-master-start = unix_signal:15 gracefully_kill_them_all \ No newline at end of file