merge master

.github/workflows/wool.yml

@@ -0,0 +1,18 @@
+on:
+  pull_request:
+  issue_comment:
+    types: [created, edited]
+
+name: Wool
+
+jobs:
+  runWool:
+    name: Run black
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@master
+
+    - uses: uc-cdis/wool@master
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+

.secinclude

@@ -0,0 +1,8 @@
+bin/*
+dev-requirements.txt
+cfg_help.py
+fence/*
+requirements.txt
+run.py
+setup.py
+wsgi.py

.travis.yml

@@ -1,22 +1,14 @@
 language: python
 
 python:
-  - "2.7"
   - "3.6"
 
-matrix:
- allow_failures:
-   - python: "3.6"
-
 sudo: false
 
 cache: pip
 
 addons:
-  postgresql: "9.4"
-
-env:
-  - REPOSITORY="uc-cdis/fence" PR_NUMBER="$TRAVIS_PULL_REQUEST"
+  postgresql: "9.6"
 
 install:
   - pip uninstall -y six || true # travis installs wrong version
@@ -26,8 +18,7 @@ install:
   - psql -c 'SELECT version();' -U postgres
   - python setup.py install
   - psql -U postgres -c "create database fence_test_tmp"
-  - if [[ $TRAVIS_PYTHON_VERSION != 3.6 ]]; then userdatamodel-init --db fence_test_tmp; fi
-  - if [[ $TRAVIS_PYTHON_VERSION == 3.6 ]]; then pip install -e git+https://git@github.com/uc-cdis/wool.git#egg=wool; fi
+  - pip list
 
 before_script:
   - sudo rm -f /etc/boto.cfg
@@ -40,4 +31,4 @@ script:
 
 after_script:
   - python-codacy-coverage -r coverage.xml
-  - if [[ $TRAVIS_PYTHON_VERSION == 3.6 && $PR_NUMBER != false ]]; then wool; fi
+  - COVERALLS_REPO_TOKEN=$COVERALLS_TOKEN coveralls

Dockerfile

@@ -1,59 +1,59 @@
 # To run: docker run -d -v /path/to/fence-config.yaml:/var/www/fence/fence-config.yaml --name=fence -p 80:80 fence
 # To check running container: docker exec -it fence /bin/bash
 
-FROM ubuntu:16.04
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    apache2 \
-    build-essential \
-    curl \
-    git \
-    # for ftp
-    lftp \
-    # for decryption dbgap files
-    mcrypt \
-    libapache2-mod-wsgi \
-    # dependency for cryptography
-    libffi-dev \
-    # dependency for pyscopg2 - which is dependency for sqlalchemy postgres engine
-    libpq-dev \
-    # dependency for cryptography
-    libssl-dev \
-    # dependency for ssh and sftp
-    openssh-client \
-    python2.7 \
-    python-dev \
-    python-pip \
-    python-setuptools \
-    vim \
-    && pip install pip==9.0.3 \
-    && pip install --upgrade setuptools \
-    && mkdir /var/www/fence \
-    && mkdir -p /var/www/.cache/Python-Eggs/ \
-    && chown www-data -R /var/www/.cache/Python-Eggs/
+FROM quay.io/cdis/python-nginx:pybase3-1.2.0
+
+ENV appname=fence
+
+RUN apk update \
+    && apk add postgresql-libs postgresql-dev libffi-dev libressl-dev \
+    && apk add linux-headers musl-dev gcc \
+    && apk add curl bash git vim make
 
+COPY . /$appname
+COPY ./deployment/uwsgi/uwsgi.ini /etc/uwsgi/uwsgi.ini
+COPY ./deployment/uwsgi/wsgi.py /$appname/wsgi.py
+WORKDIR /$appname
 
-COPY requirements.txt /fence/requirements.txt
-RUN pip install -r /fence/requirements.txt
+RUN python -m pip install --upgrade pip \
+    && python -m pip install --upgrade setuptools \
+    && pip install -r requirements.txt
+
+RUN mkdir -p /var/www/$appname \
+    && mkdir -p /var/www/.cache/Python-Eggs/ \
+    && mkdir /run/nginx/ \
+    && ln -sf /dev/stdout /var/log/nginx/access.log \
+    && ln -sf /dev/stderr /var/log/nginx/error.log \
+    && chown nginx -R /var/www/.cache/Python-Eggs/ \
+    && chown nginx /var/www/$appname
 
-COPY . /fence
-WORKDIR /fence
-COPY deployment/fence.conf /etc/apache2/sites-available/fence.conf
+RUN apk update && apk add openssh && apk add libmcrypt-dev
 
 #
-# Custom apache24 logging - see http://www.loadbalancer.org/blog/apache-and-x-forwarded-for-headers/
+# libmhash is required by mcrypt - below - no apk package available
 #
-RUN ln -s /fence/wsgi.py /var/www/fence/wsgi.py \
-    && COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" >fence/version_data.py \
-    && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >>fence/version_data.py \
-    && python setup.py develop \
-    && a2dissite 000-default \
-    && a2ensite fence \
-    && a2enmod reqtimeout \
-    && ln -sf /dev/stdout /var/log/apache2/access.log \
-    && ln -sf /dev/stderr /var/log/apache2/error.log
+RUN (cd /tmp \
+  && wget -O mhash.tar.gz https://sourceforge.net/projects/mhash/files/mhash/0.9.9.9/mhash-0.9.9.9.tar.gz/download \
+  && tar xvfz mhash.tar.gz \
+  && cd mhash-0.9.9.9 \
+  && ./configure && make && make install \
+  && /bin/rm -rf /tmp/*)
 
+#
+# mcrypt is required to decrypt dbgap user files - see fence/sync/sync_users.py
+#
+RUN (cd /tmp \
+  && wget -O mcrypt.tar.gz https://sourceforge.net/projects/mcrypt/files/MCrypt/Production/mcrypt-2.6.4.tar.gz/download \
+  && tar xvfz mcrypt.tar.gz \
+  && cd mcrypt-2.6.4 \
+  && ./configure && make && make install \
+  && /bin/rm -rf /tmp/*)
 EXPOSE 80
-WORKDIR /var/www/fence/
 
-CMD bash /fence/dockerrun.bash
+RUN COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" >$appname/version_data.py \
+    && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >>$appname/version_data.py \
+    && python setup.py develop
+
+WORKDIR /var/www/$appname
+
+CMD ["sh","-c","bash /fence/dockerrun.bash && /dockerrun.sh"]

Jenkinsfile.security

@@ -0,0 +1,6 @@
+#!groovy
+
+@Library('cdis-jenkins-lib@master') _
+
+securityPipeline {
+}

README.md

@@ -1,8 +1,10 @@
 # Fence
 
 [![Build Status](https://travis-ci.org/uc-cdis/fence.svg?branch=master)](https://travis-ci.org/uc-cdis/fence)
-[![Codacy Quality Badge](https://api.codacy.com/project/badge/Grade/1cb2ec9cc64049488d140f44027c4422)](https://www.codacy.com/app/uc-cdis/fence?utm_source=github.com&utm_medium=referral&utm_content=uc-cdis/fence&utm_campaign=Badge_Grade)
-[![Codacy Coverage Badge](https://api.codacy.com/project/badge/Coverage/1cb2ec9cc64049488d140f44027c4422)](https://www.codacy.com/app/uc-cdis/fence?utm_source=github.com&utm_medium=referral&utm_content=uc-cdis/fence&utm_campaign=Badge_Coverage)
+
+[![Codacy Badge](https://api.codacy.com/project/badge/Grade/41ff9d807efa4da8a733793b3539ba3e)](https://www.codacy.com/app/uc-cdis/fence?utm_source=github.com&utm_medium=referral&utm_content=uc-cdis/fence&utm_campaign=Badge_Grade)
+[![Coverage Status](https://coveralls.io/repos/github/uc-cdis/fence/badge.svg?branch=master)](https://coveralls.io/github/uc-cdis/fence?branch=master)
+
 
 A `fence` separates protected resources from the outside world and allows
 only trusted entities to enter.
@@ -18,7 +20,7 @@ Fence is a core service of the Gen3 stack that has multiple capabilities:
 
 1. [API Documentation](#API-documentation)
 1. [Terminologies](#Terminologies)
-1. [Identity Providers](#identity-provider)
+1. [Identity Providers](#identity-providers)
 1. [OIDC & OAuth2](#oidc--oauth2)
 1. [Accessing Data](#accessing-data)
 1. [Setup](#setup)
@@ -93,12 +95,11 @@ Relying Party - an OAuth 2.0 Client which uses (requests) OpenID Connect.
 Fence can be configured to support different Identity Providers (IdPs) for AuthN.
 At the moment, supported IDPs include:
 - Google
-- Shibboleth
+- [Shibboleth](docs/shibboleth.md)
   - NIH iTrust
   - InCommon
   - eduGAIN
 
-
 ## OIDC & OAuth2
 
 Fence acts as a central broker that supports multiple IdPs.
@@ -174,12 +175,12 @@ In the following flow, `Fence (Client Instance)` is an OP relative to `OAuth Cli
 See the [OIDC specification](http://openid.net/specs/openid-connect-core-1_0.html) for more details.
 Additionally, see the [OAuth2 specification](https://tools.ietf.org/html/rfc6749).
 
-## Role-Based Access Control
+## Access Control / Authz
 
 Currently fence works with another Gen3 service named
-[arborist](https://github.com/uc-cdis/arborist) to implement role-based access
+[arborist](https://github.com/uc-cdis/arborist) to implement attribute-based access
 control for commons users. The YAML file of access control information (see
-[#create-user-access-file]()) contains a section `rbac` which are data sent to
+[#create-user-access-file]()) contains a section `authz` which are data sent to
 arborist in order to set up the access control model.
 
 ## Accessing Data
@@ -494,9 +495,7 @@ As a Gen3 commons administrator, if you want to create an implicit oauth client
 
 ```bash
 fence-create client-create --client fancywebappname --urls 'https://betawebapp.example/fence
-https://webapp.example/fence' --public --username fancyapp --grant-types 'authorization_code
-refresh_token
-implicit'
+https://webapp.example/fence' --public --username fancyapp --grant-types authorization_code refresh_token implicit
 ```
 
 If there are more than one URL to add, use space to delimit them like this:
@@ -544,6 +543,7 @@ fence-create notify-problem-users --emails ex1@gmail.com ex2@gmail.com --auth_id
 ```
 
 `notify-problem-users` emails users in the provided list (can be fence user email or linked google email) who do not have access to any of the auth_ids provided. Also accepts a `check_linking` flag to check that each user has linked their google account.
+
 ## Default Expiration Times in Fence
 
 Table contains various artifacts in fence that have temporary lifetimes and their default values.