Merge 1d78d66 into 3765ec8

.gitignore

@@ -104,4 +104,6 @@ ENV/
 # jwt keys
 keys
 tests/resources/keys/*.pem
+
 .DS_Store
+.vscode

.secinclude

@@ -1,8 +1,7 @@
 bin/*
-dev-requirements.txt
 cfg_help.py
 fence/*
-requirements.txt
 run.py
-setup.py
 wsgi.py
+poetry.lock
+pyproject.toml

.travis.yml

@@ -11,12 +11,10 @@ addons:
   postgresql: "9.6"
 
 install:
-  - pip uninstall -y six || true # travis installs wrong version
-  - pip uninstall -y userdatamodel || true
-  - pip install -r dev-requirements.txt
-  - pip install -r requirements.txt
+  - curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python
+  - source $HOME/.poetry/env
+  - poetry install -vv
   - psql -c 'SELECT version();' -U postgres
-  - python setup.py install
   - psql -U postgres -c "create database fence_test_tmp"
   - pip list
 
@@ -27,7 +25,7 @@ before_script:
   - cd -
 
 script:
-  - py.test -vv --cov=fence --cov-report xml tests
+  - poetry run pytest -vv --cov=fence --cov-report xml tests
 
 after_script:
   - python-codacy-coverage -r coverage.xml

Dockerfile

@@ -15,10 +15,6 @@ COPY ./deployment/uwsgi/uwsgi.ini /etc/uwsgi/uwsgi.ini
 COPY ./deployment/uwsgi/wsgi.py /$appname/wsgi.py
 WORKDIR /$appname
 
-RUN python -m pip install --upgrade pip \
-    && python -m pip install --upgrade setuptools \
-    && pip install -r requirements.txt
-
 RUN mkdir -p /var/www/$appname \
     && mkdir -p /var/www/.cache/Python-Eggs/ \
     && mkdir /run/nginx/ \
@@ -50,9 +46,33 @@ RUN (cd /tmp \
   && /bin/rm -rf /tmp/*)
 EXPOSE 80
 
+# aws cli v2 - needed for storing files in s3 during usersync k8s job
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \
+    && unzip awscliv2.zip \
+    && ./aws/install \
+    && /bin/rm -rf awscliv2.zip ./aws
+
+# install poetry
+ENV POETRY_HOME="$HOME/.poetry/bin"
+RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python
+# ENV PATH="$POETRY_HOME/bin$PATH"
+# ENV PATH="usr/local/src$PATH" <-- pythonpath
+ENV PYTHONPATH="${PYTHONPATH}:/usr/local/src/"
+RUN echo $PYTHONPATH
+
 RUN COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" >$appname/version_data.py \
-    && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >>$appname/version_data.py \
-    && python setup.py develop
+    && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >>$appname/version_data.py
+
+# cache so that poetry install will run if these files change
+COPY poetry.lock pyproject.toml /$appname
+
+# install Fence and dependencies via poetry
+RUN source $HOME/.poetry/env \
+    && poetry config virtualenvs.create false \
+    && poetry install -vv --no-dev --no-interaction \
+    && pip list
+RUN pip show flask
+RUN pip show gen3config
 
 WORKDIR /var/www/$appname
 

DockerfileShib

@@ -1,6 +1,10 @@
 # To run: docker run -d -v /path/to/fence-config.yaml:/var/www/fence/fence-config.yaml --name=fence -p 80:80 fence
 # To check running container: docker exec -it fence /bin/bash
 
+# (pauline, 07/20/2020) Dockerfile for the fence-shib image.
+# This Dockerfile is NOT compatible yet with the latest Fence (Python 3
+# and depdencency management via Poetry) - for now, use Fence 2.7.x.
+
 FROM ubuntu:16.04
 
 ENV DEBIAN_FRONTEND=noninteractive

README.md

@@ -220,11 +220,11 @@ See [Fence and Google](docs/google_architecture.md) for more details on data acc
 
 #### Install Requirements and Fence
 
+Install [Poetry](https://python-poetry.org/docs/#installation).
+
 ```bash
-# Install requirements.
-pip install -r requirements.txt
-# Install fence in your preferred manner.
-python setup.py develop
+# Install Fence and dependencies
+poetry install
 ```
 
 #### Create Configuration File

fence/blueprints/data/multipart_upload.py

@@ -1,5 +1,5 @@
 import boto3
-import botocore
+from botocore.exceptions import ClientError
 from retry.api import retry_call
 
 from cdispyutils.hmac4 import generate_aws_presigned_url
@@ -37,7 +37,7 @@ def initilize_multipart_upload(bucket, key, credentials):
             tries=MAX_TRIES,
             jitter=10,
         )
-    except botocore.exceptions.ClientError as error:
+    except ClientError as error:
         logger.error(
             "Error when create multiple part upload for object with uuid {}. Detail {}".format(
                 key, error
@@ -83,7 +83,7 @@ def complete_multipart_upload(bucket, key, credentials, uploadId, parts):
             tries=MAX_TRIES,
             jitter=10,
         )
-    except botocore.exceptions.ClientError as error:
+    except ClientError as error:
         logger.error(
             "Error when completing multiple part upload for object with uuid {}. Detail {}".format(
                 key, error

fence/sync/sync_users.py

@@ -12,7 +12,7 @@
 import paramiko
 from cdislogging import get_logger
 from email_validator import validate_email, EmailNotValidError
-from gen3authz.client.arborist.client import ArboristError
+from gen3authz.client.arborist.errors import ArboristError
 from gen3users.validation import validate_user_yaml
 from paramiko.proxy import ProxyCommand
 from sqlalchemy.exc import IntegrityError

notes.txt

@@ -0,0 +1,10 @@
+not included: boto, datamodelutils, Flask_OAuthlib, httplib2, oauthlib, requests_oauthlib, pysftp, setuptools, six, temps
+not included from setup.py: oauth2client, google_api_python_client
+
+awscli (see https://github.com/uc-cdis/fence/pull/745#discussion_r423441492) 1.18.0 conflicts with botocore<1.13.0 and s3transfer, dependency of storageclient
+-> add it in dockerfile instead of poetry
+
+cdiserrors <1: gen3config, cdis-python-utils, authutils
+cdiserrors >=1: gen3authz
+fence works on 4.0.0 but not on 5.0.0 (NIH login through GDC issues?)
+-> to discuss