Merge 4ab0068 into 9826912

uc-cdis · Oct 22, 2020 · 9e26bc3 · 9e26bc3
2 parents 9826912 + 4ab0068
commit 9e26bc3
Show file tree

Hide file tree

Showing 24 changed files with 2,364 additions and 188 deletions.
diff --git a/.gitignore b/.gitignore
@@ -104,4 +104,6 @@ ENV/
 # jwt keys
 keys
 tests/resources/keys/*.pem
+
 .DS_Store
+.vscode
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -4,6 +4,7 @@ repos:
     hooks:
     -   id: detect-secrets
         args: ['--baseline', '.secrets.baseline']
+        exclude: poetry.lock
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v2.5.0
     hooks:
@@ -12,6 +13,6 @@ repos:
     -   id: no-commit-to-branch
         args: [--branch, develop, --branch, master, --pattern, release/.*]
 -   repo: https://github.com/psf/black
-    rev: 19.10b0
+    rev: 20.8b1
     hooks:
     -   id: black
diff --git a/.secinclude b/.secinclude
@@ -1,8 +1,7 @@
 bin/*
-dev-requirements.txt
 cfg_help.py
 fence/*
-requirements.txt
 run.py
-setup.py
 wsgi.py
+poetry.lock
+pyproject.toml
diff --git a/.secrets.baseline b/.secrets.baseline
@@ -1,9 +1,9 @@
 {
   "exclude": {
-    "files": null,
+    "files": "poetry.lock",
     "lines": null
   },
-  "generated_at": "2020-07-20T16:22:58Z",
+  "generated_at": "2020-10-22T16:33:03Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -61,16 +61,14 @@
     "bin/fence-create": [
       {
         "hashed_secret": "07cb451426a70236a0047e0f390f2bd1d79fd6d3",
-        "is_secret": false,
         "is_verified": false,
-        "line_number": 503,
+        "line_number": 515,
         "type": "Secret Keyword"
       }
     ],
     "fence/blueprints/storage_creds/google.py": [
       {
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 139,
         "type": "Private Key"
@@ -79,7 +77,6 @@
     "fence/blueprints/storage_creds/other.py": [
       {
         "hashed_secret": "98c144f5ecbb4dbe575147a39698b6be1a5649dd",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 66,
         "type": "Base64 High Entropy String"
@@ -88,30 +85,26 @@
     "fence/config-default.yaml": [
       {
         "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 31,
         "type": "Basic Auth Credentials"
       },
       {
         "hashed_secret": "5d07e1b80e448a213b392049888111e1779a52db",
-        "is_secret": false,
         "is_verified": false,
-        "line_number": 508,
+        "line_number": 510,
         "type": "Secret Keyword"
       }
     ],
     "fence/local_settings.example.py": [
       {
         "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 6,
         "type": "Basic Auth Credentials"
       },
       {
         "hashed_secret": "5d07e1b80e448a213b392049888111e1779a52db",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 63,
         "type": "Secret Keyword"
@@ -120,7 +113,6 @@
     "fence/resources/google/utils.py": [
       {
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 277,
         "type": "Private Key"
@@ -129,60 +121,52 @@
     "fence/utils.py": [
       {
         "hashed_secret": "8318df9ecda039deac9868adf1944a29a95c7114",
-        "is_secret": false,
         "is_verified": false,
-        "line_number": 103,
+        "line_number": 104,
         "type": "Secret Keyword"
       }
     ],
     "openapis/swagger.yaml": [
       {
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
-        "is_secret": false,
         "is_verified": false,
-        "line_number": 1861,
+        "line_number": 1863,
         "type": "Private Key"
       },
       {
         "hashed_secret": "bb8e48bd1e73662027a0f0b876b695d4c18f5ed4",
-        "is_secret": false,
         "is_verified": false,
-        "line_number": 1861,
+        "line_number": 1863,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "7861ab65194de92776ab9cd06d4d7e7e1ec2c36d",
-        "is_secret": false,
         "is_verified": false,
-        "line_number": 1941,
+        "line_number": 1943,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "d6b66ddd9ea7dbe760114bfe9a97352a5e139134",
-        "is_secret": false,
         "is_verified": false,
-        "line_number": 1963,
+        "line_number": 1965,
         "type": "JSON Web Token"
       },
       {
         "hashed_secret": "98c144f5ecbb4dbe575147a39698b6be1a5649dd",
-        "is_secret": false,
         "is_verified": false,
-        "line_number": 1975,
+        "line_number": 1977,
         "type": "Base64 High Entropy String"
       }
     ],
     "tests/conftest.py": [
       {
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1037,
         "type": "Private Key"
       },
       {
         "hashed_secret": "227dea087477346785aefd575f91dd13ab86c108",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1060,
         "type": "Base64 High Entropy String"
@@ -191,7 +175,6 @@
     "tests/credentials/google/test_credentials.py": [
       {
         "hashed_secret": "22afbfecd4124e2eb0e2a79fafdf62b207a8f8c7",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 579,
         "type": "Secret Keyword"
@@ -200,7 +183,6 @@
     "tests/keys/2018-05-01T21:29:02Z/jwt_private_key.pem": [
       {
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1,
         "type": "Private Key"
@@ -209,16 +191,22 @@
     "tests/login/test_fence_login.py": [
       {
         "hashed_secret": "d300421e208bfd0d432294de15169fd9b8975def",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 40,
         "type": "Secret Keyword"
       }
     ],
+    "tests/ras/test_ras.py": [
+      {
+        "hashed_secret": "d9db6fe5c14dc55edd34115cdf3958845ac30882",
+        "is_verified": false,
+        "line_number": 271,
+        "type": "Hex High Entropy String"
+      }
+    ],
     "tests/test-fence-config.yaml": [
       {
         "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 31,
         "type": "Basic Auth Credentials"

diff --git a/.travis.yml b/.travis.yml
@@ -11,12 +11,10 @@ addons:
   postgresql: "9.6"
 
 install:
-  - pip uninstall -y six || true # travis installs wrong version
-  - pip uninstall -y userdatamodel || true
-  - pip install -r dev-requirements.txt
-  - pip install -r requirements.txt
+  - curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python
+  - source $HOME/.poetry/env
+  - poetry install -vv
   - psql -c 'SELECT version();' -U postgres
-  - python setup.py install
   - psql -U postgres -c "create database fence_test_tmp"
   - pip list
 
@@ -27,7 +25,7 @@ before_script:
   - cd -
 
 script:
-  - py.test -vv --cov=fence --cov-report xml tests
+  - poetry run pytest -vv --cov=fence --cov-report xml tests
 
 after_script:
   - python-codacy-coverage -r coverage.xml

diff --git a/Dockerfile b/Dockerfile
@@ -8,16 +8,8 @@ ENV appname=fence
 RUN apk update \
     && apk add postgresql-libs postgresql-dev libffi-dev libressl-dev \
     && apk add linux-headers musl-dev gcc \
-    && apk add curl bash git vim make lftp
-
-COPY . /$appname
-COPY ./deployment/uwsgi/uwsgi.ini /etc/uwsgi/uwsgi.ini
-COPY ./deployment/uwsgi/wsgi.py /$appname/wsgi.py
-WORKDIR /$appname
-
-RUN python -m pip install --upgrade pip \
-    && python -m pip install --upgrade setuptools \
-    && pip install -r requirements.txt
+    && apk add curl bash git vim make lftp \
+    && apk update && apk add openssh && apk add libmcrypt-dev
 
 RUN mkdir -p /var/www/$appname \
     && mkdir -p /var/www/.cache/Python-Eggs/ \
@@ -27,8 +19,6 @@ RUN mkdir -p /var/www/$appname \
     && chown nginx -R /var/www/.cache/Python-Eggs/ \
     && chown nginx /var/www/$appname
 
-RUN apk update && apk add openssh && apk add libmcrypt-dev
-
 #
 # libmhash is required by mcrypt - below - no apk package available
 #
@@ -50,9 +40,31 @@ RUN (cd /tmp \
   && /bin/rm -rf /tmp/*)
 EXPOSE 80
 
+# aws cli v2 - needed for storing files in s3 during usersync k8s job
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \
+    && unzip awscliv2.zip \
+    && ./aws/install \
+    && /bin/rm -rf awscliv2.zip ./aws
+
+# install poetry
+RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python
+
+COPY . /$appname
+COPY ./deployment/uwsgi/uwsgi.ini /etc/uwsgi/uwsgi.ini
+COPY ./deployment/uwsgi/wsgi.py /$appname/wsgi.py
+WORKDIR /$appname
+
+# cache so that poetry install will run if these files change
+COPY poetry.lock pyproject.toml /$appname/
+
+# install Fence and dependencies via poetry
+RUN source $HOME/.poetry/env \
+    && poetry config virtualenvs.create false \
+    && poetry install -vv --no-dev --no-interaction \
+    && poetry show -v
+
 RUN COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" >$appname/version_data.py \
-    && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >>$appname/version_data.py \
-    && python setup.py develop
+    && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >>$appname/version_data.py
 
 WORKDIR /var/www/$appname
 

diff --git a/DockerfileShib b/DockerfileShib
@@ -1,6 +1,10 @@
 # To run: docker run -d -v /path/to/fence-config.yaml:/var/www/fence/fence-config.yaml --name=fence -p 80:80 fence
 # To check running container: docker exec -it fence /bin/bash
 
+# (pauline, 07/20/2020) Dockerfile for the fence-shib image.
+# This Dockerfile is NOT compatible yet with the latest Fence (Python 3
+# and depdencency management via Poetry) - for now, use Fence 2.7.x.
+
 FROM ubuntu:16.04
 
 ENV DEBIAN_FRONTEND=noninteractive

diff --git a/README.md b/README.md
@@ -221,11 +221,11 @@ See [Fence and Google](docs/google_architecture.md) for more details on data acc
 
 #### Install Requirements and Fence
 
+Install [Poetry](https://python-poetry.org/docs/#installation).
+
 ```bash
-# Install requirements.
-pip install -r requirements.txt
-# Install fence in your preferred manner.
-python setup.py develop
+# Install Fence and dependencies
+poetry install
 ```
 
 #### Create Configuration File

diff --git a/TECHDEBT.md b/TECHDEBT.md
@@ -5,7 +5,7 @@
 - Impact: (If this tech debt affected your work somehow, add a +1 here with a date and note)
   - +1 Zoe 2020 July 15 This is an example of a +1
   - +1 Vahid Oct 2020
-  
+
 ##### Problem:
 Fence puts OAuth2 scopes into the 'aud' claim of access tokens.
 ##### Why it was done this way:

diff --git a/bin/fence-create b/bin/fence-create
@@ -109,11 +109,11 @@ def parse_arguments():
     client_modify.add_argument("--description", required=False)
     client_modify.add_argument("--allowed-scopes", required=False, nargs="+")
     client_modify.add_argument(
-        "--append", 
+        "--append",
         help="append either new allowed scopes or urls instead of replacing",
         action="store_true",
         default=False,
-        )
+    )
     client_modify.add_argument(
         "--set-auto-approve",
         help="set the oidc process to skip user consent step",
@@ -447,7 +447,10 @@ def main():
         )
     elif args.action == "dbgap-download-access-files":
         download_dbgap_files(
-            dbGaP, STORAGE_CREDENTIALS, DB, folder=args.folder,
+            dbGaP,
+            STORAGE_CREDENTIALS,
+            DB,
+            folder=args.folder,
         )
     elif args.action == "google-manage-keys":
         remove_expired_google_service_account_keys(DB)

diff --git a/deployment/uwsgi/uwsgi.ini b/deployment/uwsgi/uwsgi.ini
@@ -23,6 +23,8 @@ vacuum = true
 pythonpath = /var/www/fence/
 pythonpath = /fence/
 pythonpath = /usr/local/lib/python3.6/site-packages/
+# poetry installs git dependencies at /usr/local/src
+pythonpath = /usr/local/src/*
 
 # Initialize application in worker processes, not master. This prevents the
 # workers from all trying to open the same database connections at startup.

diff --git a/dev-requirements.txt b/dev-requirements.txt
diff --git a/docs/google_architecture.md b/docs/google_architecture.md
@@ -114,7 +114,7 @@ Quick summary of minimal end-user actions to be able to specify `userProject`:
 * Click "add", Click "Select a role" dropdown, then "Manage roles", then "Create new role"/"Add role"
     * For the role: name it "BillingAdmin", add the single permission `serviceusage.services.use`
 * Go back to "IAM"
-* Click "add", Click "Select a role" dropdown, select new "BillingAdmin" custom role, in "New members" field, paste the email you copied from `primary_google_service_account` 
+* Click "add", Click "Select a role" dropdown, select new "BillingAdmin" custom role, in "New members" field, paste the email you copied from `primary_google_service_account`
 * Now you can specify the `userProject` in signed URL requests to be the project ID for the project you just setup IAM billing permission on. `userProject=YOUR-GOOGLE-PROJECT-ID`
 
 #### Required Google Cloud Platform (GCP) Configuration for Billing Project