Merge 655a106 into a0d91bc

uc-cdis · Feb 20, 2020 · 18e18f0 · 18e18f0
2 parents a0d91bc + 655a106
commit 18e18f0
Show file tree

Hide file tree

Showing 6 changed files with 387 additions and 177 deletions.
diff --git a/bin/fence-create b/bin/fence-create
@@ -352,6 +352,8 @@ def main():
         os.path.dirname(os.path.realpath(__file__))
     )
     dbGaP = os.environ.get("dbGaP") or config.get("dbGaP")
+    if not isinstance(dbGaP, list):
+        dbGaP = [dbGaP]
     STORAGE_CREDENTIALS = os.environ.get("STORAGE_CREDENTIALS") or config.get(
         "STORAGE_CREDENTIALS"
     )

diff --git a/docs/usersync.md b/docs/usersync.md
@@ -2,12 +2,40 @@
 
 Usersync is a script that parses user access information from multiple sources (user.yaml files, dbGaP user authorization "telemetry" files AKA whitelists) and keeps users' access to Gen3 resources up to date by updating the Fence and Arborist databases.
 
+
+
 ## Usersync flow
 
 ![Usersync Flow](images/usersync.png)
 
 > The access from the user.yaml file and the dbGaP authorization files is combined (see example below), but the user.yaml file overrides the user information (such as email) obtained from the dbGaP authorization files.
 
+## Configuration
+
+Configuration for user sync lives in fence-config.yaml for each respective environment. An example of the fence-config can be found [fence/config-default.yaml](https://github.com/uc-cdis/fence/blob/master/fence/config-default.yaml).
+
+You can configure one or more dbGaP SFTP servers to sync telemetry files from. To configure one single dbGaP server, add credentials and information to the fence-config.yaml under `dbGaP`, this is outlined in [here](https://github.com/uc-cdis/fence/blob/4.14.0/fence/config-default.yaml#L389-L433)
+
+To configure additional dbGaP servers, include in the config.yaml a list of dbGaP servers under `dbGaP`, like so:
+
+```
+dbGaP:
+- info:
+    host:
+    username:
+    password:
+    ...
+  protocol: 'sftp'
+  ...
+  ...
+- info:
+    host:
+    username:
+    ...
+```
+
+An example can be found in the config used for unit testing [tests/test-fence-config.yaml](https://github.com/uc-cdis/fence/blob/master/tests/test-fence-config.yaml)
+
 ## Usersync result example
 
 ### Example of user.yaml file:
@@ -18,7 +46,7 @@ Usersync is a script that parses user access information from multiple sources (
 ```
 # authz information follows the attribute-based access control (ABAC) model
 authz:
-  resources: 
+  resources:
     - name: programs
       subresources:
         - name: myprogram

diff --git a/fence/config-default.yaml b/fence/config-default.yaml
@@ -387,50 +387,50 @@ ITRUST_GLOBAL_LOGOUT: 'https://auth.nih.gov/siteminderagent/smlogout.asp?mode=ni
 # User syncing and access can also be done throught a User Access file. See
 # fence's README for more information
 dbGaP:
-  info:
-    host: ''
-    username: ''
-    password: ''
-    port: 22
-    proxy: ''
-    proxy_user: ''
-  protocol: 'sftp'
-  decrypt_key: ''
-  # parse out the consent from the dbgap accession number such that something
-  # like "phs000123.v1.p1.c2" becomes "phs000123.c2".
-  #
-  # NOTE: when this is "false" the above would become "phs000123"
-  parse_consent_code: true
-  # A consent of "c999" can indicate access to that study's "exchange area data"
-  # and when a user has access to one study's exchange area data, they
-  # have access to the parent study's "common exchange area data" that is not study
-  # specific. The following config is whether or not to parse/handle "c999" codes
-  # for access to the common exchange area data
-  #
-  # NOTE: When enabled you MUST also provide a mapping to the
-  # `study_common_exchange_areas` from study -> parent common exchange area resource
-  enable_common_exchange_area_access: false
-  # The below configuration is a mapping from studies to their "common exchange area data"
-  # Fence project name a user gets access to when parsing c999 exchange area codes (and
-  # subsequently gives access to an Arborist resource representing this common area
-  # as well)
-  study_common_exchange_areas:
-    'example': 'test_common_exchange_area'
-    # 'studyX': 'test_common_exchange_area'
-    # 'studyY': 'test_common_exchange_area'
-    # 'studyZ': 'test_common_exchange_area'
-  # A mapping from the dbgap study / Fence project to which authorization namespaces the
-  # actual data lives in. For example, `studyX` data may exist in multiple organizations, so
-  # we need to know how to map authorization to all orgs resources
-  study_to_resource_namespaces:
-    '_default': ['/']
-    'test_common_exchange_area': ['/dbgap/']
-    # above are for default support and exchange area support
-    # below are further examples
+  - info:
+      host: ''
+      username: ''
+      password: ''
+      port: 22
+      proxy: ''
+      proxy_user: ''
+    protocol: 'sftp'
+    decrypt_key: ''
+    # parse out the consent from the dbgap accession number such that something
+    # like "phs000123.v1.p1.c2" becomes "phs000123.c2".
+    #
+    # NOTE: when this is "false" the above would become "phs000123"
+    parse_consent_code: true
+    # A consent of "c999" can indicate access to that study's "exchange area data"
+    # and when a user has access to one study's exchange area data, they
+    # have access to the parent study's "common exchange area data" that is not study
+    # specific. The following config is whether or not to parse/handle "c999" codes
+    # for access to the common exchange area data
     #
-    # 'studyX': ['/orgA/', '/orgB/']
-    # 'studyX.c2': ['/orgB/', '/orgC/']
-    # 'studyZ': ['/orgD/']
+    # NOTE: When enabled you MUST also provide a mapping to the
+    # `study_common_exchange_areas` from study -> parent common exchange area resource
+    enable_common_exchange_area_access: false
+    # The below configuration is a mapping from studies to their "common exchange area data"
+    # Fence project name a user gets access to when parsing c999 exchange area codes (and
+    # subsequently gives access to an Arborist resource representing this common area
+    # as well)
+    study_common_exchange_areas:
+      'example': 'test_common_exchange_area'
+      # 'studyX': 'test_common_exchange_area'
+      # 'studyY': 'test_common_exchange_area'
+      # 'studyZ': 'test_common_exchange_area'
+    # A mapping from the dbgap study / Fence project to which authorization namespaces the
+    # actual data lives in. For example, `studyX` data may exist in multiple organizations, so
+    # we need to know how to map authorization to all orgs resources
+    study_to_resource_namespaces:
+      '_default': ['/']
+      'test_common_exchange_area': ['/dbgap/']
+      # above are for default support and exchange area support
+      # below are further examples
+      #
+      # 'studyX': ['/orgA/', '/orgB/']
+      # 'studyX.c2': ['/orgB/', '/orgC/']
+      # 'studyZ': ['/orgD/']
 
 # Regex to match an assession number that has consent information in forms like:
 #   phs00301123.c999
@@ -484,21 +484,21 @@ AWS_CREDENTIALS:
     aws_access_key_id: ''
     aws_secret_access_key: ''
 
-# NOTE: the region is optonal for s3_buckets, however it should be specified to avoid a 
-# call to GetBucketLocation which you make lack the AWS ACLs for. 
+# NOTE: the region is optonal for s3_buckets, however it should be specified to avoid a
+# call to GetBucketLocation which you make lack the AWS ACLs for.
 S3_BUCKETS:
   bucket1:
     cred: 'CRED1'
-    region: 'us-east-1' 
+    region: 'us-east-1'
   bucket2:
     cred: 'CRED2'
-    region: 'us-east-1' 
+    region: 'us-east-1'
   bucket3:
     cred: '*'
-    region: 'us-east-1' 
+    region: 'us-east-1'
   bucket4:
     cred: 'CRED1'
-    region: 'us-east-1' 
+    region: 'us-east-1'
     role-arn: 'arn:aws:iam::role1'
 
 # `DATA_UPLOAD_BUCKET` specifies an S3 bucket to which data files are uploaded,