Refactor DCC configuration (#122)

* Move DCC config folders in dcc folder

* Add dcc tenant config file

* Move test fileview configs to test directory

* Add github link to each dcc config file

* Update links in dca demo config

* Remove dcc_config.csv in favour of tenants.json

* Update paths to template config files and logos for each DCC

* Fix name of EL dca config file

* Remove brackets from demo upsert config

* Remove out array from json config menu

* Update links to template drop down for tests

* Use relative filepaths instead of githubusercontent link to config files in demo config

* Use relative filepath to dca demo config in tenants.json

* Use relative filepath instead of url to reference config files.

* Update configs to use relative filepath instead of githubusercontent url

* Add schemas for tenants.json and dca_config.json

* Change tenants array to an oject

* change tenants from array to object and add tenant object

* Update dca config schema

* correct manifest submit type for GF

* Add uri-refernce format to the URL properties of dca config schema

* Change tenant schema to an array of objects. Define object specifications in defs

* Update tenants.json to an object of arrays

* Add github workflow to validate tenants.json

* Fix whitespace

* remove - from workflow name

* remove path trigger for testing

* use docker form of json validate

* switch json inputs to env vars

* fix variable name

* add repo to json filepath

* tenants.schema not tenant.schema

* maybe we don't need the repo dir

* remove filler ID field and try using http for schema

* test removing the schema

* commit an invalid tenants.json to test validation

* Fix tenants.json

* Add tenant json paths to trigger workflow

* Update workflow description

* test validate dcc config workflow

* fix indentation

* Modify smoke test config for test

* try filtering modified files

* fix smoke test config

* test smoke test change

* Change file to trigger dca config validation workflow

* remove extra echo

* test edit to config

* remove extra parenthesis from echo

* test edit to smoke test json

* print config var

* change smoke test

* remove single quotes in grep

* test formatting output with xargs

* remove schema from dca config schema for compatibility

* set output to json

* try splitting file changed list with tr

* use space-delimited string for changed file output

* quote echo

* echo the changed files variable

* create filter_file step id to reference varible

* fix smoke test config

* remove printing config variable for debugging

* Update template path for mrt test config

* change files changed string to csv instead of space-delimited to handle filenames with spaces

* pipe changed files output to tr again to remove new liens

* make json files list comma-separated

* print env variables to debug

* print env variables to debug 2

* print env variables to debug 3

* use sed to remove last comma

* Remove printing of env var for debugging

* test adding template config workflow in R

* remove template config generation workflow. This is in the example data model repo.

* Add json schema for template dropdown config

* Add workflow to validate template config files

* filter files that trigger validation workflows on PR

* only run validation if template json files have changed

.github/workflows/validate-dcc-config.yml

@@ -0,0 +1,39 @@
+# ------------------------------------------------------------------------------
+# Validate a dcc config file against the dca_config schema
+# ------------------------------------------------------------------------------
+name: validate dcc config
+on:
+  push:
+    paths:
+      - 'dcc/*dca_config.json'
+      - 'test/*dca_config.json'
+      - 'schemas/dca_config.schema.json'
+  pull_request:
+    branches:
+      - 'main'
+      - 'staging'
+      - 'dev'
+    paths:
+      - 'dcc/*dca_config.json'
+      - 'test/*dca_config.json'
+      - 'schemas/dca_config.schema.json'
+
+jobs:
+  validate-dcc-config:
+    runs-on: ubuntu-latest
+    steps:
+      - id: files
+        uses: masesgroup/retrieve-changed-files@v3
+        with:
+          format: 'csv'
+      - id: filter_files
+        name: Filter files to dca_config.json files
+        run: |
+          echo "configs=$(echo ${{ steps.files.outputs.added_modified }} | tr ',' '\n' | grep dca_config\\.json | tr '\n' ',' | sed 's/\(.*\),/\1 /')" >> "$GITHUB_OUTPUT"
+      - uses: actions/checkout@v4
+      - name: Validate JSON
+        uses: docker://orrosenblatt/validate-json-action:latest
+        env:
+          INPUT_SCHEMA: schemas/dca_config.schema.json
+          INPUT_JSONS: ${{ steps.filter_files.outputs.configs }}
+

.github/workflows/validate-template-config.yml

@@ -0,0 +1,39 @@
+# ------------------------------------------------------------------------------
+# Validate a dca template config file against the dca template config schema
+# ------------------------------------------------------------------------------
+name: Validate Template Config
+on:
+  push:
+    paths:
+      - 'dcc/*dca-template-config.json'
+      - 'test/*dca-template-config.json'
+      - 'schemas/dca_template_config.schema.json'
+  pull_request:
+    branches:
+      - 'main'
+      - 'staging'
+      - 'dev'
+    paths:
+      - 'dcc/*dca-template-config.json'
+      - 'test/*dca-template-config.json'
+      - 'schemas/dca_template_config.schema.json'
+
+jobs:
+  validate-template-config:
+    runs-on: ubuntu-latest
+    steps:
+      - id: files
+        uses: masesgroup/retrieve-changed-files@v3
+        with:
+          format: 'csv'
+      - id: filter_files
+        name: Filter files to dca-template-config.json files
+        run: |
+          echo "configs=$(echo ${{ steps.files.outputs.added_modified }} | tr ',' '\n' | grep dca-template-config\\.json | tr '\n' ',' | sed 's/\(.*\),/\1 /')" >> "$GITHUB_OUTPUT"
+      - uses: actions/checkout@v4
+      - name: Validate JSON
+        if: ${{ steps.filter_files.outputs.configs != '' }}
+        uses: docker://orrosenblatt/validate-json-action:latest
+        env:
+          INPUT_SCHEMA: schemas/dca_template_config.schema.json
+          INPUT_JSONS: ${{ steps.filter_files.outputs.configs }}

.github/workflows/validate-tenants-config.yml

@@ -0,0 +1,28 @@
+# ------------------------------------------------------------------------------
+# Validate the tenants.json file against the tenants.schema.json schema
+# ------------------------------------------------------------------------------
+name: validate tenants config
+on:
+  push:
+    paths:
+      - 'tenants.json'
+      - 'schemas/tenants.schema.json'
+  pull_request:
+    branches:
+      - 'main'
+      - 'staging'
+      - 'dev'
+    paths:
+      - 'tenants.json'
+      - 'schemas/tenants.schema.json'
+
+jobs:
+  validate-tenants-config:
+    runs-on: ubuntu-latest
+    steps:        
+      - uses: actions/checkout@v3
+      - name: Validate JSON
+        uses: docker://orrosenblatt/validate-json-action:latest
+        env:
+          INPUT_SCHEMA: schemas/tenants.schema.json
+          INPUT_JSONS: tenants.json

dcc/ADKP/adkp_dca_config.json

@@ -0,0 +1,19 @@
+{
+  "project_name":"AD Knowledge Portal",
+  "synapse_asset_view":"syn51324810",
+  "data_model_url":"https://raw.githubusercontent.com/adknowledgeportal/data-models/main/divco.data.model.v1.jsonld",
+  "template_menu_config_file":"dcc/ADKP/dca-template-config.json",
+  "logo_location":"dcc/ADKP/ADKnowledgePortal.png",
+  "logo_link":"https://adknowledgeportal.synapse.org",
+  "manifest_output_format":"excel",
+  "manifest_use_annotations":false,
+  "validate_restrict_rules":false,
+  "submit_use_schema_labels":true,
+  "submit_table_manipulation":"replace",
+  "submit_manifest_record_type":"file_only",
+  "submit_hide_blanks":false,
+  "use_compliance_dashboard":false,
+  "primary_col":"#2a668d",
+  "secondary_col":"#184e71",
+  "sidebar_col":"#191919"
+}

dcc/BTC/btc_dca_config.json

@@ -0,0 +1,17 @@
+{
+  "project_name":"BTC DCC",
+  "synapse_asset_view":"syn51407795",
+  "data_model_url":"https://raw.githubusercontent.com/Sage-Bionetworks/btc-data-models/main/btc.model.jsonld",
+  "template_menu_config_file":"dcc/BTC/dca-template-config.json",
+  "manifest_output_format":"excel",
+  "manifest_use_annotations":false,
+  "validate_restrict_rules":false,
+  "submit_use_schema_labels":true,
+  "submit_table_manipulation":"replace",
+  "submit_manifest_record_type":"table_and_file",
+  "submit_hide_blanks":false,
+  "use_compliance_dashboard":false,
+  "primary_col":"#2a668d",
+  "secondary_col":"#184e72",
+  "sidebar_col":"#191920"
+}

dcc/EL/el_dca_config.json

@@ -0,0 +1,17 @@
+{
+  "project_name":"EL",
+  "synapse_asset_view":"syn51753858",
+  "data_model_url":"https://raw.githubusercontent.com/Sage-Bionetworks/ELITE-data-models/main/example.model.jsonld",
+  "template_menu_config_file":"dcc/EL/dca-template-config.json",
+  "manifest_output_format":"excel",
+  "manifest_use_annotations":false,
+  "validate_restrict_rules":false,
+  "submit_use_schema_labels":true,
+  "submit_table_manipulation":"replace",
+  "submit_manifest_record_type":"table_and_file",
+  "submit_hide_blanks":false,
+  "use_compliance_dashboard":false,
+  "primary_col":"#2a668d",
+  "secondary_col":"#184e71",
+  "sidebar_col":"#191919"
+}

dcc/GF/gf_dca_config.json

@@ -0,0 +1,17 @@
+{
+  "project_name":"Gray Foundation",
+  "synapse_asset_view":"syn28142805",
+  "data_model_url":"https://raw.githubusercontent.com/gf-dcc/data-model/v1.1.0-rc/GF.jsonld",
+  "template_menu_config_file":"dcc/GF/dca-template-config.json",
+  "manifest_output_format":"google_sheet",
+  "manifest_use_annotations":false,
+  "validate_restrict_rules":true,
+  "submit_use_schema_labels":true,
+  "submit_table_manipulation":"replace",
+  "submit_manifest_record_type":"table_and_file",
+  "submit_hide_blanks":false,
+  "use_compliance_dashboard":false,
+  "primary_col":"#2a668d",
+  "secondary_col":"#184e71",
+  "sidebar_col":"#191919"
+}

dcc/HTAN/htan_dca_config.json

@@ -0,0 +1,19 @@
+{
+  "project_name":"HTAN All Projects",
+  "synapse_asset_view":"syn20446927",
+  "data_model_url":"https://raw.githubusercontent.com/ncihtan/data-models/main/HTAN.model.jsonld",
+  "template_menu_config_file":"dcc/HTAN/dca-template-config.json",
+  "logo_location":"dcc/HTAN/HTAN_text_logo.png",
+  "logo_link":"https://humantumoratlas.org",
+  "manifest_output_format":"google_sheet",
+  "manifest_use_annotations":false,
+  "validate_restrict_rules":false,
+  "submit_use_schema_labels":true,
+  "submit_table_manipulation":"replace",
+  "submit_manifest_record_type":"file_only",
+  "submit_hide_blanks":false,
+  "use_compliance_dashboard":false,
+  "primary_col":"#605ca8",
+  "secondary_col":"#5F008C",
+  "sidebar_col":"#191919"
+}

dcc/MC2/mc2_dca_config.json

@@ -0,0 +1,19 @@
+{
+  "project_name":"Cancer Complexity Knowledge Portal - Database",
+  "synapse_asset_view":"syn27210848",
+  "data_model_url":"https://raw.githubusercontent.com/mc2-center/data-models/main/mc2.model.jsonld",
+  "template_menu_config_file":"dcc/MC2/dca-template-config.json",
+  "logo_location":"dcc/MC2/cckp_logo.png",
+  "logo_link":"https://cancercomplexity.synapse.org",
+  "manifest_output_format":"excel",
+  "manifest_use_annotations":false,
+  "validate_restrict_rules":false,
+  "submit_use_schema_labels":false,
+  "submit_table_manipulation":"upsert",
+  "submit_manifest_record_type":"table_and_file",
+  "submit_hide_blanks":false,
+  "use_compliance_dashboard":false,
+  "primary_col":"#407BA0",
+  "secondary_col":"#5BB0B5",
+  "sidebar_col":"#191919"
+}

dcc/NF-OSI/nf-osi_dca_config.json

@@ -0,0 +1,17 @@
+{
+  "project_name":"NF-OSI",
+  "synapse_asset_view":"syn16858331",
+  "data_model_url":"https://raw.githubusercontent.com/nf-osi/nf-metadata-dictionary/v8.0.1/NF.jsonld",
+  "template_menu_config_file":"dcc/NF-OSI/dca-template-config.json",
+  "manifest_output_format":"google_sheet",
+  "manifest_use_annotations":true,
+  "validate_restrict_rules":false,
+  "submit_use_schema_labels":true,
+  "submit_table_manipulation":"replace",
+  "submit_manifest_record_type":"table_and_file",
+  "submit_hide_blanks":true,
+  "use_compliance_dashboard":false,
+  "primary_col":"#2a668d",
+  "secondary_col":"#184e71",
+  "sidebar_col":"#191919"
+}

dcc/VEOIBD/veoibd_dca_config.json

@@ -0,0 +1,19 @@
+{
+  "project_name":"VEOIBD",
+  "synapse_asset_view":"syn51397378",
+  "data_model_url":"https://raw.githubusercontent.com/VEOIBD/data_models/main/veoibd.data.model.jsonld",
+  "template_menu_config_file":"dcc/VEOIBD/dca-template-config.json",
+  "logo_location":"dcc/VEOIBD/VEOIBD Logo.png",
+  "logo_link":"https://veoibd.org",
+  "manifest_output_format":"excel",
+  "manifest_use_annotations":false,
+  "validate_restrict_rules":true,
+  "submit_use_schema_labels":true,
+  "submit_table_manipulation":"upsert",
+  "submit_manifest_record_type":"file_only",
+  "submit_hide_blanks":false,
+  "use_compliance_dashboard":false,
+  "primary_col":"#2a668d",
+  "secondary_col":"#184e72",
+  "sidebar_col":"#191920"
+}

dcc/demo/dca_demo_dca_config.json

@@ -0,0 +1,21 @@
+{
+  "project_name":"DCA Demo",
+  "synapse_asset_view":"syn33715412",
+  "data_model_url":"https://raw.githubusercontent.com/Sage-Bionetworks/data-models/main/example.model.jsonld",
+  "data_model_info": "",
+  "template_menu_config_file":"dcc/demo/dca-template-config.json",
+  "logo_location":"dcc/demo/Logo_Sage_Logomark.png",
+  "logo_link":"https://synapse.org",
+  "dcc_help_link": "",
+  "manifest_output_format":"google_sheet",
+  "manifest_use_annotations":false,
+  "validate_restrict_rules":false,
+  "submit_use_schema_labels":true,
+  "submit_table_manipulation":"replace",
+  "submit_manifest_record_type":"table_and_file",
+  "submit_hide_blanks":false,
+  "use_compliance_dashboard":false,
+  "primary_col":"#2a668d",
+  "secondary_col":"#184e71",
+  "sidebar_col":"#191919"
+}

dcc/demo_upsert/demo_upsert_dca_config.json

@@ -0,0 +1,17 @@
+{
+  "project_name":"DCA Demo Upsert",
+  "synapse_asset_view":"syn51489635",
+  "data_model_url":"https://raw.githubusercontent.com/Sage-Bionetworks/schematic/develop/tests/data/example.model.jsonld",
+  "template_menu_config_file":"dcc/demo_upsert/dca-template-config.json",
+  "manifest_output_format":"excel",
+  "manifest_use_annotations":false,
+  "validate_restrict_rules":false,
+  "submit_use_schema_labels":false,
+  "submit_table_manipulation":"upsert",
+  "submit_manifest_record_type":"table_and_file",
+  "submit_hide_blanks":false,
+  "use_compliance_dashboard":false,
+  "primary_col":"#2a668d",
+  "secondary_col":"#184e71",
+  "sidebar_col":"#191919"
+}