feat: Onboard IDC v8 dataset (#319)

* cleanup: delete BQ view queries * feat: adds IDC dataset to v8 * feat: fix black linter on GitHub checks * cleanup: rollback python linter revisions * fix: upgrade black due to `click` dependency error * fix: upgrade black due to `click` dependency error
GoogleCloudPlatform · Mar 31, 2022 · 0f112e0 · 0f112e0
1 parent cac6f18
commit 0f112e0
Show file tree

Hide file tree

Showing 70 changed files with 105 additions and 4,887 deletions.
diff --git a/datasets/idc/infra/idc_dataset.tf b/datasets/idc/infra/idc_dataset.tf
@@ -101,6 +101,16 @@ output "bigquery_dataset-idc_v7-dataset_id" {
   value = google_bigquery_dataset.idc_v7.dataset_id
 }
 
+resource "google_bigquery_dataset" "idc_v8" {
+  dataset_id  = "idc_v8"
+  project     = var.project_id
+  description = "Imaging Data Commons (IDC) - The Cancer Imaging Archive (TCIA) v8 data"
+}
+
+output "bigquery_dataset-idc_v8-dataset_id" {
+  value = google_bigquery_dataset.idc_v8.dataset_id
+}
+
 resource "google_bigquery_dataset" "idc_current" {
   dataset_id  = "idc_current"
   project     = var.project_id

diff --git a/...ets/idc/pipelines/_images/generate_bq_views/queries/current/analysis_results_metadata.sql b/...ets/idc/pipelines/_images/generate_bq_views/queries/current/analysis_results_metadata.sql
diff --git a/datasets/idc/pipelines/_images/generate_bq_views/queries/current/auxiliary_metadata.sql b/datasets/idc/pipelines/_images/generate_bq_views/queries/current/auxiliary_metadata.sql
diff --git a/datasets/idc/pipelines/_images/generate_bq_views/queries/current/dicom_all.sql b/datasets/idc/pipelines/_images/generate_bq_views/queries/current/dicom_all.sql
diff --git a/datasets/idc/pipelines/_images/generate_bq_views/queries/current/dicom_derived_all.sql b/datasets/idc/pipelines/_images/generate_bq_views/queries/current/dicom_derived_all.sql
diff --git a/datasets/idc/pipelines/_images/generate_bq_views/queries/current/dicom_metadata.sql b/datasets/idc/pipelines/_images/generate_bq_views/queries/current/dicom_metadata.sql
diff --git a/datasets/idc/pipelines/_images/generate_bq_views/queries/current/dicom_metadata_curated.sql b/datasets/idc/pipelines/_images/generate_bq_views/queries/current/dicom_metadata_curated.sql
diff --git a/datasets/idc/pipelines/_images/generate_bq_views/queries/current/dicom_pivot_v7.sql b/datasets/idc/pipelines/_images/generate_bq_views/queries/current/dicom_pivot_v7.sql
diff --git a/datasets/idc/pipelines/_images/generate_bq_views/queries/current/measurement_groups.sql b/datasets/idc/pipelines/_images/generate_bq_views/queries/current/measurement_groups.sql
diff --git a/datasets/idc/pipelines/_images/generate_bq_views/queries/current/nlst_canc.sql b/datasets/idc/pipelines/_images/generate_bq_views/queries/current/nlst_canc.sql
diff --git a/datasets/idc/pipelines/_images/generate_bq_views/queries/current/nlst_ctab.sql b/datasets/idc/pipelines/_images/generate_bq_views/queries/current/nlst_ctab.sql
diff --git a/datasets/idc/pipelines/_images/generate_bq_views/queries/current/nlst_ctabc.sql b/datasets/idc/pipelines/_images/generate_bq_views/queries/current/nlst_ctabc.sql
diff --git a/datasets/idc/pipelines/_images/generate_bq_views/queries/current/nlst_prsn.sql b/datasets/idc/pipelines/_images/generate_bq_views/queries/current/nlst_prsn.sql
diff --git a/datasets/idc/pipelines/_images/generate_bq_views/queries/current/nlst_screen.sql b/datasets/idc/pipelines/_images/generate_bq_views/queries/current/nlst_screen.sql
diff --git a/...idc/pipelines/_images/generate_bq_views/queries/current/original_collections_metadata.sql b/...idc/pipelines/_images/generate_bq_views/queries/current/original_collections_metadata.sql
diff --git a/...sets/idc/pipelines/_images/generate_bq_views/queries/current/qualitative_measurements.sql b/...sets/idc/pipelines/_images/generate_bq_views/queries/current/qualitative_measurements.sql
diff --git a/...ets/idc/pipelines/_images/generate_bq_views/queries/current/quantitative_measurements.sql b/...ets/idc/pipelines/_images/generate_bq_views/queries/current/quantitative_measurements.sql
diff --git a/datasets/idc/pipelines/_images/generate_bq_views/queries/current/segmentations.sql b/datasets/idc/pipelines/_images/generate_bq_views/queries/current/segmentations.sql
diff --git a/datasets/idc/pipelines/_images/generate_bq_views/queries/current/tcga_biospecimen_rel9.sql b/datasets/idc/pipelines/_images/generate_bq_views/queries/current/tcga_biospecimen_rel9.sql
diff --git a/datasets/idc/pipelines/_images/generate_bq_views/queries/current/tcga_clinical_rel9.sql b/datasets/idc/pipelines/_images/generate_bq_views/queries/current/tcga_clinical_rel9.sql
diff --git a/datasets/idc/pipelines/_images/generate_bq_views/queries/current/version_metadata.sql b/datasets/idc/pipelines/_images/generate_bq_views/queries/current/version_metadata.sql
diff --git a/datasets/idc/pipelines/_images/generate_bq_views/queries/v1/dicom_all.sql b/datasets/idc/pipelines/_images/generate_bq_views/queries/v1/dicom_all.sql
diff --git a/datasets/idc/pipelines/_images/generate_bq_views/queries/v1/dicom_pivot_v1.sql b/datasets/idc/pipelines/_images/generate_bq_views/queries/v1/dicom_pivot_v1.sql
diff --git a/datasets/idc/pipelines/_images/generate_bq_views/queries/v1/measurement_groups.sql b/datasets/idc/pipelines/_images/generate_bq_views/queries/v1/measurement_groups.sql