Fix: Renamed dataset from sunroof to sunroof_solar

datasets/sunroof/_terraform/solar_potential_by_census_tract_pipeline.tf → datasets/sunroof_solar/_terraform/solar_potential_by_censustract_pipeline.tf

@@ -15,25 +15,25 @@
  */
 
 
-resource "google_bigquery_table" "solar_potential_by_census_tract" {
+resource "google_bigquery_table" "sunroof_solar_solar_potential_by_censustract" {
   project    = var.project_id
-  dataset_id = "sunroof"
-  table_id   = "solar_potential_by_census_tract"
+  dataset_id = "sunroof_solar"
+  table_id   = "solar_potential_by_censustract"
 
   description = "Sunroof Solar Potential By Census Tract"
 
 
 
 
   depends_on = [
-    google_bigquery_dataset.sunroof
+    google_bigquery_dataset.sunroof_solar
   ]
 }
 
-output "bigquery_table-solar_potential_by_census_tract-table_id" {
-  value = google_bigquery_table.solar_potential_by_census_tract.table_id
+output "bigquery_table-sunroof_solar_solar_potential_by_censustract-table_id" {
+  value = google_bigquery_table.sunroof_solar_solar_potential_by_censustract.table_id
 }
 
-output "bigquery_table-solar_potential_by_census_tract-id" {
-  value = google_bigquery_table.solar_potential_by_census_tract.id
+output "bigquery_table-sunroof_solar_solar_potential_by_censustract-id" {
+  value = google_bigquery_table.sunroof_solar_solar_potential_by_censustract.id
 }

datasets/sunroof/_terraform/solar_potential_by_postal_code_pipeline.tf → datasets/sunroof_solar/_terraform/solar_potential_by_postal_code_pipeline.tf

@@ -15,9 +15,9 @@
  */
 
 
-resource "google_bigquery_table" "solar_potential_by_postal_code" {
+resource "google_bigquery_table" "sunroof_solar_solar_potential_by_postal_code" {
   project    = var.project_id
-  dataset_id = "sunroof"
+  dataset_id = "sunroof_solar"
   table_id   = "solar_potential_by_postal_code"
 
   description = "Sunroof Solar Potential By Postal Code"
@@ -26,14 +26,14 @@ resource "google_bigquery_table" "solar_potential_by_postal_code" {
 
 
   depends_on = [
-    google_bigquery_dataset.sunroof
+    google_bigquery_dataset.sunroof_solar
   ]
 }
 
-output "bigquery_table-solar_potential_by_postal_code-table_id" {
-  value = google_bigquery_table.solar_potential_by_postal_code.table_id
+output "bigquery_table-sunroof_solar_solar_potential_by_postal_code-table_id" {
+  value = google_bigquery_table.sunroof_solar_solar_potential_by_postal_code.table_id
 }
 
-output "bigquery_table-solar_potential_by_postal_code-id" {
-  value = google_bigquery_table.solar_potential_by_postal_code.id
+output "bigquery_table-sunroof_solar_solar_potential_by_postal_code-id" {
+  value = google_bigquery_table.sunroof_solar_solar_potential_by_postal_code.id
 }

datasets/sunroof_solar/_terraform/sunroof_solar_dataset.tf

@@ -0,0 +1,26 @@
+/**
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+resource "google_bigquery_dataset" "sunroof_solar" {
+  dataset_id  = "sunroof_solar"
+  project     = var.project_id
+  description = "Project Sunroof computes how much sunlight hits your roof in a year. Solar viability is determined using a methodology found here: https://www.google.com/get/sunroof/data-explorer/data-explorer-methodology.pdf The use of this data is subject to Google\u0027s Terms of Service. Feel free to include this data from Project Sunroof in other analyses, materials, reports, and communications with the following attribution: Source: Google Project Sunroof data explorer (August 2017)."
+}
+
+output "bigquery_dataset-sunroof_solar-dataset_id" {
+  value = google_bigquery_dataset.sunroof_solar.dataset_id
+}

datasets/sunroof/dataset.yaml → datasets/sunroof_solar/dataset.yaml

@@ -13,14 +13,15 @@
 # limitations under the License.
 
 dataset:
-  name: sunroof
-  friendly_name: sunroof
-  description: sunroof
+  name: sunroof_solar
+  friendly_name: sunroof_solar
+  description: sunroof_solar
   dataset_sources: ~
   terms_of_use: ~
 
 
 resources:
   - type: bigquery_dataset
-    dataset_id: sunroof
-    description: sunroof
+    dataset_id: sunroof_solar
+    description: |-
+      Project Sunroof computes how much sunlight hits your roof in a year. Solar viability is determined using a methodology found here: https://www.google.com/get/sunroof/data-explorer/data-explorer-methodology.pdf The use of this data is subject to Google's Terms of Service. Feel free to include this data from Project Sunroof in other analyses, materials, reports, and communications with the following attribution: Source: Google Project Sunroof data explorer (August 2017).

datasets/sunroof/solar_potential_by_census_tract/pipeline.yaml → datasets/sunroof_solar/solar_potential_by_censustract/pipeline.yaml

@@ -16,13 +16,13 @@
 resources:
 
   - type: bigquery_table
-    table_id: "solar_potential_by_census_tract"
+    table_id: "solar_potential_by_censustract"
     description: "Sunroof Solar Potential By Census Tract"
 
 dag:
   airflow_version: 2
   initialize:
-    dag_id: solar_potential_by_census_tract
+    dag_id: solar_potential_by_censustract
     default_args:
       owner: "Google"
       depends_on_past: False
@@ -40,7 +40,7 @@ dag:
       args:
 
         task_id: "transform_csv"
-        name: "solar_potential_by_census_tract"
+        name: "solar_potential_by_censustract"
         namespace: "default"
         affinity:
           nodeAffinity:
@@ -52,27 +52,27 @@ dag:
                       values:
                         - "pool-e2-standard-4"
         image_pull_policy: "Always"
-        image: "{{ var.json.sunroof.container_registry.run_csv_transform_kub }}"
+        image: "{{ var.json.sunroof_solar.container_registry.run_csv_transform_kub }}"
         env_vars:
           SOURCE_URL: "gs://project-sunroof/csv/latest/project-sunroof-census_tract.csv"
           SOURCE_FILE: "files/data.csv"
           TARGET_FILE: "files/data_output.csv"
           CHUNKSIZE: "750000"
           TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}"
-          TARGET_GCS_PATH: "data/sunroof/solar_potential_by_census_tract/data_output.csv"
+          TARGET_GCS_PATH: "data/sunroof/solar_potential_by_censustract/data_output.csv"
         resources:
           limit_memory: "8G"
           limit_cpu: "3"
 
     - operator: "GoogleCloudStorageToBigQueryOperator"
-      description: "Task to load CSV data to a BigQuery table"
+      description: "Load CSV data to a BigQuery table"
 
       args:
         task_id: "load_to_bq"
         bucket: "{{ var.value.composer_bucket }}"
-        source_objects: ["data/sunroof/solar_potential_by_census_tract/data_output.csv"]
+        source_objects: ["data/sunroof/solar_potential_by_censustract/data_output.csv"]
         source_format: "CSV"
-        destination_project_dataset_table: "sunroof.solar_potential_by_census_tract"
+        destination_project_dataset_table: "sunroof.solar_potential_by_censustract"
         skip_leading_rows: 1
         allow_quoted_newlines: True
         write_disposition: "WRITE_TRUNCATE"

datasets/sunroof/solar_potential_by_census_tract/solar_potential_by_census_tract_dag.py → datasets/sunroof_solar/solar_potential_by_censustract/solar_potential_by_censustract_dag.py

@@ -25,7 +25,7 @@
 
 
 with DAG(
-    dag_id="sunroof.solar_potential_by_census_tract",
+    dag_id="sunroof_solar.solar_potential_by_censustract",
     default_args=default_args,
     max_active_runs=1,
     schedule_interval="@daily",
@@ -36,7 +36,7 @@
     # Run CSV transform within kubernetes pod
     transform_csv = kubernetes_pod.KubernetesPodOperator(
         task_id="transform_csv",
-        name="solar_potential_by_census_tract",
+        name="solar_potential_by_censustract",
         namespace="default",
         affinity={
             "nodeAffinity": {
@@ -56,14 +56,14 @@
             }
         },
         image_pull_policy="Always",
-        image="{{ var.json.sunroof.container_registry.run_csv_transform_kub }}",
+        image="{{ var.json.sunroof_solar.container_registry.run_csv_transform_kub }}",
         env_vars={
             "SOURCE_URL": "gs://project-sunroof/csv/latest/project-sunroof-census_tract.csv",
             "SOURCE_FILE": "files/data.csv",
             "TARGET_FILE": "files/data_output.csv",
             "CHUNKSIZE": "750000",
             "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}",
-            "TARGET_GCS_PATH": "data/sunroof/solar_potential_by_census_tract/data_output.csv",
+            "TARGET_GCS_PATH": "data/sunroof/solar_potential_by_censustract/data_output.csv",
         },
         resources={"limit_memory": "8G", "limit_cpu": "3"},
     )
@@ -72,9 +72,9 @@
     load_to_bq = gcs_to_bigquery.GCSToBigQueryOperator(
         task_id="load_to_bq",
         bucket="{{ var.value.composer_bucket }}",
-        source_objects=["data/sunroof/solar_potential_by_census_tract/data_output.csv"],
+        source_objects=["data/sunroof/solar_potential_by_censustract/data_output.csv"],
         source_format="CSV",
-        destination_project_dataset_table="sunroof.solar_potential_by_census_tract",
+        destination_project_dataset_table="sunroof.solar_potential_by_censustract",
         skip_leading_rows=1,
         allow_quoted_newlines=True,
         write_disposition="WRITE_TRUNCATE",

datasets/sunroof/solar_potential_by_postal_code/pipeline.yaml → datasets/sunroof_solar/solar_potential_by_postal_code/pipeline.yaml

@@ -52,7 +52,7 @@ dag:
                       values:
                         - "pool-e2-standard-4"
         image_pull_policy: "Always"
-        image: "{{ var.json.sunroof.container_registry.run_csv_transform_kub }}"
+        image: "{{ var.json.sunroof_solar.container_registry.run_csv_transform_kub }}"
         env_vars:
           SOURCE_URL: "gs://project-sunroof/csv/latest/project-sunroof-postal_code.csv"
           SOURCE_FILE: "files/data.csv"

datasets/sunroof/solar_potential_by_postal_code/solar_potential_by_postal_code_dag.py → datasets/sunroof_solar/solar_potential_by_postal_code/solar_potential_by_postal_code_dag.py

@@ -25,7 +25,7 @@
 
 
 with DAG(
-    dag_id="sunroof.solar_potential_by_postal_code",
+    dag_id="sunroof_solar.solar_potential_by_postal_code",
     default_args=default_args,
     max_active_runs=1,
     schedule_interval="@daily",
@@ -56,7 +56,7 @@
             }
         },
         image_pull_policy="Always",
-        image="{{ var.json.sunroof.container_registry.run_csv_transform_kub }}",
+        image="{{ var.json.sunroof_solar.container_registry.run_csv_transform_kub }}",
         env_vars={
             "SOURCE_URL": "gs://project-sunroof/csv/latest/project-sunroof-postal_code.csv",
             "SOURCE_FILE": "files/data.csv",