Feat: Add Municipal Calendar to San Francisco Dataset (#480)

* feat: san francisco pipelines integrated into one dag. * Feat: Adding municipal calendar pipeline. Not production Ready. * fix: Continuing SF municipal calendar changes. * feat: Added sf_calendar. Is tested successfully. * fix: Removed cluster create/delete. * feat: Added muni_routes * fix: Resolved reference to schema file. * feat: Added SF Municipal Shapes * feat: Added SF Municipal Stops * Feat: Added SFPD Incidents. * fix: Misc Fixes. * fix: Integration Testing. * fix: Flake etc. * fix: Misc. * fix: Misc.
GoogleCloudPlatform · Sep 28, 2022 · a21c2ef · a21c2ef
1 parent 2da413e
commit a21c2ef
Show file tree

Hide file tree

Showing 22 changed files with 4,078 additions and 0 deletions.
diff --git a/datasets/san_francisco/dataset.yaml b/datasets/san_francisco/dataset.yaml
@@ -0,0 +1,24 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+dataset:
+  name: san_francisco
+  friendly_name: ~
+  description: ~
+  dataset_sources: ~
+  terms_of_use: ~
+resources:
+  - type: bigquery_dataset
+    dataset_id: san_francisco
+    description: "san francisco dataset"
diff --git a/datasets/san_francisco/infra/provider.tf b/datasets/san_francisco/infra/provider.tf
@@ -0,0 +1,28 @@
+/**
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+provider "google" {
+  project                     = var.project_id
+  impersonate_service_account = var.impersonating_acct
+  region                      = var.region
+}
+
+data "google_client_openid_userinfo" "me" {}
+
+output "impersonating-account" {
+  value = data.google_client_openid_userinfo.me.email
+}
diff --git a/datasets/san_francisco/infra/san_francisco_dataset.tf b/datasets/san_francisco/infra/san_francisco_dataset.tf
@@ -0,0 +1,26 @@
+/**
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+resource "google_bigquery_dataset" "san_francisco_311_service_requests" {
+  dataset_id  = "san_francisco_311_service_requests"
+  project     = var.project_id
+  description = "san_francisco_311_service_requests"
+}
+
+output "bigquery_dataset-san_francisco_311_service_requests-dataset_id" {
+  value = google_bigquery_dataset.san_francisco_311_service_requests.dataset_id
+}
diff --git a/datasets/san_francisco/infra/san_francisco_pipeline.tf b/datasets/san_francisco/infra/san_francisco_pipeline.tf
@@ -0,0 +1,34 @@
+/**
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+resource "google_bigquery_table" "bqt_san_francisco_311_service_requests" {
+  project     = var.project_id
+  dataset_id  = "san_francisco"
+  table_id    = "311_service_requests"
+  description = "san_francisco_etl_process"
+  depends_on = [
+    google_bigquery_dataset.san_francisco
+  ]
+}
+
+output "bigquery_table-bqt_san_francisco_311_service_requests-table_id" {
+  value = google_bigquery_table.bqt_san_francisco_311_service_requests.table_id
+}
+
+output "bigquery_table-bqt_san_francisco_311_service_requests-id" {
+  value = google_bigquery_table.bqt_san_francisco_311_service_requests.id
+}
diff --git a/datasets/san_francisco/infra/variables.tf b/datasets/san_francisco/infra/variables.tf
@@ -0,0 +1,23 @@
+/**
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+variable "project_id" {}
+variable "bucket_name_prefix" {}
+variable "impersonating_acct" {}
+variable "region" {}
+variable "env" {}
+
diff --git a/datasets/san_francisco/pipelines/_images/run_csv_transform_kub/Dockerfile b/datasets/san_francisco/pipelines/_images/run_csv_transform_kub/Dockerfile
@@ -0,0 +1,21 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM python:3.8
+ENV PYTHONUNBUFFERED True
+COPY requirements.txt ./
+RUN python3 -m pip install --no-cache-dir -r requirements.txt
+WORKDIR /custom
+COPY ./csv_transform.py .
+CMD ["python3", "csv_transform.py"]