From 59e3aeda633698c97588195bc4daf87523e00be8 Mon Sep 17 00:00:00 2001
From: Nicholas Large <84149918+nlarge-google@users.noreply.github.com>
Date: Thu, 25 Aug 2022 16:05:44 -0500
Subject: [PATCH] Fix: Resolve Issue With Name Node Corruption In New york
 Dataset (#459)

* fix: reference to container registry

* Fix: Resolve issue that cluster would not create as intended.

* fix: Change name of cluster in order to resolve potential corrupted node issues with creating new-york cluster object.

* fix: Checking in dag file
---
 .../new_york/pipelines/new_york/new_york_dag.py    | 13 +++++++------
 datasets/new_york/pipelines/new_york/pipeline.yaml | 14 ++++++++------
 2 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/datasets/new_york/pipelines/new_york/new_york_dag.py b/datasets/new_york/pipelines/new_york/new_york_dag.py
index c210cba57..391fd5086 100644
--- a/datasets/new_york/pipelines/new_york/new_york_dag.py
+++ b/datasets/new_york/pipelines/new_york/new_york_dag.py
@@ -36,9 +36,10 @@
         project_id="{{ var.value.gcp_project }}",
         location="us-central1-c",
         body={
-            "name": "new-york",
+            "name": "pubds-new-york",
             "initial_node_count": 4,
             "network": "{{ var.value.vpc_network }}",
+            "ip_allocation_policy": {"cluster_ipv4_cidr_block": "/26"},
             "node_config": {
                 "machine_type": "e2-standard-16",
                 "oauth_scopes": [
@@ -55,7 +56,7 @@
         name="311_service_requests",
         project_id="{{ var.value.gcp_project }}",
         location="us-central1-c",
-        cluster_name="new-york",
+        cluster_name="pubds-new-york",
         namespace="default",
         image_pull_policy="Always",
         image="{{ var.json.new_york.container_registry.run_csv_transform_kub_ny_311_service_requests }}",
@@ -86,7 +87,7 @@
         name="citibike_stations",
         project_id="{{ var.value.gcp_project }}",
         location="us-central1-c",
-        cluster_name="new-york",
+        cluster_name="pubds-new-york",
         namespace="default",
         image_pull_policy="Always",
         image="{{ var.json.new_york.container_registry.run_csv_transform_kub_citibike_stations }}",
@@ -121,7 +122,7 @@
         name="nypd_mv_collisions",
         project_id="{{ var.value.gcp_project }}",
         location="us-central1-c",
-        cluster_name="new-york",
+        cluster_name="pubds-new-york",
         namespace="default",
         image_pull_policy="Always",
         image="{{ var.json.new_york.container_registry.run_csv_transform_kub_nypd_mv_collisions }}",
@@ -155,7 +156,7 @@
         name="tree_census_1995",
         project_id="{{ var.value.gcp_project }}",
         location="us-central1-c",
-        cluster_name="new-york",
+        cluster_name="pubds-new-york",
         namespace="default",
         image_pull_policy="Always",
         image="{{ var.json.new_york.container_registry.run_csv_transform_kub_tree_census_1995 }}",
@@ -181,7 +182,7 @@
         task_id="delete_cluster",
         project_id="{{ var.value.gcp_project }}",
         location="us-central1-c",
-        name="new-york",
+        name="pubds-new-york",
     )
 
     (
diff --git a/datasets/new_york/pipelines/new_york/pipeline.yaml b/datasets/new_york/pipelines/new_york/pipeline.yaml
index 56f935677..b17ef3395 100644
--- a/datasets/new_york/pipelines/new_york/pipeline.yaml
+++ b/datasets/new_york/pipelines/new_york/pipeline.yaml
@@ -39,9 +39,11 @@ dag:
         project_id: "{{ var.value.gcp_project }}"
         location: "us-central1-c"
         body:
-          name: new-york
+          name: pubds-new-york
           initial_node_count: 4
           network: "{{ var.value.vpc_network }}"
+          ip_allocation_policy:
+            cluster_ipv4_cidr_block: "/26"
           node_config:
             machine_type: e2-standard-16
             oauth_scopes:
@@ -55,7 +57,7 @@ dag:
         name: "311_service_requests"
         project_id: "{{ var.value.gcp_project }}"
         location: "us-central1-c"
-        cluster_name: new-york
+        cluster_name: pubds-new-york
         namespace: "default"
         image_pull_policy: "Always"
         image: "{{ var.json.new_york.container_registry.run_csv_transform_kub_ny_311_service_requests }}"
@@ -225,7 +227,7 @@ dag:
         name: "citibike_stations"
         project_id: "{{ var.value.gcp_project }}"
         location: "us-central1-c"
-        cluster_name: new-york
+        cluster_name: pubds-new-york
         namespace: "default"
         image_pull_policy: "Always"
         image: "{{ var.json.new_york.container_registry.run_csv_transform_kub_citibike_stations }}"
@@ -366,7 +368,7 @@ dag:
         name: "nypd_mv_collisions"
         project_id: "{{ var.value.gcp_project }}"
         location: "us-central1-c"
-        cluster_name: new-york
+        cluster_name: pubds-new-york
         namespace: "default"
         image_pull_policy: "Always"
         image: "{{ var.json.new_york.container_registry.run_csv_transform_kub_nypd_mv_collisions }}"
@@ -511,7 +513,7 @@ dag:
         name: "tree_census_1995"
         project_id: "{{ var.value.gcp_project }}"
         location: "us-central1-c"
-        cluster_name: new-york
+        cluster_name: pubds-new-york
         namespace: "default"
         image_pull_policy: "Always"
         image: "{{ var.json.new_york.container_registry.run_csv_transform_kub_tree_census_1995 }}"
@@ -600,7 +602,7 @@ dag:
         task_id: "delete_cluster"
         project_id: "{{ var.value.gcp_project }}"
         location: "us-central1-c"
-        name: new-york
+        name: pubds-new-york
 
   graph_paths:
     - "create_cluster >> [transform_csv_nypd_mv_collisions, transform_csv_ny_citibike_stations, transform_csv_ny_tree_census_1995, transform_csv_ny_311_service_requests ] >> delete_cluster"