Fix: Resolve Issue With Name Node Corruption In New york Dataset (#459)

* fix: reference to container registry * Fix: Resolve issue that cluster would not create as intended. * fix: Change name of cluster in order to resolve potential corrupted node issues with creating new-york cluster object. * fix: Checking in dag file
GoogleCloudPlatform · Aug 25, 2022 · 59e3aed · 59e3aed
1 parent eb409c4
commit 59e3aed
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 12 deletions.
diff --git a/datasets/new_york/pipelines/new_york/new_york_dag.py b/datasets/new_york/pipelines/new_york/new_york_dag.py
@@ -36,9 +36,10 @@
         project_id="{{ var.value.gcp_project }}",
         location="us-central1-c",
         body={
-            "name": "new-york",
+            "name": "pubds-new-york",
             "initial_node_count": 4,
             "network": "{{ var.value.vpc_network }}",
+            "ip_allocation_policy": {"cluster_ipv4_cidr_block": "/26"},
             "node_config": {
                 "machine_type": "e2-standard-16",
                 "oauth_scopes": [
@@ -55,7 +56,7 @@
         name="311_service_requests",
         project_id="{{ var.value.gcp_project }}",
         location="us-central1-c",
-        cluster_name="new-york",
+        cluster_name="pubds-new-york",
         namespace="default",
         image_pull_policy="Always",
         image="{{ var.json.new_york.container_registry.run_csv_transform_kub_ny_311_service_requests }}",
@@ -86,7 +87,7 @@
         name="citibike_stations",
         project_id="{{ var.value.gcp_project }}",
         location="us-central1-c",
-        cluster_name="new-york",
+        cluster_name="pubds-new-york",
         namespace="default",
         image_pull_policy="Always",
         image="{{ var.json.new_york.container_registry.run_csv_transform_kub_citibike_stations }}",
@@ -121,7 +122,7 @@
         name="nypd_mv_collisions",
         project_id="{{ var.value.gcp_project }}",
         location="us-central1-c",
-        cluster_name="new-york",
+        cluster_name="pubds-new-york",
         namespace="default",
         image_pull_policy="Always",
         image="{{ var.json.new_york.container_registry.run_csv_transform_kub_nypd_mv_collisions }}",
@@ -155,7 +156,7 @@
         name="tree_census_1995",
         project_id="{{ var.value.gcp_project }}",
         location="us-central1-c",
-        cluster_name="new-york",
+        cluster_name="pubds-new-york",
         namespace="default",
         image_pull_policy="Always",
         image="{{ var.json.new_york.container_registry.run_csv_transform_kub_tree_census_1995 }}",
@@ -181,7 +182,7 @@
         task_id="delete_cluster",
         project_id="{{ var.value.gcp_project }}",
         location="us-central1-c",
-        name="new-york",
+        name="pubds-new-york",
     )
 
     (

diff --git a/datasets/new_york/pipelines/new_york/pipeline.yaml b/datasets/new_york/pipelines/new_york/pipeline.yaml
@@ -39,9 +39,11 @@ dag:
         project_id: "{{ var.value.gcp_project }}"
         location: "us-central1-c"
         body:
-          name: new-york
+          name: pubds-new-york
           initial_node_count: 4
           network: "{{ var.value.vpc_network }}"
+          ip_allocation_policy:
+            cluster_ipv4_cidr_block: "/26"
           node_config:
             machine_type: e2-standard-16
             oauth_scopes:
@@ -55,7 +57,7 @@ dag:
         name: "311_service_requests"
         project_id: "{{ var.value.gcp_project }}"
         location: "us-central1-c"
-        cluster_name: new-york
+        cluster_name: pubds-new-york
         namespace: "default"
         image_pull_policy: "Always"
         image: "{{ var.json.new_york.container_registry.run_csv_transform_kub_ny_311_service_requests }}"
@@ -225,7 +227,7 @@ dag:
         name: "citibike_stations"
         project_id: "{{ var.value.gcp_project }}"
         location: "us-central1-c"
-        cluster_name: new-york
+        cluster_name: pubds-new-york
         namespace: "default"
         image_pull_policy: "Always"
         image: "{{ var.json.new_york.container_registry.run_csv_transform_kub_citibike_stations }}"
@@ -366,7 +368,7 @@ dag:
         name: "nypd_mv_collisions"
         project_id: "{{ var.value.gcp_project }}"
         location: "us-central1-c"
-        cluster_name: new-york
+        cluster_name: pubds-new-york
         namespace: "default"
         image_pull_policy: "Always"
         image: "{{ var.json.new_york.container_registry.run_csv_transform_kub_nypd_mv_collisions }}"
@@ -511,7 +513,7 @@ dag:
         name: "tree_census_1995"
         project_id: "{{ var.value.gcp_project }}"
         location: "us-central1-c"
-        cluster_name: new-york
+        cluster_name: pubds-new-york
         namespace: "default"
         image_pull_policy: "Always"
         image: "{{ var.json.new_york.container_registry.run_csv_transform_kub_tree_census_1995 }}"
@@ -600,7 +602,7 @@ dag:
         task_id: "delete_cluster"
         project_id: "{{ var.value.gcp_project }}"
         location: "us-central1-c"
-        name: new-york
+        name: pubds-new-york
 
   graph_paths:
     - "create_cluster >> [transform_csv_nypd_mv_collisions, transform_csv_ny_citibike_stations, transform_csv_ny_tree_census_1995, transform_csv_ny_311_service_requests ] >> delete_cluster"