From 241ec5bc13c20a5ab9f9a8284cd77484f365352b Mon Sep 17 00:00:00 2001 From: Ram Srivatsa Kannan Date: Thu, 2 Nov 2023 14:46:33 -0700 Subject: [PATCH 1/4] Missing the instance types that were part of our cass fleet --- .../profiles/pricing/aws/3yr-reserved.json | 5 ++ .../hardware/profiles/shapes/aws.json | 67 +++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/service_capacity_modeling/hardware/profiles/pricing/aws/3yr-reserved.json b/service_capacity_modeling/hardware/profiles/pricing/aws/3yr-reserved.json index 318b8d7..d9eb4c6 100644 --- a/service_capacity_modeling/hardware/profiles/pricing/aws/3yr-reserved.json +++ b/service_capacity_modeling/hardware/profiles/pricing/aws/3yr-reserved.json @@ -29,19 +29,23 @@ "i3.xlarge": {"annual_cost": 1155.9}, "i3.2xlarge": {"annual_cost": 2312}, "i3.4xlarge": {"annual_cost": 4624.3}, + "i3en.large": {"annual_cost": 749.8}, "i3en.xlarge": {"annual_cost": 1499.6}, "i3en.2xlarge": {"annual_cost": 2999}, "i3en.3xlarge": {"annual_cost": 4498}, + "i3en.6xlarge": {"annual_cost": 8996}, "i4i.large": {"annual_cost": 605.33}, "i4i.xlarge": {"annual_cost": 1210.66}, "i4i.2xlarge": {"annual_cost": 2421.00}, "i4i.4xlarge": {"annual_cost": 4841.67}, "i4i.8xlarge": {"annual_cost": 9683.66}, + "i4i.16xlarge": {"annual_cost": 19366.68}, "m5d.large": {"annual_cost": 372.2}, "m5d.xlarge": {"annual_cost": 744.3}, "m5d.2xlarge": {"annual_cost": 1488.6}, "m5d.4xlarge": {"annual_cost": 2977.7}, "m5d.8xlarge": {"annual_cost": 5955}, + "m5d.12xlarge": {"annual_cost": 8933.1}, "m6id.large": {"annual_cost": 390.67}, "m6id.xlarge": {"annual_cost": 781.66}, "m6id.2xlarge": {"annual_cost": 1563.32}, @@ -56,6 +60,7 @@ "r5d.2xlarge": {"annual_cost": 1897}, "r5d.4xlarge": {"annual_cost": 3794}, "r5d.8xlarge": {"annual_cost": 7588}, + "r5d.8xlarge": {"annual_cost": 15176}, "r5dn.large": {"annual_cost": 518.3}, "r5dn.xlarge": {"annual_cost": 1037}, "r5dn.2xlarge": {"annual_cost": 2073.6}, diff --git a/service_capacity_modeling/hardware/profiles/shapes/aws.json b/service_capacity_modeling/hardware/profiles/shapes/aws.json index 3959a4e..271d9ab 100644 --- a/service_capacity_modeling/hardware/profiles/shapes/aws.json +++ b/service_capacity_modeling/hardware/profiles/shapes/aws.json @@ -48,6 +48,14 @@ "net_mbps": 10000, "drive": null }, + "m5.12xlarge": { + "name": "m5.12xlarge", + "cpu": 48, + "cpu_ghz": 3.1, + "ram_gib": 192, + "net_mbps": 10000, + "drive": null + }, "m6i.large": { "name": "m6i.large", "cpu": 2, @@ -332,7 +340,41 @@ "block_size_kib": 4, "single_tenant": true } }, + "i4i.16xlarge": { + "name": "i4i.16xlarge", + "cpu": 64, + "cpu_ghz": 3.5, + "ram_gib": 495.82, + "net_mbps": 35000, + "drive": { + "name": "ephem", "size_gib": 3492, + "read_io_latency_ms": { + "minimum_value":0.05, + "low":0.10, "mid":0.125, "high":0.17, + "maximum_value":2.05, "confidence":0.9 + }, + "read_io_per_s": 16000000, "write_io_per_s": 880000, + "block_size_kib": 4, "single_tenant": true + } + }, + "i3en.large": { + "name": "i3en.large", + "cpu": 2, + "cpu_ghz": 3.1, + "ram_gib": 15.8, + "net_mbps": 4000, + "drive": { + "name": "ephem", "size_gib": 1150, + "read_io_latency_ms": { + "minimum_value":0.07, + "low":0.08, "mid":0.12, "high":0.20, + "maximum_value":2, "confidence":0.9 + }, + "read_io_per_s": 42500, "write_io_per_s": 32500, + "block_size_kib": 4, "single_tenant": false + } + }, "i3en.xlarge": { "name": "i3en.xlarge", "cpu": 4, @@ -379,6 +421,18 @@ "block_size_kib": 4, "single_tenant": true } }, + "i3en.6xlarge": { + "name": "i3en.6xlarge", + "cpu": 24, + "cpu_ghz": 3.1, + "ram_gib": 186.62, + "net_mbps": 12000, + "drive": {"name": "ephem", "size_gib": 14000, + "read_io_latency_ms": {"minimum_value":0.07, "low":0.08, "mid":0.12, "high":0.16, "maximum_value":2, "confidence":0.9}, + "read_io_per_s": 500000, "write_io_per_s": 400000, + "block_size_kib": 4, "single_tenant": true + } + }, "m5d.large": { "name": "m5d.large", "cpu": 2, @@ -650,6 +704,19 @@ "block_size_kib": 4, "single_tenant": true } }, + "r5d.16xlarge": { + "name": "r5d.16xlarge", + "cpu": 64, + "cpu_ghz": 3.1, + "ram_gib": 512, + "net_mbps": 20000, + "drive": { + "name": "ephem", "size_gib": 2200, + "read_io_latency_ms": {"minimum_value":0.07, "low":0.08, "mid":0.12, "high":0.20, "maximum_value":2, "confidence":0.9}, + "read_io_per_s": 933332, "write_io_per_s": 466666, + "block_size_kib": 4, "single_tenant": true + } + }, "r5dn.large": { "name": "r5dn.large", "cpu": 2, From 8a212cab400f66ff5f0073f5018e0d1f2242cbc0 Mon Sep 17 00:00:00 2001 From: Ram Srivatsa Kannan Date: Thu, 2 Nov 2023 14:55:19 -0700 Subject: [PATCH 2/4] Throw error gracefully when instance type definitions are not present --- service_capacity_modeling/capacity_planner.py | 24 ++++++++++++++----- .../profiles/pricing/aws/3yr-reserved.json | 2 +- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/service_capacity_modeling/capacity_planner.py b/service_capacity_modeling/capacity_planner.py index a5318ae..41eb892 100644 --- a/service_capacity_modeling/capacity_planner.py +++ b/service_capacity_modeling/capacity_planner.py @@ -189,13 +189,25 @@ def _set_instance_objects( ): if desires.current_clusters: for zonal_cluster_capacity in desires.current_clusters.zonal: - zonal_cluster_capacity.cluster_instance = hardware.instances[ - zonal_cluster_capacity.cluster_instance_name - ] + if zonal_cluster_capacity.cluster_instance_name in hardware.instances: + zonal_cluster_capacity.cluster_instance = hardware.instances[ + zonal_cluster_capacity.cluster_instance_name + ] + else: + raise ValueError( + f"Model not trained to right size cass clusters that are of " + f"instance types {zonal_cluster_capacity.cluster_instance_name}" + ) for regional_cluster_capacity in desires.current_clusters.regional: - regional_cluster_capacity.cluster_instance = hardware.instances[ - regional_cluster_capacity.cluster_instance_name - ] + if regional_cluster_capacity.cluster_instance_name in hardware.instances: + regional_cluster_capacity.cluster_instance = hardware.instances[ + regional_cluster_capacity.cluster_instance_name + ] + else: + raise ValueError( + f"Model not trained to right size cass clusters that are of " + f"instance types {regional_cluster_capacity.cluster_instance_name}" + ) def _allow_instance( diff --git a/service_capacity_modeling/hardware/profiles/pricing/aws/3yr-reserved.json b/service_capacity_modeling/hardware/profiles/pricing/aws/3yr-reserved.json index d9eb4c6..0046f05 100644 --- a/service_capacity_modeling/hardware/profiles/pricing/aws/3yr-reserved.json +++ b/service_capacity_modeling/hardware/profiles/pricing/aws/3yr-reserved.json @@ -7,6 +7,7 @@ "m5.2xlarge": {"annual_cost": 1264.6}, "m5.4xlarge": {"annual_cost": 2529.7}, "m5.8xlarge": {"annual_cost": 5059.33}, + "m5.12xlarge": {"annual_cost": 7589.1}, "m6i.large": {"annual_cost": 332.0}, "m6i.xlarge": {"annual_cost": 664.0}, "m6i.2xlarge": {"annual_cost": 1328.00}, @@ -45,7 +46,6 @@ "m5d.2xlarge": {"annual_cost": 1488.6}, "m5d.4xlarge": {"annual_cost": 2977.7}, "m5d.8xlarge": {"annual_cost": 5955}, - "m5d.12xlarge": {"annual_cost": 8933.1}, "m6id.large": {"annual_cost": 390.67}, "m6id.xlarge": {"annual_cost": 781.66}, "m6id.2xlarge": {"annual_cost": 1563.32}, From 2d5ee5c6be1f09253d3a5ba9d25729ca14b67ca7 Mon Sep 17 00:00:00 2001 From: Ram Srivatsa Kannan Date: Thu, 2 Nov 2023 16:17:26 -0700 Subject: [PATCH 3/4] Addressed Arun comments --- service_capacity_modeling/hardware/profiles/shapes/aws.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/service_capacity_modeling/hardware/profiles/shapes/aws.json b/service_capacity_modeling/hardware/profiles/shapes/aws.json index 271d9ab..59f3100 100644 --- a/service_capacity_modeling/hardware/profiles/shapes/aws.json +++ b/service_capacity_modeling/hardware/profiles/shapes/aws.json @@ -330,7 +330,7 @@ "ram_gib": 247.76, "net_mbps": 18750, "drive": { - "name": "ephem", "size_gib": 3492, + "name": "ephem", "size_gib": 6984, "read_io_latency_ms": { "minimum_value":0.05, "low":0.10, "mid":0.125, "high":0.17, @@ -347,7 +347,7 @@ "ram_gib": 495.82, "net_mbps": 35000, "drive": { - "name": "ephem", "size_gib": 3492, + "name": "ephem", "size_gib": 13968, "read_io_latency_ms": { "minimum_value":0.05, "low":0.10, "mid":0.125, "high":0.17, @@ -426,7 +426,7 @@ "cpu": 24, "cpu_ghz": 3.1, "ram_gib": 186.62, - "net_mbps": 12000, + "net_mbps": 24000, "drive": {"name": "ephem", "size_gib": 14000, "read_io_latency_ms": {"minimum_value":0.07, "low":0.08, "mid":0.12, "high":0.16, "maximum_value":2, "confidence":0.9}, "read_io_per_s": 500000, "write_io_per_s": 400000, From 6a51e7e1ec90f26280ade735a3fe1f87c5893bac Mon Sep 17 00:00:00 2001 From: Ram Srivatsa Kannan Date: Thu, 2 Nov 2023 17:49:52 -0700 Subject: [PATCH 4/4] Addressed remove cass from the error message --- service_capacity_modeling/capacity_planner.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/service_capacity_modeling/capacity_planner.py b/service_capacity_modeling/capacity_planner.py index 41eb892..7ebaae8 100644 --- a/service_capacity_modeling/capacity_planner.py +++ b/service_capacity_modeling/capacity_planner.py @@ -195,8 +195,8 @@ def _set_instance_objects( ] else: raise ValueError( - f"Model not trained to right size cass clusters that are of " - f"instance types {zonal_cluster_capacity.cluster_instance_name}" + f"Model not trained to right size clusters that are of instance" + f" types {zonal_cluster_capacity.cluster_instance_name}" ) for regional_cluster_capacity in desires.current_clusters.regional: if regional_cluster_capacity.cluster_instance_name in hardware.instances: @@ -205,8 +205,8 @@ def _set_instance_objects( ] else: raise ValueError( - f"Model not trained to right size cass clusters that are of " - f"instance types {regional_cluster_capacity.cluster_instance_name}" + f"Model not trained to right size clusters that are of instance" + f" types {regional_cluster_capacity.cluster_instance_name}" )