Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add blue green upgrade settings. #6618

Merged
merged 12 commits into from
Nov 9, 2022
Merged

Add blue green upgrade settings. #6618

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
a88287f
Add blue green upgrade settings.
jjk-g Jul 12, 2022
dea0109
Make standard_rollout_policy required.
jjk-g Oct 10, 2022
2d054c5
PR feedback changes
jjk-g Oct 12, 2022
a7fca59
Merge branch 'GoogleCloudPlatform:main' into main
jjk-g Oct 25, 2022
17fe54e
Validate inactive strategy parameters
jjk-g Oct 25, 2022
c944a67
Fix upgrade_settings MaxItems
jjk-g Oct 27, 2022
66b6c5a
Add MaxItems 1 to several fields
jjk-g Nov 3, 2022
3df8195
Add Computed to batch count and percentage
jjk-g Nov 3, 2022
c862d3b
Switch defaults to computed and optional
jjk-g Nov 3, 2022
d7c17df
Add documentation
jjk-g Nov 4, 2022
9c26b8b
Fix surge parameters error on create
jjk-g Nov 8, 2022
2bcbfd0
Fix merge conflict for flatteners
jjk-g Nov 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
196 changes: 186 additions & 10 deletions mmv1/third_party/terraform/resources/resource_container_node_pool.go.erb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,53 @@ func resourceContainerNodePool() *schema.Resource {
}
}

var schemaBlueGreenSettings = &schema.Schema{
Type: schema.TypeList,
Optional: true,
Computed: true,
shuyama1 marked this conversation as resolved.
Show resolved Hide resolved
jjk-g marked this conversation as resolved.
Show resolved Hide resolved
MaxItems: 1,
Elem: &schema.Resource{
jjk-g marked this conversation as resolved.
Show resolved Hide resolved
Schema: map[string]*schema.Schema{
"standard_rollout_policy": {
Type: schema.TypeList,
Required: true,
shuyama1 marked this conversation as resolved.
Show resolved Hide resolved
jjk-g marked this conversation as resolved.
Show resolved Hide resolved
MaxItems: 1,
Description: `Standard rollout policy is the default policy for blue-green.`,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"batch_percentage": {
Type: schema.TypeFloat,
Optional: true,
Computed: true,
Description: `Percentage of the blue pool nodes to drain in a batch.`,
ValidateFunc: validation.FloatBetween(0.0, 1.0),
},
"batch_node_count": {
Type: schema.TypeInt,
Optional: true,
Computed: true,
Description: `Number of blue nodes to drain in a batch.`,
},
"batch_soak_duration": {
Type: schema.TypeString,
Optional: true,
Computed: true,
Description: `Soak time after each batch gets drained.`,
},
},
},
},
"node_pool_soak_duration": {
Type: schema.TypeString,
Optional: true,
Computed: true,
Description: `Time needed after draining entire blue pool. After this period, blue pool will be cleaned up.`,
},
},
},
Description: `Settings for BlueGreen node pool upgrade.`,
}

var schemaNodePool = map[string]*schema.Schema{
"autoscaling": &schema.Schema{
Type: schema.TypeList,
Expand Down Expand Up @@ -170,17 +217,29 @@ var schemaNodePool = map[string]*schema.Schema{
Schema: map[string]*schema.Schema{
"max_surge": {
Type: schema.TypeInt,
Required: true,
Optional: true,
Computed: true,
shuyama1 marked this conversation as resolved.
Show resolved Hide resolved
ValidateFunc: validation.IntAtLeast(0),
Description: `The number of additional nodes that can be added to the node pool during an upgrade. Increasing max_surge raises the number of nodes that can be upgraded simultaneously. Can be set to 0 or greater.`,
},

"max_unavailable": {
Type: schema.TypeInt,
Required: true,
Optional: true,
Computed: true,
shuyama1 marked this conversation as resolved.
Show resolved Hide resolved
ValidateFunc: validation.IntAtLeast(0),
Description: `The number of nodes that can be simultaneously unavailable during an upgrade. Increasing max_unavailable raises the number of nodes that can be upgraded in parallel. Can be set to 0 or greater.`,
},

"strategy": {
Type: schema.TypeString,
Optional: true,
Default: "SURGE",
ValidateFunc: validation.StringInSlice([]string{"SURGE", "BLUE_GREEN"}, false),
Description: `Update strategy for the given nodepool.`,
},

"blue_green_settings": schemaBlueGreenSettings,
},
},
},
Expand Down Expand Up @@ -822,18 +881,99 @@ func expandNodePool(d *schema.ResourceData, prefix string) (*container.NodePool,
upgradeSettingsConfig := v.([]interface{})[0].(map[string]interface{})
np.UpgradeSettings = &container.UpgradeSettings{}

if v, ok := upgradeSettingsConfig["strategy"]; ok {
np.UpgradeSettings.Strategy = v.(string)
}

if v, ok := upgradeSettingsConfig["max_surge"]; ok {
if np.UpgradeSettings.Strategy != "SURGE" {
return nil, fmt.Errorf("Surge upgrade settings may not be changed when surge strategy is not enabled")
}
np.UpgradeSettings.MaxSurge = int64(v.(int))
}

if v, ok := upgradeSettingsConfig["max_unavailable"]; ok {
if np.UpgradeSettings.Strategy != "SURGE" {
return nil, fmt.Errorf("Surge upgrade settings may not be changed when surge strategy is not enabled")
}
np.UpgradeSettings.MaxUnavailable = int64(v.(int))
}

if v, ok := upgradeSettingsConfig["blue_green_settings"]; ok && len(v.([]interface{})) > 0 {
blueGreenSettingsConfig := v.([]interface{})[0].(map[string]interface{})
np.UpgradeSettings.BlueGreenSettings = &container.BlueGreenSettings{}

if np.UpgradeSettings.Strategy != "BLUE_GREEN" {
return nil, fmt.Errorf("Blue-green upgrade settings may not be changed when blue-green strategy is not enabled")
}

if v, ok := blueGreenSettingsConfig["node_pool_soak_duration"]; ok {
np.UpgradeSettings.BlueGreenSettings.NodePoolSoakDuration = v.(string)
}

if v, ok := blueGreenSettingsConfig["standard_rollout_policy"]; ok && len(v.([]interface{})) > 0 {
standardRolloutPolicyConfig := v.([]interface{})[0].(map[string]interface{})
standardRolloutPolicy := &container.StandardRolloutPolicy{}

if v, ok := standardRolloutPolicyConfig["batch_soak_duration"]; ok {
standardRolloutPolicy.BatchSoakDuration = v.(string)
}
if v, ok := standardRolloutPolicyConfig["batch_node_count"]; ok {
standardRolloutPolicy.BatchNodeCount = int64(v.(int))
}
if v, ok := standardRolloutPolicyConfig["batch_percentage"]; ok {
standardRolloutPolicy.BatchPercentage = v.(float64)
}

np.UpgradeSettings.BlueGreenSettings.StandardRolloutPolicy = standardRolloutPolicy
}
}
}

return np, nil
}

func flattenStandardRolloutPolicy(rp *container.StandardRolloutPolicy) []map[string]interface{} {
if rp == nil {
return nil
}

return []map[string]interface{}{
{
"batch_node_count": rp.BatchNodeCount,
"batch_percentage": rp.BatchPercentage,
"batch_soak_duration": rp.BatchSoakDuration,
},
}
}

func flattenBlueGreenSettings(bg *container.BlueGreenSettings) []map[string]interface{} {
if bg == nil {
return nil
}
return []map[string]interface{}{
{
"node_pool_soak_duration": bg.NodePoolSoakDuration,
"standard_rollout_policy": flattenStandardRolloutPolicy(bg.StandardRolloutPolicy),
},
}
}

func flattenUpgradeSettings(us *container.UpgradeSettings) []map[string]interface{} {
if us == nil {
return nil
}

upgradeSettings := make(map[string]interface{})

upgradeSettings["blue_green_settings"] = flattenBlueGreenSettings(us.BlueGreenSettings)
upgradeSettings["max_surge"] = us.MaxSurge
upgradeSettings["max_unavailable"] = us.MaxUnavailable

upgradeSettings["strategy"] = us.Strategy
return []map[string]interface{}{upgradeSettings}
}

func flattenNodePool(d *schema.ResourceData, config *Config, np *container.NodePool, prefix string) (map[string]interface{}, error) {
userAgent, err := generateUserAgentString(d, config.userAgent)
if err != nil {
Expand Down Expand Up @@ -921,12 +1061,7 @@ func flattenNodePool(d *schema.ResourceData, config *Config, np *container.NodeP
}

if np.UpgradeSettings != nil {
nodePool["upgrade_settings"] = []map[string]interface{}{
{
"max_surge": np.UpgradeSettings.MaxSurge,
"max_unavailable": np.UpgradeSettings.MaxUnavailable,
},
}
nodePool["upgrade_settings"] = flattenUpgradeSettings(np.UpgradeSettings)
} else {
delete(nodePool, "upgrade_settings")
}
Expand Down Expand Up @@ -1364,8 +1499,49 @@ func nodePoolUpdate(d *schema.ResourceData, meta interface{}, nodePoolInfo *Node
upgradeSettings := &container.UpgradeSettings{}
if v, ok := d.GetOk(prefix + "upgrade_settings"); ok {
upgradeSettingsConfig := v.([]interface{})[0].(map[string]interface{})
upgradeSettings.MaxSurge = int64(upgradeSettingsConfig["max_surge"].(int))
upgradeSettings.MaxUnavailable = int64(upgradeSettingsConfig["max_unavailable"].(int))
upgradeSettings.Strategy = upgradeSettingsConfig["strategy"].(string)

if d.HasChange(prefix + "upgrade_settings.0.max_surge") {
if upgradeSettings.Strategy != "SURGE" {
return fmt.Errorf("%v Surge upgrade settings may not be changed when surge strategy is not enabled", v)
}
if v, ok := upgradeSettingsConfig["max_surge"]; ok {
upgradeSettings.MaxSurge = int64(v.(int))
}
}

if d.HasChange(prefix + "upgrade_settings.0.max_unavailable") {
if upgradeSettings.Strategy != "SURGE" {
return fmt.Errorf("%v Surge upgrade settings may not be changed when surge strategy is not enabled", v)
}
if v, ok := upgradeSettingsConfig["max_unavailable"]; ok {
upgradeSettings.MaxUnavailable = int64(v.(int))
}
}

if d.HasChange(prefix + "upgrade_settings.0.blue_green_settings") {
if upgradeSettings.Strategy != "BLUE_GREEN" {
return fmt.Errorf("Blue-green upgrade settings may not be changed when blue-green strategy is not enabled")
}

blueGreenSettings := &container.BlueGreenSettings{}
blueGreenSettingsConfig := upgradeSettingsConfig["blue_green_settings"].([]interface{})[0].(map[string]interface{})
blueGreenSettings.NodePoolSoakDuration = blueGreenSettingsConfig["node_pool_soak_duration"].(string)

if v, ok := blueGreenSettingsConfig["standard_rollout_policy"]; ok && len(v.([]interface{})) > 0 {
standardRolloutPolicy := &container.StandardRolloutPolicy{}
standardRolloutPolicyConfig := v.([]interface{})[0].(map[string]interface{})
standardRolloutPolicy.BatchSoakDuration = standardRolloutPolicyConfig["batch_soak_duration"].(string)
if v, ok := standardRolloutPolicyConfig["batch_node_count"]; ok {
standardRolloutPolicy.BatchNodeCount = int64(v.(int))
}
if v, ok := standardRolloutPolicyConfig["batch_percentage"]; ok {
standardRolloutPolicy.BatchPercentage = v.(float64)
}
blueGreenSettings.StandardRolloutPolicy = standardRolloutPolicy
}
upgradeSettings.BlueGreenSettings = blueGreenSettings
}
}
req := &container.UpdateNodePoolRequest{
UpgradeSettings: upgradeSettings,
Expand Down
54 changes: 46 additions & 8 deletions mmv1/third_party/terraform/tests/resource_container_node_pool_test.go.erb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -519,15 +519,31 @@ func TestAccContainerNodePool_withUpgradeSettings(t *testing.T) {
CheckDestroy: testAccCheckContainerClusterDestroyProducer(t),
Steps: []resource.TestStep{
{
Config: testAccContainerNodePool_withUpgradeSettings(cluster, np, 2, 3),
Config: testAccContainerNodePool_withUpgradeSettings(cluster, np, 2, 3, "SURGE", "", 0, 0.0, ""),
},
{
ResourceName: "google_container_node_pool.with_upgrade_settings",
ImportState: true,
ImportStateVerify: true,
},
{
Config: testAccContainerNodePool_withUpgradeSettings(cluster, np, 1, 1),
Config: testAccContainerNodePool_withUpgradeSettings(cluster, np, 1, 1, "SURGE", "", 0, 0.0, ""),
},
{
Copy link
Member

@shuyama1 shuyama1 Nov 4, 2022 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Running some tests locally. When I only run the last two steps of the test (tests with only BLUE_GREEN config). I get errors like:

provider_test.go:315: Step 1/4 error: Error running apply: exit status 1
        
        Error: Surge upgrade settings may not be changed when surge strategy is not enabled
        
          with google_container_node_pool.with_upgrade_settings,
          on terraform_plugin_test.tf line 13, in resource "google_container_node_pool" "with_upgrade_settings":
          13: resource "google_container_node_pool" "with_upgrade_settings" {

Any ideas why this error occurs?

ResourceName: "google_container_node_pool.with_upgrade_settings",
ImportState: true,
ImportStateVerify: true,
},
{
Config: testAccContainerNodePool_withUpgradeSettings(cluster, np, 0, 0, "BLUE_GREEN", "100s", 1, 0.0, "0s"),
jjk-g marked this conversation as resolved.
Show resolved Hide resolved
},
{
ResourceName: "google_container_node_pool.with_upgrade_settings",
ImportState: true,
ImportStateVerify: true,
},
{
Config: testAccContainerNodePool_withUpgradeSettings(cluster, np, 0, 0, "BLUE_GREEN", "100s", 0, 0.5, "1s"),
},
{
ResourceName: "google_container_node_pool.with_upgrade_settings",
Expand Down Expand Up @@ -2211,7 +2227,32 @@ resource "google_container_node_pool" "with_boot_disk_kms_key" {
}
<% end -%>

func testAccContainerNodePool_withUpgradeSettings(clusterName string, nodePoolName string, maxSurge int, maxUnavailable int) string {
func makeUpgradeSettings(maxSurge int, maxUnavailable int, strategy string, nodePoolSoakDuration string, batchNodeCount int, batchPercentage float64, batchSoakDuration string) string {
if strategy == "BLUE_GREEN" {
return fmt.Sprintf(`
upgrade_settings {
strategy = "%s"
blue_green_settings {
node_pool_soak_duration = "%s"
standard_rollout_policy {
batch_node_count = %d
batch_percentage = %f
batch_soak_duration = "%s"
}
}
}
`, strategy, nodePoolSoakDuration, batchNodeCount, batchPercentage, batchSoakDuration)
}
return fmt.Sprintf(`
upgrade_settings {
max_surge = %d
max_unavailable = %d
strategy = "%s"
}
`, maxSurge, maxUnavailable, strategy)
}

func testAccContainerNodePool_withUpgradeSettings(clusterName string, nodePoolName string, maxSurge int, maxUnavailable int, strategy string, nodePoolSoakDuration string, batchNodeCount int, batchPercentage float64, batchSoakDuration string) string {
return fmt.Sprintf(`
data "google_container_engine_versions" "central1" {
location = "us-central1"
Expand All @@ -2229,12 +2270,9 @@ resource "google_container_node_pool" "with_upgrade_settings" {
location = "us-central1"
cluster = "${google_container_cluster.cluster.name}"
initial_node_count = 1
upgrade_settings {
max_surge = %d
max_unavailable = %d
}
%s
}
`, clusterName, nodePoolName, maxSurge, maxUnavailable)
`, clusterName, nodePoolName, makeUpgradeSettings(maxSurge, maxUnavailable, strategy, nodePoolSoakDuration, batchNodeCount, batchPercentage, batchSoakDuration))
}

func testAccContainerNodePool_withGPU(cluster, np string) string {
Expand Down