Skip to content

Fix/tke cluster level retry #1029

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
May 10, 2022
Merged

Fix/tke cluster level retry #1029

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d13d66d
fix: tke - add cluster level update retry
Kagashino May 6, 2022
53659c0
fix: tke - add RESOURCEUNAVAILABLE retryable modify code
Kagashino May 6, 2022
6f9be5e
fix: tke - cidr conflicts
Kagashino May 10, 2022
3b684d0
fix: tke auth - add retry code
Kagashino May 10, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 0 additions & 108 deletions go.sum
View file
Open in desktop

Large diffs are not rendered by default.

34 changes: 34 additions & 0 deletions tencentcloud/basic_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -536,6 +536,40 @@ locals {
// End of TcaPlus DB

// TKE Service

// List sample CIDRs to avoid conflict when running multiple cluster testcase parallel
const TkeCIDRs = `
variable "tke_cidr_a" {
default = [
"10.31.0.0/23",
"10.31.2.0/24",
"10.31.3.0/25",
"10.31.3.128/26",
"10.31.3.192/26"
]
}

variable "tke_cidr_b" {
default = [
"172.18.0.0/20",
"172.18.16.0/21",
"172.18.24.0/21",
"172.18.32.0/20",
"172.18.48.0/20"
]
}

variable "tke_cidr_c" {
default = [
"192.168.0.0/18",
"192.168.64.0/19",
"192.168.96.0/20",
"192.168.112.0/21",
"192.168.120.0/21"
]
}
`

const TkeInstanceType = `
data "tencentcloud_instance_types" "ins_type" {
availability_zone = "` + defaultCvmAZone + `"
Expand Down
12 changes: 11 additions & 1 deletion tencentcloud/resource_tc_kubernetes_auth_attachment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@ package tencentcloud
import (
"context"

"github.com/hashicorp/terraform-plugin-sdk/helper/resource"

"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
tke "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/tke/v20180525"
"github.com/tencentcloudstack/terraform-provider-tencentcloud/tencentcloud/internal/helper"
Expand Down Expand Up @@ -133,7 +135,15 @@ func resourceTencentCloudTKEAuthAttachmentCreate(d *schema.ResourceData, meta in
request.ServiceAccounts.AutoCreateDiscoveryAnonymousAuth = helper.Bool(v.(bool))
}

if err := service.ModifyClusterAuthenticationOptions(ctx, request); err != nil {
err := resource.Retry(writeRetryTimeout, func() *resource.RetryError {
err := service.ModifyClusterAuthenticationOptions(ctx, request)
if err != nil {
return retryError(err, tke.RESOURCEUNAVAILABLE_CLUSTERSTATE)
}
return nil
})

if err != nil {
return err
}

Expand Down
4 changes: 2 additions & 2 deletions tencentcloud/resource_tc_kubernetes_auth_attachment_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ func TestAccTencentCloudTkeAuthAttachResource(t *testing.T) {
}

func testAccTkeAuthAttach() string {
return `
return TkeCIDRs + `
variable "availability_zone" {
default = "ap-guangzhou-3"
}
Expand All @@ -38,7 +38,7 @@ data "tencentcloud_vpc_subnets" "vpc" {

resource "tencentcloud_kubernetes_cluster" "managed_cluster" {
vpc_id = data.tencentcloud_vpc_subnets.vpc.instance_list.0.vpc_id
cluster_cidr = "10.31.0.0/16"
cluster_cidr = var.tke_cidr_a.1
cluster_max_pod_num = 32
cluster_name = "for-auth-attachment"
cluster_desc = "test cluster desc"
Expand Down
11 changes: 10 additions & 1 deletion tencentcloud/resource_tc_kubernetes_cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2667,7 +2667,16 @@ func resourceTencentCloudTkeClusterUpdate(d *schema.ResourceData, meta interface
clusterDesc := d.Get("cluster_desc").(string)
clusterLevel := d.Get("cluster_level").(string)
autoUpgradeClusterLevel := d.Get("auto_upgrade_cluster_level").(bool)
if err := tkeService.ModifyClusterAttribute(ctx, id, projectId, clusterName, clusterDesc, clusterLevel, autoUpgradeClusterLevel); err != nil {
err := resource.Retry(writeRetryTimeout, func() *resource.RetryError {
err := tkeService.ModifyClusterAttribute(ctx, id, projectId, clusterName, clusterDesc, clusterLevel, autoUpgradeClusterLevel)
if err != nil {
// create and update immediately may cause cluster level syntax error, this error can wait until cluster level state normal
return retryError(err, tke.INTERNALERROR_UNEXPECTEDINTERNAL, tke.RESOURCEUNAVAILABLE)
}
return nil
})

if err != nil {
return err
}
}
Expand Down
163 changes: 47 additions & 116 deletions tencentcloud/resource_tc_kubernetes_cluster_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,10 @@ func TestAccTencentCloudTkeResourceBasic(t *testing.T) {
CheckDestroy: testAccCheckTkeDestroy,
Steps: []resource.TestStep{
{
Config: testAccTkeCluster("test", "test"),
Config: testAccTkeCluster,
Check: resource.ComposeTestCheckFunc(
testAccCheckTkeExists(testTkeClusterResourceKey),
resource.TestCheckResourceAttr(testTkeClusterResourceKey, "cluster_cidr", "10.31.0.0/16"),
resource.TestCheckResourceAttr(testTkeClusterResourceKey, "cluster_cidr", "10.31.0.0/23"),
resource.TestCheckResourceAttr(testTkeClusterResourceKey, "cluster_max_pod_num", "32"),
resource.TestCheckResourceAttr(testTkeClusterResourceKey, "cluster_name", "test"),
resource.TestCheckResourceAttr(testTkeClusterResourceKey, "cluster_desc", "test cluster desc"),
Expand All @@ -72,45 +72,20 @@ func TestAccTencentCloudTkeResourceBasic(t *testing.T) {
resource.TestCheckResourceAttr(testTkeClusterResourceKey, "tags.test", "test"),
resource.TestCheckResourceAttr(testTkeClusterResourceKey, "security_policy.#", "2"),
resource.TestCheckResourceAttrSet(testTkeClusterResourceKey, "cluster_external_endpoint"),
resource.TestCheckResourceAttr(testTkeClusterResourceKey, "cluster_level", "L5"),
resource.TestCheckResourceAttr(testTkeClusterResourceKey, "auto_upgrade_cluster_level", "true"),
resource.TestCheckResourceAttr(testTkeClusterResourceKey, "labels.test1", "test1"),
resource.TestCheckResourceAttr(testTkeClusterResourceKey, "labels.test2", "test2"),
),
},
{
Config: testAccTkeCluster("abc", "abc"),
Config: testAccTkeClusterUpdate,
Check: resource.ComposeTestCheckFunc(
testAccCheckTkeExists(testTkeClusterResourceKey),
resource.TestCheckResourceAttr(testTkeClusterResourceKey, "cluster_name", "test2"),
resource.TestCheckResourceAttr(testTkeClusterResourceKey, "cluster_desc", "test cluster desc2"),
resource.TestCheckNoResourceAttr(testTkeClusterResourceKey, "tags.test"),
resource.TestCheckResourceAttr(testTkeClusterResourceKey, "tags.abc", "abc"),
),
},
},
})
}

func TestAccTencentCloudTkeResourceClusterLevel(t *testing.T) {
resource.Test(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
CheckDestroy: testAccCheckTkeDestroy,
Steps: []resource.TestStep{
{
Config: testAccTkeClusterLevel,
Check: resource.ComposeTestCheckFunc(
testAccCheckTkeExists(testTkeClusterResourceKey),
resource.TestCheckResourceAttr(testTkeClusterResourceKey, "cluster_cidr", "192.168.0.0/16"),
resource.TestCheckResourceAttr(testTkeClusterResourceKey, "cluster_max_pod_num", "32"),
resource.TestCheckResourceAttr(testTkeClusterResourceKey, "cluster_name", "test"),
resource.TestCheckResourceAttr(testTkeClusterResourceKey, "cluster_level", "L5"),
resource.TestCheckResourceAttr(testTkeClusterResourceKey, "auto_upgrade_cluster_level", "true"),
resource.TestCheckResourceAttr(testTkeClusterResourceKey, "worker_instances_list.#", "1"),
resource.TestCheckResourceAttrSet(testTkeClusterResourceKey, "worker_instances_list.0.instance_id"),
),
},
{
Config: testAccTkeClusterLevelUpdate,
Check: resource.ComposeTestCheckFunc(
testAccCheckTkeExists(testTkeClusterResourceKey),
resource.TestCheckResourceAttr(testTkeClusterResourceKey, "cluster_level", "L20"),
resource.TestCheckResourceAttr(testTkeClusterResourceKey, "auto_upgrade_cluster_level", "false"),
),
Expand Down Expand Up @@ -204,31 +179,25 @@ func testAccCheckTkeExists(n string) resource.TestCheckFunc {
}
}

func testAccTkeCluster(key, value string) string {
return fmt.Sprintf(TkeInstanceType+defaultImages+`
const TkeDeps = TkeExclusiveNetwork + TkeInstanceType + TkeCIDRs + defaultImages

const testAccTkeCluster = TkeDeps + `
variable "availability_zone" {
default = "ap-guangzhou-3"
}

variable "cluster_cidr" {
default = "10.31.0.0/16"
}

data "tencentcloud_vpc_subnets" "vpc" {
is_default = true
availability_zone = var.availability_zone
}

resource "tencentcloud_kubernetes_cluster" "managed_cluster" {
vpc_id = data.tencentcloud_vpc_subnets.vpc.instance_list.0.vpc_id
cluster_cidr = var.cluster_cidr
vpc_id = local.vpc_id
cluster_cidr = var.tke_cidr_a.0
cluster_max_pod_num = 32
cluster_name = "test"
cluster_desc = "test cluster desc"
cluster_max_service_num = 32
cluster_internet = true
cluster_version = "1.18.4"
cluster_os = "tlinux2.2(tkernel3)x86_64"
cluster_level = "L5"
auto_upgrade_cluster_level = true
managed_cluster_internet_security_policies = ["3.3.3.3", "1.1.1.1"]
worker_config {
count = 1
Expand All @@ -239,7 +208,7 @@ resource "tencentcloud_kubernetes_cluster" "managed_cluster" {
internet_charge_type = "TRAFFIC_POSTPAID_BY_HOUR"
internet_max_bandwidth_out = 100
public_ip_assigned = true
subnet_id = data.tencentcloud_vpc_subnets.vpc.instance_list.0.subnet_id
subnet_id = local.subnet_id
img_id = var.default_img_id

data_disk {
Expand All @@ -260,7 +229,7 @@ resource "tencentcloud_kubernetes_cluster" "managed_cluster" {
cluster_deploy_type = "MANAGED_CLUSTER"

tags = {
"%s" = "%s"
"test" = "test"
}

unschedulable = 0
Expand All @@ -273,35 +242,25 @@ resource "tencentcloud_kubernetes_cluster" "managed_cluster" {
"root-dir=/var/lib/kubelet"
]
}
`, key, value,
)
}

const testAccTkeClusterLevel = TkeInstanceType + defaultImages + `
`
const testAccTkeClusterUpdate = TkeDeps + `
variable "availability_zone" {
default = "ap-guangzhou-3"
}

variable "cluster_cidr" {
default = "192.168.0.0/16"
}

data "tencentcloud_vpc_subnets" "vpc" {
is_default = true
availability_zone = var.availability_zone
}

resource "tencentcloud_kubernetes_cluster" "managed_cluster" {
vpc_id = data.tencentcloud_vpc_subnets.vpc.instance_list.0.vpc_id
cluster_cidr = var.cluster_cidr
vpc_id = local.vpc_id
cluster_cidr = var.tke_cidr_a.0
cluster_max_pod_num = 32
cluster_name = "test"
cluster_desc = "test cluster desc"
cluster_name = "test2"
cluster_desc = "test cluster desc2"
cluster_max_service_num = 32
cluster_internet = true
cluster_version = "1.18.4"
cluster_os = var.default_img
cluster_level = "L5"
auto_upgrade_cluster_level = true
cluster_os = "tlinux2.2(tkernel3)x86_64"
cluster_level = "L20"
auto_upgrade_cluster_level = false
managed_cluster_internet_security_policies = ["3.3.3.3", "1.1.1.1"]
worker_config {
count = 1
availability_zone = var.availability_zone
Expand All @@ -311,9 +270,18 @@ resource "tencentcloud_kubernetes_cluster" "managed_cluster" {
internet_charge_type = "TRAFFIC_POSTPAID_BY_HOUR"
internet_max_bandwidth_out = 100
public_ip_assigned = true
subnet_id = data.tencentcloud_vpc_subnets.vpc.instance_list.0.subnet_id
subnet_id = local.subnet_id
img_id = var.default_img_id

data_disk {
disk_type = "CLOUD_PREMIUM"
disk_size = 50
file_system = "ext3"
auto_format_and_mount = "true"
mount_target = "/var/lib/docker"
disk_partition = "/dev/sdb1"
}

enhanced_security_service = false
enhanced_monitor_service = false
user_data = "dGVzdA=="
Expand All @@ -322,55 +290,18 @@ resource "tencentcloud_kubernetes_cluster" "managed_cluster" {

cluster_deploy_type = "MANAGED_CLUSTER"

unschedulable = 0
}
`

const testAccTkeClusterLevelUpdate = TkeInstanceType + defaultImages + `
variable "availability_zone" {
default = "ap-guangzhou-3"
}

variable "cluster_cidr" {
default = "192.168.0.0/16"
}

data "tencentcloud_vpc_subnets" "vpc" {
is_default = true
availability_zone = var.availability_zone
}

resource "tencentcloud_kubernetes_cluster" "managed_cluster" {
vpc_id = data.tencentcloud_vpc_subnets.vpc.instance_list.0.vpc_id
cluster_cidr = var.cluster_cidr
cluster_max_pod_num = 32
cluster_name = "test"
cluster_desc = "test cluster desc"
cluster_max_service_num = 32
cluster_version = "1.18.4"
cluster_os = var.default_img
cluster_level = "L20"
auto_upgrade_cluster_level = false
worker_config {
count = 1
availability_zone = var.availability_zone
instance_type = local.type1
system_disk_type = "CLOUD_SSD"
system_disk_size = 60
internet_charge_type = "TRAFFIC_POSTPAID_BY_HOUR"
internet_max_bandwidth_out = 100
public_ip_assigned = true
subnet_id = data.tencentcloud_vpc_subnets.vpc.instance_list.0.subnet_id
img_id = var.default_img_id

enhanced_security_service = false
enhanced_monitor_service = false
user_data = "dGVzdA=="
password = "ZZXXccvv1212"
tags = {
"abc" = "abc"
}

cluster_deploy_type = "MANAGED_CLUSTER"

unschedulable = 0

labels = {
"test1" = "test1",
"test2" = "test2",
}
extra_args = [
"root-dir=/var/lib/kubelet"
]
}
`
6 changes: 2 additions & 4 deletions tencentcloud/service_tencentcloud_tke.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -989,10 +989,8 @@ func (me *TkeService) ModifyClusterAttribute(ctx context.Context, id string, pro
request.ClusterLevel = &clusterLevel
}

if autoUpgradeClusterLevel {
request.AutoUpgradeClusterLevel = &tke.AutoUpgradeClusterLevel{
IsAutoUpgrade: &autoUpgradeClusterLevel,
}
request.AutoUpgradeClusterLevel = &tke.AutoUpgradeClusterLevel{
IsAutoUpgrade: &autoUpgradeClusterLevel,
}

ratelimit.Check(request.GetAction())
Expand Down