resource_ibm_container_vpc_worker_pool.go

package ibm

import (
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/hashicorp/terraform-plugin-sdk/helper/resource"
	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"

	v1 "github.com/IBM-Cloud/bluemix-go/api/container/containerv1"
	v2 "github.com/IBM-Cloud/bluemix-go/api/container/containerv2"
	"github.com/IBM-Cloud/bluemix-go/bmxerror"
)

const (
	workerDesired = "deployed"
)

func resourceIBMContainerVpcWorkerPool() *schema.Resource {

	return &schema.Resource{
		Create:   resourceIBMContainerVpcWorkerPoolCreate,
		Update:   resourceIBMContainerVpcWorkerPoolUpdate,
		Read:     resourceIBMContainerVpcWorkerPoolRead,
		Delete:   resourceIBMContainerVpcWorkerPoolDelete,
		Exists:   resourceIBMContainerVpcWorkerPoolExists,
		Importer: &schema.ResourceImporter{},
		Timeouts: &schema.ResourceTimeout{
			Create: schema.DefaultTimeout(90 * time.Minute),
			Delete: schema.DefaultTimeout(90 * time.Minute),
		},

		Schema: map[string]*schema.Schema{
			"cluster": {
				Type:        schema.TypeString,
				Required:    true,
				ForceNew:    true,
				Description: "Cluster name",
			},

			"flavor": {
				Type:        schema.TypeString,
				Required:    true,
				ForceNew:    true,
				Description: "cluster node falvor",
			},

			"worker_pool_name": {
				Type:        schema.TypeString,
				Required:    true,
				ForceNew:    true,
				Description: "worker pool name",
			},

			"zones": {
				Type:        schema.TypeSet,
				Required:    true,
				Description: "Zones info",
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"name": {
							Type:        schema.TypeString,
							Required:    true,
							Description: "zone name",
						},

						"subnet_id": {
							Type:        schema.TypeString,
							Required:    true,
							Description: "subnet ID",
						},
					},
				},
			},

			"labels": {
				Type:             schema.TypeMap,
				Optional:         true,
				Computed:         true,
				DiffSuppressFunc: applyOnce,
				Elem:             &schema.Schema{Type: schema.TypeString},
				Description:      "Labels",
			},

			"resource_group_id": {
				Type:        schema.TypeString,
				Optional:    true,
				Computed:    true,
				Description: "ID of the resource group.",
				ForceNew:    true,
			},
			"vpc_id": {
				Type:        schema.TypeString,
				Required:    true,
				Description: "The vpc id where the cluster is",
				ForceNew:    true,
			},
			"worker_count": {
				Type:        schema.TypeInt,
				Required:    true,
				Description: "The number of workers",
			},
			"entitlement": {
				Type:             schema.TypeString,
				Optional:         true,
				DiffSuppressFunc: applyOnce,
				Description:      "Entitlement option reduces additional OCP Licence cost in Openshift Clusters",
			},
		},
	}
}

func resourceIBMContainerVpcWorkerPoolCreate(d *schema.ResourceData, meta interface{}) error {

	wpClient, err := meta.(ClientSession).VpcContainerAPI()
	if err != nil {
		return err
	}

	clusterNameorID := d.Get("cluster").(string)
	var zonei []interface{}

	zone := []v2.Zone{}

	if res, ok := d.GetOk("zones"); ok {
		zonei = res.(*schema.Set).List()
		for _, e := range zonei {
			r, _ := e.(map[string]interface{})
			zoneParam := v2.Zone{
				ID:       r["name"].(string),
				SubnetID: r["subnet_id"].(string),
			}
			zone = append(zone, zoneParam)
		}

	}

	// for _, e := range d.Get("zones").(*schema.Set).List() {
	// 	value := e.(map[string]interface{})
	// 	id := value["id"].(string)
	// 	subnetid := value["subnet_id"].(string)

	// }

	workerPoolConfig := v2.WorkerPoolConfig{
		Name:        d.Get("worker_pool_name").(string),
		VpcID:       d.Get("vpc_id").(string),
		Flavor:      d.Get("flavor").(string),
		WorkerCount: d.Get("worker_count").(int),
		Zones:       zone,
	}

	if l, ok := d.GetOk("labels"); ok {
		labels := make(map[string]string)
		for k, v := range l.(map[string]interface{}) {
			labels[k] = v.(string)
		}
		workerPoolConfig.Labels = labels
	}
	params := v2.WorkerPoolRequest{
		WorkerPoolConfig: workerPoolConfig,
		Cluster:          clusterNameorID,
	}

	// Update workerpoolConfig with Entitlement option if provided
	if v, ok := d.GetOk("entitlement"); ok {
		workerPoolConfig.Entitlement = v.(string)
	}

	workerPoolsAPI := wpClient.WorkerPools()
	targetEnv, err := getVpcClusterTargetHeader(d, meta)
	if err != nil {
		return err
	}

	res, err := workerPoolsAPI.CreateWorkerPool(params, targetEnv)
	if err != nil {
		return err
	}

	d.SetId(fmt.Sprintf("%s/%s", clusterNameorID, res.ID))

	//wait for workerpool availability
	_, err = WaitForWorkerPoolAvailable(d, meta, clusterNameorID, res.ID, d.Timeout(schema.TimeoutCreate), targetEnv)
	if err != nil {
		return fmt.Errorf(
			"Error waiting for workerpool (%s) to become ready: %s", d.Id(), err)
	}

	return resourceIBMContainerVpcWorkerPoolUpdate(d, meta)
}

func resourceIBMContainerVpcWorkerPoolUpdate(d *schema.ResourceData, meta interface{}) error {

	if d.HasChange("worker_count") {
		clusterNameOrID := d.Get("cluster").(string)
		workerPoolName := d.Get("worker_pool_name").(string)
		count := d.Get("worker_count").(int)
		targetEnv, err := getVpcClusterTargetHeader(d, meta)
		if err != nil {
			return err
		}
		ClusterClient, err := meta.(ClientSession).ContainerAPI()
		if err != nil {
			return err
		}
		Env := v1.ClusterTargetHeader{ResourceGroup: targetEnv.ResourceGroup}

		err = ClusterClient.WorkerPools().ResizeWorkerPool(clusterNameOrID, workerPoolName, count, Env)
		if err != nil {
			return fmt.Errorf(
				"Error updating the worker_count %d: %s", count, err)
		}
	}

	if d.HasChange("zones") && !d.IsNewResource() {
		clusterID := d.Get("cluster").(string)
		workerPoolName := d.Get("worker_pool_name").(string)
		targetEnv, err := getVpcClusterTargetHeader(d, meta)
		if err != nil {
			return err
		}
		oldList, newList := d.GetChange("zones")
		if oldList == nil {
			oldList = new(schema.Set)
		}
		if newList == nil {
			newList = new(schema.Set)
		}
		os := oldList.(*schema.Set)
		ns := newList.(*schema.Set)
		remove := os.Difference(ns).List()
		add := ns.Difference(os).List()
		if len(add) > 0 {
			csClient, err := meta.(ClientSession).VpcContainerAPI()
			if err != nil {
				return err
			}
			for _, zone := range add {
				newZone := zone.(map[string]interface{})
				zoneParam := v2.WorkerPoolZone{
					Cluster:      clusterID,
					Id:           newZone["name"].(string),
					SubnetID:     newZone["subnet_id"].(string),
					WorkerPoolID: workerPoolName,
				}
				err = csClient.WorkerPools().CreateWorkerPoolZone(zoneParam, targetEnv)
				if err != nil {
					return fmt.Errorf("Error adding zone to conatiner vpc cluster: %s", err)
				}
				_, err = WaitForWorkerPoolAvailable(d, meta, clusterID, workerPoolName, d.Timeout(schema.TimeoutCreate), targetEnv)
				if err != nil {
					return fmt.Errorf(
						"Error waiting for workerpool (%s) to become ready: %s", d.Id(), err)
				}

			}
		}
		if len(remove) > 0 {
			for _, zone := range remove {
				oldZone := zone.(map[string]interface{})
				ClusterClient, err := meta.(ClientSession).ContainerAPI()
				if err != nil {
					return err
				}
				Env := v1.ClusterTargetHeader{ResourceGroup: targetEnv.ResourceGroup}
				err = ClusterClient.WorkerPools().RemoveZone(clusterID, oldZone["name"].(string), workerPoolName, Env)
				if err != nil {
					return fmt.Errorf("Error deleting zone to conatiner vpc cluster: %s", err)
				}
				_, err = WaitForV2WorkerZoneDeleted(clusterID, workerPoolName, oldZone["name"].(string), meta, d.Timeout(schema.TimeoutDelete), targetEnv)
				if err != nil {
					return fmt.Errorf(
						"Error waiting for deleting workers of worker pool (%s) of cluster (%s):  %s", workerPoolName, clusterID, err)
				}
			}
		}
	}
	return resourceIBMContainerVpcWorkerPoolRead(d, meta)
}

func resourceIBMContainerVpcWorkerPoolRead(d *schema.ResourceData, meta interface{}) error {
	wpClient, err := meta.(ClientSession).VpcContainerAPI()
	if err != nil {
		return err
	}
	parts, err := idParts(d.Id())
	if err != nil {
		return err
	}
	cluster := parts[0]
	workerPoolID := parts[1]

	workerPoolsAPI := wpClient.WorkerPools()
	targetEnv, err := getVpcClusterTargetHeader(d, meta)
	if err != nil {
		return err
	}

	workerPool, err := workerPoolsAPI.GetWorkerPool(cluster, workerPoolID, targetEnv)
	if err != nil {
		return err
	}

	var zones = make([]map[string]interface{}, 0)
	for _, zone := range workerPool.Zones {
		for _, subnet := range zone.Subnets {
			zoneInfo := map[string]interface{}{
				"name":      zone.ID,
				"subnet_id": subnet.ID,
			}
			zones = append(zones, zoneInfo)
		}
	}

	cls, err := wpClient.Clusters().GetCluster(cluster, targetEnv)
	if err != nil {
		return fmt.Errorf("Error retrieving conatiner vpc cluster: %s", err)
	}

	d.Set("worker_pool_name", workerPool.PoolName)
	d.Set("flavor", workerPool.Flavor)
	d.Set("worker_count", workerPool.WorkerCount)
	d.Set("provider", workerPool.Provider)
	d.Set("labels", workerPool.Labels)
	d.Set("zones", zones)
	d.Set("resource_group_id", cls.ResourceGroupID)
	d.Set("cluster", cluster)
	d.Set("vpc_id", workerPool.VpcID)

	controller, err := getBaseController(meta)
	if err != nil {
		return err
	}
	d.Set(ResourceControllerURL, controller+"/kubernetes/clusters")
	return nil
}

func resourceIBMContainerVpcWorkerPoolDelete(d *schema.ResourceData, meta interface{}) error {
	wpClient, err := meta.(ClientSession).VpcContainerAPI()
	if err != nil {
		return err
	}
	parts, err := idParts(d.Id())
	if err != nil {
		return err
	}
	clusterNameorID := parts[0]
	workerPoolNameorID := parts[1]

	workerPoolsAPI := wpClient.WorkerPools()
	targetEnv, err := getVpcClusterTargetHeader(d, meta)
	if err != nil {
		return err
	}

	err = workerPoolsAPI.DeleteWorkerPool(clusterNameorID, workerPoolNameorID, targetEnv)
	if err != nil {
		return err
	}
	_, err = WaitForVpcWorkerDelete(clusterNameorID, workerPoolNameorID, meta, d.Timeout(schema.TimeoutDelete), targetEnv)
	if err != nil {
		return fmt.Errorf(
			"Error waiting for removing workers of worker pool (%s) of cluster (%s): %s", workerPoolNameorID, clusterNameorID, err)
	}
	d.SetId("")
	return nil
}

func resourceIBMContainerVpcWorkerPoolExists(d *schema.ResourceData, meta interface{}) (bool, error) {
	wpClient, err := meta.(ClientSession).VpcContainerAPI()
	if err != nil {
		return false, err
	}
	parts, err := idParts(d.Id())
	if err != nil {
		return false, err
	}
	cluster := parts[0]
	workerPoolID := parts[1]

	workerPoolsAPI := wpClient.WorkerPools()
	targetEnv, err := getVpcClusterTargetHeader(d, meta)
	if err != nil {
		return false, err
	}

	workerPool, err := workerPoolsAPI.GetWorkerPool(cluster, workerPoolID, targetEnv)
	if err != nil {
		if apiErr, ok := err.(bmxerror.RequestFailure); ok {
			if apiErr.StatusCode() == 404 {
				return false, nil
			}
		}
		return false, fmt.Errorf("Error communicating with the API: %s", err)
	}

	return workerPool.ID == workerPoolID, nil
}

// WaitForWorkerPoolAvailable Waits for worker creation
func WaitForWorkerPoolAvailable(d *schema.ResourceData, meta interface{}, clusterNameOrID, workerPoolNameOrID string, timeout time.Duration, target v2.ClusterTargetHeader) (interface{}, error) {
	wpClient, err := meta.(ClientSession).VpcContainerAPI()
	if err != nil {
		return nil, err
	}
	log.Printf("Waiting for workerpool (%s) to be available.", d.Id())
	// id := d.Id()

	stateConf := &resource.StateChangeConf{
		Pending:    []string{"provision_pending"},
		Target:     []string{workerDesired},
		Refresh:    vpcWorkerPoolStateRefreshFunc(wpClient.Workers(), clusterNameOrID, workerPoolNameOrID, target),
		Timeout:    timeout,
		Delay:      10 * time.Second,
		MinTimeout: 10 * time.Second,
	}

	return stateConf.WaitForState()
}

func vpcWorkerPoolStateRefreshFunc(client v2.Workers, instanceID string, workerPoolNameOrID string, target v2.ClusterTargetHeader) resource.StateRefreshFunc {
	return func() (interface{}, string, error) {
		workerFields, err := client.ListByWorkerPool(instanceID, "", false, target)
		if err != nil {
			return nil, "", fmt.Errorf("Error retrieving workers for cluster: %s", err)
		}
		// Check active transactions
		//Check for worker state to be deployed
		//Done worker has two fields desiredState and actualState , so check for those 2
		for _, e := range workerFields {
			if e.PoolName == workerPoolNameOrID || e.PoolID == workerPoolNameOrID {
				if strings.Compare(e.LifeCycle.ActualState, "deployed") != 0 {
					log.Printf("worker: %s state: %s", e.ID, e.LifeCycle.ActualState)
					return workerFields, "provision_pending", nil
				}
			}
		}
		return workerFields, workerDesired, nil
	}
}

func WaitForVpcWorkerDelete(clusterNameOrID, workerPoolNameOrID string, meta interface{}, timeout time.Duration, target v2.ClusterTargetHeader) (interface{}, error) {
	wpClient, err := meta.(ClientSession).VpcContainerAPI()
	if err != nil {
		return nil, err
	}

	stateConf := &resource.StateChangeConf{
		Pending:    []string{"deleting"},
		Target:     []string{workerDeleteState},
		Refresh:    vpcworkerPoolDeleteStateRefreshFunc(wpClient.Workers(), clusterNameOrID, workerPoolNameOrID, target),
		Timeout:    timeout,
		Delay:      10 * time.Second,
		MinTimeout: 10 * time.Second,
	}

	return stateConf.WaitForState()
}

func vpcworkerPoolDeleteStateRefreshFunc(client v2.Workers, instanceID, workerPoolNameOrID string, target v2.ClusterTargetHeader) resource.StateRefreshFunc {
	return func() (interface{}, string, error) {
		workerFields, err := client.ListByWorkerPool(instanceID, "", true, target)
		if err != nil {
			return nil, "", fmt.Errorf("Error retrieving workers for cluster: %s", err)
		}
		//Done worker has two fields desiredState and actualState , so check for those 2
		for _, e := range workerFields {
			if e.PoolName == workerPoolNameOrID || e.PoolID == workerPoolNameOrID {
				if strings.Compare(e.LifeCycle.ActualState, "deleted") != 0 {
					log.Printf("Deleting worker %s", e.ID)
					return workerFields, "deleting", nil
				}
			}
		}
		return workerFields, workerDeleteState, nil
	}
}