Skip to content

Commit

Permalink
Defer client initialization to improve resilience (#767)
Browse files Browse the repository at this point in the history 
* Lazy client initialization and client accessor functions

* Don't fail ConfigureFunc on invalid configuration. Print warnings about future failures.

* Fix credentials validation for acceptance tests.

* Propagate client setup errors

* Document credential interpolation issues
  • Loading branch information
@alexsomesan
alexsomesan committed Feb 28, 2020
1 parent 789b0be commit ea15241
Show file tree
Hide file tree
Showing 59 changed files with 895 additions and 237 deletions.
65 changes: 50 additions & 15 deletions kubernetes/provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
apimachineryschema "k8s.io/apimachinery/pkg/runtime/schema"
kubernetes "k8s.io/client-go/kubernetes"
_ "k8s.io/client-go/plugin/pkg/client/auth"
"k8s.io/client-go/rest"
restclient "k8s.io/client-go/rest"

"k8s.io/client-go/tools/clientcmd"
Expand Down Expand Up @@ -186,9 +187,43 @@ func Provider() terraform.ResourceProvider {
return p
}

type KubeClientsets struct {
MainClientset *kubernetes.Clientset
AggregatorClientset *aggregator.Clientset
type KubeClientsets interface {
MainClientset() (*kubernetes.Clientset, error)
AggregatorClientset() (*aggregator.Clientset, error)
}

type kubeClientsets struct {
config *rest.Config
mainClientset *kubernetes.Clientset
aggregatorClientset *aggregator.Clientset
}

func (k kubeClientsets) MainClientset() (*kubernetes.Clientset, error) {
if k.mainClientset != nil {
return k.mainClientset, nil
}
if k.config != nil {
kc, err := kubernetes.NewForConfig(k.config)
if err != nil {
return nil, fmt.Errorf("Failed to configure client: %s", err)
}
k.mainClientset = kc
}
return k.mainClientset, nil
}

func (k kubeClientsets) AggregatorClientset() (*aggregator.Clientset, error) {
if k.aggregatorClientset != nil {
return k.aggregatorClientset, nil
}
if k.config != nil {
ac, err := aggregator.NewForConfig(k.config)
if err != nil {
return nil, fmt.Errorf("Failed to configure client: %s", err)
}
k.aggregatorClientset = ac
}
return k.aggregatorClientset, nil
}

func providerConfigure(d *schema.ResourceData, terraformVersion string) (interface{}, error) {
Expand All @@ -199,7 +234,11 @@ func providerConfigure(d *schema.ResourceData, terraformVersion string) (interfa
return nil, err
}
if cfg == nil {
return nil, fmt.Errorf("Failed to initialize config")
// This is a TEMPORARY measure to work around https://github.com/hashicorp/terraform/issues/24055
// IMPORTANT: this will NOT enable a workaround of issue: https://github.com/hashicorp/terraform/issues/4149
// IMPORTANT: if the supplied configuration is incomplete or invalid
///IMPORTANT: provider operations will fail or attempt to connect to localhost endpoints
cfg = &restclient.Config{}
}

cfg.UserAgent = fmt.Sprintf("HashiCorp/1.0 Terraform/%s", terraformVersion)
Expand All @@ -211,17 +250,12 @@ func providerConfigure(d *schema.ResourceData, terraformVersion string) (interfa
}
}

k, err := kubernetes.NewForConfig(cfg)
if err != nil {
return nil, fmt.Errorf("Failed to configure: %s", err)
m := kubeClientsets{
config: cfg,
mainClientset: nil,
aggregatorClientset: nil,
}

a, err := aggregator.NewForConfig(cfg)
if err != nil {
return nil, fmt.Errorf("Failed to configure: %s", err)
}

return &KubeClientsets{k, a}, nil
return m, nil
}

func initializeConfiguration(d *schema.ResourceData) (*restclient.Config, error) {
Expand Down Expand Up @@ -321,7 +355,8 @@ func initializeConfiguration(d *schema.ResourceData) (*restclient.Config, error)
cc := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loader, overrides)
cfg, err := cc.ClientConfig()
if err != nil {
return nil, fmt.Errorf("Failed to initialize config: %s", err)
log.Printf("[WARN] Invalid provider configuration was supplied. Provider operations likely to fail.")
return nil, nil
}

log.Printf("[INFO] Successfully initialized config")
Expand Down
20 changes: 13 additions & 7 deletions kubernetes/provider_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,12 +158,11 @@ func testAccPreCheck(t *testing.T) {
os.Getenv("KUBE_CTX") != "" ||
os.Getenv("KUBECONFIG") != "" ||
os.Getenv("KUBE_CONFIG") != ""
hasUserCredentials := os.Getenv("KUBE_USER") != "" && os.Getenv("KUBE_PASSWORD") != ""
hasClientCert := os.Getenv("KUBE_CLIENT_CERT_DATA") != "" && os.Getenv("KUBE_CLIENT_KEY_DATA") != ""
hasStaticCfg := (os.Getenv("KUBE_HOST") != "" &&
os.Getenv("KUBE_USER") != "" &&
os.Getenv("KUBE_PASSWORD") != "" &&
os.Getenv("KUBE_CLIENT_CERT_DATA") != "" &&
os.Getenv("KUBE_CLIENT_KEY_DATA") != "" &&
os.Getenv("KUBE_CLUSTER_CA_CERT_DATA") != "")
os.Getenv("KUBE_CLUSTER_CA_CERT_DATA") != "") &&
(hasUserCredentials || hasClientCert || os.Getenv("KUBE_TOKEN") != "")

if !hasFileCfg && !hasStaticCfg {
t.Fatalf("File config (KUBE_CTX_AUTH_INFO and KUBE_CTX_CLUSTER) or static configuration"+
Expand Down Expand Up @@ -205,7 +204,10 @@ func getClusterVersion() (*gversion.Version, error) {
return nil, fmt.Errorf("Provider not initialized, unable to check cluster version")
}

conn := meta.(*KubeClientsets).MainClientset
conn, err := meta.(KubeClientsets).MainClientset()
if err != nil {
return nil, err
}
serverVersion, err := conn.ServerVersion()

if err != nil {
Expand Down Expand Up @@ -306,7 +308,11 @@ func getFirstNode() (api.Node, error) {
if meta == nil {
return api.Node{}, errors.New("Provider not initialized, unable to get cluster node")
}
conn := meta.(*KubeClientsets).MainClientset
conn, err := meta.(KubeClientsets).MainClientset()
if err != nil {
return api.Node{}, err
}

resp, err := conn.CoreV1().Nodes().List(metav1.ListOptions{})
if err != nil {
return api.Node{}, err
Expand Down
29 changes: 22 additions & 7 deletions kubernetes/resource_kubernetes_api_service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,10 @@ func resourceKubernetesAPIService() *schema.Resource {
}

func resourceKubernetesAPIServiceCreate(d *schema.ResourceData, meta interface{}) error {
conn := meta.(*KubeClientsets).AggregatorClientset
conn, err := meta.(KubeClientsets).AggregatorClientset()
if err != nil {
return err
}

metadata := expandMetadata(d.Get("metadata").([]interface{}))
svc := v1.APIService{
Expand All @@ -120,7 +123,10 @@ func resourceKubernetesAPIServiceCreate(d *schema.ResourceData, meta interface{}
}

func resourceKubernetesAPIServiceRead(d *schema.ResourceData, meta interface{}) error {
conn := meta.(*KubeClientsets).AggregatorClientset
conn, err := meta.(KubeClientsets).AggregatorClientset()
if err != nil {
return err
}

name := d.Id()

Expand All @@ -147,7 +153,10 @@ func resourceKubernetesAPIServiceRead(d *schema.ResourceData, meta interface{})
}

func resourceKubernetesAPIServiceUpdate(d *schema.ResourceData, meta interface{}) error {
conn := meta.(*KubeClientsets).AggregatorClientset
conn, err := meta.(KubeClientsets).AggregatorClientset()
if err != nil {
return err
}

name := d.Id()

Expand All @@ -174,12 +183,15 @@ func resourceKubernetesAPIServiceUpdate(d *schema.ResourceData, meta interface{}
}

func resourceKubernetesAPIServiceDelete(d *schema.ResourceData, meta interface{}) error {
conn := meta.(*KubeClientsets).AggregatorClientset
conn, err := meta.(KubeClientsets).AggregatorClientset()
if err != nil {
return err
}

name := d.Id()

log.Printf("[INFO] Deleting API service: %#v", name)
err := conn.ApiregistrationV1().APIServices().Delete(name, &meta_v1.DeleteOptions{})
err = conn.ApiregistrationV1().APIServices().Delete(name, &meta_v1.DeleteOptions{})
if err != nil {
return err
}
Expand All @@ -191,12 +203,15 @@ func resourceKubernetesAPIServiceDelete(d *schema.ResourceData, meta interface{}
}

func resourceKubernetesAPIServiceExists(d *schema.ResourceData, meta interface{}) (bool, error) {
conn := meta.(*KubeClientsets).AggregatorClientset
conn, err := meta.(KubeClientsets).AggregatorClientset()
if err != nil {
return false, err
}

name := d.Id()

log.Printf("[INFO] Checking API service %s", name)
_, err := conn.ApiregistrationV1().APIServices().Get(name, meta_v1.GetOptions{})
_, err = conn.ApiregistrationV1().APIServices().Get(name, meta_v1.GetOptions{})
if err != nil {
if statusErr, ok := err.(*errors.StatusError); ok && statusErr.ErrStatus.Code == 404 {
return false, nil
Expand Down
12 changes: 9 additions & 3 deletions kubernetes/resource_kubernetes_api_service_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,10 @@ func TestAccKubernetesAPIService_importBasic(t *testing.T) {
}

func testAccCheckKubernetesAPIServiceDestroy(s *terraform.State) error {
conn := testAccProvider.Meta().(*KubeClientsets).AggregatorClientset
conn, err := testAccProvider.Meta().(KubeClientsets).AggregatorClientset()
if err != nil {
return err
}

for _, rs := range s.RootModule().Resources {
if rs.Type != "kubernetes_api_service" {
Expand All @@ -163,11 +166,14 @@ func testAccCheckKubernetesAPIServiceExists(n string) resource.TestCheckFunc {
return fmt.Errorf("Not found: %s", n)
}

conn := testAccProvider.Meta().(*KubeClientsets).AggregatorClientset
conn, err := testAccProvider.Meta().(KubeClientsets).AggregatorClientset()
if err != nil {
return err
}

name := rs.Primary.ID

_, err := conn.ApiregistrationV1().APIServices().Get(name, meta_v1.GetOptions{})
_, err = conn.ApiregistrationV1().APIServices().Get(name, meta_v1.GetOptions{})
if err != nil {
return err
}
Expand Down
29 changes: 22 additions & 7 deletions kubernetes/resource_kubernetes_cluster_role.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,10 @@ func resourceKubernetesClusterRole() *schema.Resource {
}

func resourceKubernetesClusterRoleCreate(d *schema.ResourceData, meta interface{}) error {
conn := meta.(*KubeClientsets).MainClientset
conn, err := meta.(KubeClientsets).MainClientset()
if err != nil {
return err
}

metadata := expandMetadata(d.Get("metadata").([]interface{}))
cRole := api.ClusterRole{
Expand All @@ -57,7 +60,10 @@ func resourceKubernetesClusterRoleCreate(d *schema.ResourceData, meta interface{
}

func resourceKubernetesClusterRoleUpdate(d *schema.ResourceData, meta interface{}) error {
conn := meta.(*KubeClientsets).MainClientset
conn, err := meta.(KubeClientsets).MainClientset()
if err != nil {
return err
}

name := d.Id()
ops := patchMetadata("metadata.0.", "/metadata/", d)
Expand All @@ -81,7 +87,10 @@ func resourceKubernetesClusterRoleUpdate(d *schema.ResourceData, meta interface{
}

func resourceKubernetesClusterRoleRead(d *schema.ResourceData, meta interface{}) error {
conn := meta.(*KubeClientsets).MainClientset
conn, err := meta.(KubeClientsets).MainClientset()
if err != nil {
return err
}

name := d.Id()
log.Printf("[INFO] Reading cluster role %s", name)
Expand All @@ -105,11 +114,14 @@ func resourceKubernetesClusterRoleRead(d *schema.ResourceData, meta interface{})
}

func resourceKubernetesClusterRoleDelete(d *schema.ResourceData, meta interface{}) error {
conn := meta.(*KubeClientsets).MainClientset
conn, err := meta.(KubeClientsets).MainClientset()
if err != nil {
return err
}

name := d.Id()
log.Printf("[INFO] Deleting cluster role: %#v", name)
err := conn.RbacV1().ClusterRoles().Delete(name, &metav1.DeleteOptions{})
err = conn.RbacV1().ClusterRoles().Delete(name, &metav1.DeleteOptions{})
if err != nil {
return err
}
Expand All @@ -120,11 +132,14 @@ func resourceKubernetesClusterRoleDelete(d *schema.ResourceData, meta interface{
}

func resourceKubernetesClusterRoleExists(d *schema.ResourceData, meta interface{}) (bool, error) {
conn := meta.(*KubeClientsets).MainClientset
conn, err := meta.(KubeClientsets).MainClientset()
if err != nil {
return false, err
}

name := d.Id()
log.Printf("[INFO] Checking cluster role %s", name)
_, err := conn.RbacV1().ClusterRoles().Get(name, metav1.GetOptions{})
_, err = conn.RbacV1().ClusterRoles().Get(name, metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
return false, nil
Expand Down
31 changes: 23 additions & 8 deletions kubernetes/resource_kubernetes_cluster_role_binding.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,10 @@ func resourceKubernetesClusterRoleBinding() *schema.Resource {
}

func resourceKubernetesClusterRoleBindingCreate(d *schema.ResourceData, meta interface{}) error {
conn := meta.(*KubeClientsets).MainClientset
conn, err := meta.(KubeClientsets).MainClientset()
if err != nil {
return err
}

metadata := expandMetadata(d.Get("metadata").([]interface{}))
binding := &api.ClusterRoleBinding{
Expand All @@ -57,7 +60,7 @@ func resourceKubernetesClusterRoleBindingCreate(d *schema.ResourceData, meta int
Subjects: expandRBACSubjects(d.Get("subject").([]interface{})),
}
log.Printf("[INFO] Creating new ClusterRoleBinding: %#v", binding)
binding, err := conn.RbacV1().ClusterRoleBindings().Create(binding)
binding, err = conn.RbacV1().ClusterRoleBindings().Create(binding)

if err != nil {
return err
Expand All @@ -69,7 +72,10 @@ func resourceKubernetesClusterRoleBindingCreate(d *schema.ResourceData, meta int
}

func resourceKubernetesClusterRoleBindingRead(d *schema.ResourceData, meta interface{}) error {
conn := meta.(*KubeClientsets).MainClientset
conn, err := meta.(KubeClientsets).MainClientset()
if err != nil {
return err
}

name := d.Id()
log.Printf("[INFO] Reading ClusterRoleBinding %s", name)
Expand Down Expand Up @@ -103,7 +109,10 @@ func resourceKubernetesClusterRoleBindingRead(d *schema.ResourceData, meta inter
}

func resourceKubernetesClusterRoleBindingUpdate(d *schema.ResourceData, meta interface{}) error {
conn := meta.(*KubeClientsets).MainClientset
conn, err := meta.(KubeClientsets).MainClientset()
if err != nil {
return err
}

name := d.Id()

Expand All @@ -128,11 +137,14 @@ func resourceKubernetesClusterRoleBindingUpdate(d *schema.ResourceData, meta int
}

func resourceKubernetesClusterRoleBindingDelete(d *schema.ResourceData, meta interface{}) error {
conn := meta.(*KubeClientsets).MainClientset
conn, err := meta.(KubeClientsets).MainClientset()
if err != nil {
return err
}

name := d.Id()
log.Printf("[INFO] Deleting ClusterRoleBinding: %#v", name)
err := conn.RbacV1().ClusterRoleBindings().Delete(name, &meta_v1.DeleteOptions{})
err = conn.RbacV1().ClusterRoleBindings().Delete(name, &meta_v1.DeleteOptions{})
if err != nil {
return err
}
Expand All @@ -143,11 +155,14 @@ func resourceKubernetesClusterRoleBindingDelete(d *schema.ResourceData, meta int
}

func resourceKubernetesClusterRoleBindingExists(d *schema.ResourceData, meta interface{}) (bool, error) {
conn := meta.(*KubeClientsets).MainClientset
conn, err := meta.(KubeClientsets).MainClientset()
if err != nil {
return false, err
}

name := d.Id()
log.Printf("[INFO] Checking ClusterRoleBinding %s", name)
_, err := conn.RbacV1().ClusterRoleBindings().Get(name, meta_v1.GetOptions{})
_, err = conn.RbacV1().ClusterRoleBindings().Get(name, meta_v1.GetOptions{})
if err != nil {
if statusErr, ok := err.(*errors.StatusError); ok && statusErr.ErrStatus.Code == 404 {
return false, nil
Expand Down

0 comments on commit ea15241

Please sign in to comment.