updated to plugin 1.10, added acceptance tests for groups and users. …

…Added support for default roles and a data source to fetch the default all_user roles via creating a dummy user. Changed environment variable to include databricks prefix and databricks_azure for the azure components. Restructured storage types to include separate dbfs and s3 list blocks.

.env.template

@@ -1,9 +1,9 @@
-TOKEN=<dbapitoken for int test>
-HOST=<host for integration test>
-ARM_CLIENT_ID=<enterprise app client id>
-ARM_CLIENT_SECRET=<enterprise app secret>
-ARM_TENANT_ID=<active directory tenant id>
-ARM_SUBSCRIPTION_ID=<subscription id in which the workspace is being built>
+DATABRICKS_TOKEN=<dbapitoken for int test>
+DATABRICKS_HOST=<host for integration test>
+DATABRICKS_AZURE_CLIENT_ID=<enterprise app client id>
+DATABRICKS_AZURE_CLIENT_SECRET=<enterprise app secret>
+DATABRICKS_AZURE_TENANT_ID=<active directory tenant id>
+DATABRICKS_AZURE_SUBSCRIPTION_ID=<subscription id in which the workspace is being built>
 TEST_MANAGED_RESOURCE_GROUP=<managed resource group id for integration testing>
 TEST_WORKSPACE_NAME=<workspace name for integration testing>
 TEST_RESOURCE_GROUP=<workspace resource group for integration testing>

Makefile

@@ -56,7 +56,7 @@ vendor:
 # INTEGRATION TESTING WITH TERRAFORM EXAMPLES
 terraform-acc: fmt build
 	@echo "==> Running Terraform Acceptance Tests..."
-	@TF_ACC=1 go test -v -short ./databricks/...
+	@TF_ACC=1 gotestsum --format short-verbose --raw-command go test -v -json -short -coverprofile=coverage.out ./...
 
 terraform-setup: build
 	@echo "==> Initializing Terraform..."
@@ -72,10 +72,7 @@ snapshot:
 
 hugo:
 	@echo "==> Making Docs..."
-	@rm -rf website/content/_index.files/*
-	@cp dist/*.gz website/content/_index.files/
-	@cp dist/*.zip website/content/_index.files/
-	@cd website && hugo
+	@cd website && hugo -d ../docs/
 
 internal-docs-sync:
 	@echo "==> Uploading Website..."

client/model/user.go

@@ -47,6 +47,15 @@ type User struct {
 	InheritedRoles   []RoleListItem         `json:"inherited_roles,omitempty"`
 }
 
+// UserList contains a list of Users fetched from a list api call from SCIM api
+type UserList struct {
+	TotalResults int32  `json:"totalResults,omitempty"`
+	StartIndex   int32  `json:"startIndex,omitempty"`
+	ItemsPerPage int32  `json:"itemsPerPage,omitempty"`
+	Schemas      []URN  `json:"schemas,omitempty"`
+	Resources    []User `json:"resources,omitempty"`
+}
+
 // UserPatchRequest is a struct that contains all the information for a PATCH request to the SCIM users api
 type UserPatchRequest struct {
 	Schemas    []URN                 `json:"schemas,omitempty"`

client/service/groups.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/databrickslabs/databricks-terraform/client/model"
+	"log"
 	"net/http"
 	"sort"
 )
@@ -66,7 +67,7 @@ func (a GroupsAPI) Read(groupID string) (model.Group, error) {
 	if err != nil {
 		return group, err
 	}
-
+	log.Println(group)
 	//get inherited groups
 	var groups []model.Group
 	for _, inheritedGroup := range group.Groups {

client/service/groups_test.go

@@ -33,10 +33,10 @@ func TestScimGroupAPI_Create(t *testing.T) {
 			},
 			args: args{
 				Schemas:      []model.URN{model.GroupSchema},
-				Entitlements: []model.ValueListItem{{Value:"test-entitlement"}},
+				Entitlements: []model.ValueListItem{{Value: "test-entitlement"}},
 				DisplayName:  "test group",
-				Roles:        []model.ValueListItem{{Value:"test-role"}},
-				Members:      []model.ValueListItem{{Value:"test-member"}},
+				Roles:        []model.ValueListItem{{Value: "test-role"}},
+				Members:      []model.ValueListItem{{Value: "test-member"}},
 			},
 			wantErr: false,
 		},
@@ -47,10 +47,10 @@ func TestScimGroupAPI_Create(t *testing.T) {
 			responseStatus: http.StatusBadRequest,
 			args: args{
 				Schemas:      []model.URN{model.GroupSchema},
-				Entitlements: []model.ValueListItem{{Value:"test-entitlement"}},
+				Entitlements: []model.ValueListItem{{Value: "test-entitlement"}},
 				DisplayName:  "test group",
-				Roles:        []model.ValueListItem{{Value:"test-role"}},
-				Members:      []model.ValueListItem{{Value:"test-member"}},
+				Roles:        []model.ValueListItem{{Value: "test-role"}},
+				Members:      []model.ValueListItem{{Value: "test-member"}},
 			},
 			wantErr: true,
 		},

client/service/main_test.go

@@ -42,8 +42,8 @@ func compare(t *testing.T, a interface{}, b interface{}) {
 
 func GetIntegrationDBAPIClient() *DBApiClient {
 	var config DBApiClientConfig
-	config.Token = os.Getenv("TOKEN")
-	config.Host = os.Getenv("HOST")
+	config.Token = os.Getenv("DATABRICKS_TOKEN")
+	config.Host = os.Getenv("DATABRICKS_HOST")
 
 	var c DBApiClient
 	c.SetConfig(&config)

client/service/tokens.go

@@ -2,7 +2,7 @@ package service
 
 import (
 	"encoding/json"
-	"errors"
+	"fmt"
 	"github.com/databrickslabs/databricks-terraform/client/model"
 	"net/http"
 )
@@ -58,7 +58,7 @@ func (a TokensAPI) Read(tokenID string) (model.TokenInfo, error) {
 			return tokenInfoRecord, nil
 		}
 	}
-	return tokenInfo, errors.New("Unable to locate token: " + tokenID)
+	return tokenInfo, fmt.Errorf("Unable to locate token: %s", tokenID)
 }
 
 // Delete will delete the token given a token id

client/service/users.go

@@ -2,10 +2,14 @@ package service
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"github.com/databrickslabs/databricks-terraform/client/model"
+	"log"
 	"net/http"
 	"sort"
+	"strings"
+	"time"
 )
 
 // UsersAPI exposes the scim user API
@@ -182,6 +186,50 @@ func (a UsersAPI) RemoveUserAsAdmin(userID string, adminGroupID string) error {
 	return err
 }
 
+func (a UsersAPI) GetOrCreateDefaultMetaUser(metaUserDisplayName string, metaUserName string, deleteAfterCreate bool) (user model.User, err error) {
+	//var user model.User
+	var users model.UserList
+
+	metaUserQuery := fmt.Sprintf("/preview/scim/v2/Users?filter=displayName+eq+%s", metaUserDisplayName)
+
+	resp, err := a.Client.performQuery(http.MethodGet, metaUserQuery, "2.0", scimHeaders, nil, nil)
+	if err != nil {
+		return user, err
+	}
+	err = json.Unmarshal(resp, &users)
+	if err != nil {
+		return user, err
+	}
+
+	resources := users.Resources
+	if len(resources) == 1 {
+		return resources[0], err
+	} else if len(resources) > 1 {
+		return model.User{}, errors.New("more than one meta user")
+	}
+
+	log.Printf("Meta User not found will create a new meta user with name: %s\n", metaUserDisplayName)
+
+	newCreatedUser, err := a.Create(metaUserName, metaUserDisplayName, nil, nil)
+	if err != nil {
+		if strings.Contains(err.Error(), "already exists") {
+			time.Sleep(time.Second * 1)
+			return a.GetOrCreateDefaultMetaUser(metaUserDisplayName, metaUserName, deleteAfterCreate)
+		}
+		return user, err
+	}
+	if deleteAfterCreate {
+		defer func() {
+			deferErr := a.Delete(newCreatedUser.ID)
+			err = deferErr
+		}()
+	}
+	return newCreatedUser, err
+	//newCreatedUserFullInfo, err := a.Read(newCreatedUser.ID)
+	//return newCreatedUserFullInfo, err
+
+}
+
 func (a UsersAPI) getInheritedAndNonInheritedRoles(user model.User, groups []model.Group) (inherited []model.RoleListItem, unInherited []model.RoleListItem, err error) {
 	allRoles := user.Roles
 	var inheritedRoles []model.RoleListItem

client/service/users_test.go

@@ -37,9 +37,9 @@ func TestScimUserAPI_Create(t *testing.T) {
 			args: args{
 				Schemas:      []model.URN{model.UserSchema},
 				UserName:     "test.user@databricks.com",
-				Entitlements: []model.EntitlementsListItem{{Value:"test-entitlement"}},
+				Entitlements: []model.EntitlementsListItem{{Value: "test-entitlement"}},
 				DisplayName:  "test user",
-				Roles:        []model.RoleListItem{{Value:"test-role"}},
+				Roles:        []model.RoleListItem{{Value: "test-role"}},
 			},
 			wantErr: false,
 		},
@@ -51,9 +51,9 @@ func TestScimUserAPI_Create(t *testing.T) {
 			args: args{
 				Schemas:      []model.URN{model.UserSchema},
 				UserName:     "test.user@databricks.com",
-				Entitlements: []model.EntitlementsListItem{{Value:"test-entitlement"}},
+				Entitlements: []model.EntitlementsListItem{{Value: "test-entitlement"}},
 				DisplayName:  "test user",
-				Roles:        []model.RoleListItem{{Value:"test-role"}},
+				Roles:        []model.RoleListItem{{Value: "test-role"}},
 			},
 			wantErr: true,
 		},
@@ -99,9 +99,9 @@ func TestScimUserAPI_Update(t *testing.T) {
 				&args{
 					Schemas:      []model.URN{model.UserSchema},
 					UserName:     "test.user@databricks.com",
-					Entitlements: []model.EntitlementsListItem{{Value:"test-entitlement"}},
+					Entitlements: []model.EntitlementsListItem{{Value: "test-entitlement"}},
 					DisplayName:  "test user",
-					Roles:        []model.RoleListItem{{Value:"test-role"}},
+					Roles:        []model.RoleListItem{{Value: "test-role"}},
 				},
 			},
 			wantErr: false,
@@ -118,9 +118,9 @@ func TestScimUserAPI_Update(t *testing.T) {
 				&args{
 					Schemas:      []model.URN{model.UserSchema},
 					UserName:     "test.user@databricks.com",
-					Entitlements: []model.EntitlementsListItem{{Value:"test-entitlement"}},
+					Entitlements: []model.EntitlementsListItem{{Value: "test-entitlement"}},
 					DisplayName:  "test user",
-					Roles:        []model.RoleListItem{{Value:"test-role"}},
+					Roles:        []model.RoleListItem{{Value: "test-role"}},
 				},
 			},
 			wantErr: true,

databricks/data_source_databricks_default_user_roles.go

@@ -0,0 +1,41 @@
+package databricks
+
+import (
+	"github.com/databrickslabs/databricks-terraform/client/service"
+	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
+)
+
+func dataSourceDefaultUserRoles() *schema.Resource {
+	return &schema.Resource{
+		Read: func(d *schema.ResourceData, m interface{}) error {
+			client := m.(service.DBApiClient)
+
+			defaultRolesUserName := d.Get("default_username").(string)
+			metaUser, err := client.Users().GetOrCreateDefaultMetaUser(defaultRolesUserName, defaultRolesUserName, true)
+			if err != nil {
+				return err
+			}
+			d.SetId(metaUser.ID)
+			err = d.Set("default_username", metaUser.UserName)
+			if err != nil {
+				return err
+			}
+
+			err = d.Set("roles", getListOfRoles(metaUser.Roles))
+			return err
+		},
+		Schema: map[string]*schema.Schema{
+			"default_username": &schema.Schema{
+				Type:     schema.TypeString,
+				Required: true,
+				ForceNew: true,
+			},
+			"roles": &schema.Schema{
+				Type:     schema.TypeList,
+				Computed: true,
+				Elem:     &schema.Schema{Type: schema.TypeString},
+				ForceNew: true,
+			},
+		},
+	}
+}