implement GCP passthrough mode (and add service API checks)

- [x] handle the case when the root creds are in passthrough mode.
- [x] add checks for passthrough and mint mode where we check that the service APIs are enabled/available before proceeding to process the CredentialsRequest.

Note that passthrough mode is handled differently than in AWS. In AWS we work with a static list of permissions that are needed for the cluster to run with. For GCP passthrough mode, the decision on whether we can satisfy a CredentialsRequest is dynamic. All we absolutely need for passthrough mode is the ability to list service APIs (to determine whether any particular service API is enabled), the ability to get the details for a specific role (so we can determine whether the role exists and the permissions attached to that role), and the ability to get the details of a project (so we can get the project number for a given project name). All the other permissions attached to the root creds in passthrough mode would pass/fail a permissions check during the TestIamPermissions() call.

pkg/controller/credentialsrequest/credentialsrequest_controller.go

@@ -398,6 +398,8 @@ func (r *ReconcileCredentialsRequest) Reconcile(request reconcile.Request) (reco
 	}
 	if syncErr != nil {
 		logger.Errorf("error syncing credentials: %v", syncErr)
+		// TODO: set condition if previously satisfied credrequest can now
+		// not be satisfied (but keeping provisioned==True).
 		cr.Status.Provisioned = false
 
 		switch t := syncErr.(type) {

pkg/controller/credentialsrequest/credentialsrequest_controller_gcp_test.go

@@ -53,13 +53,19 @@ import (
 const (
 	testRootGCPAuth                  = "ROOTAUTH"
 	testServiceAccountKeyPrivateData = "SECRET SERVICE ACCOUNT KEY DATA"
+	testOldPassthroughPrivateData    = "OLD SERVICE ACCOUNT KEY DATA"
 	testGCPServiceAccountID          = "a-test-svc-acct"
 	testRoleName                     = "roles/appengine.appAdmin"
+	testServiceAPIName               = "appengine.googleapis.com"
 	testGCPProjectName               = "test-GCP-project"
 	testServiceAccountKeyName        = "testGCPKeyName"
 )
 
 var (
+	testRolePermissions = []string{
+		"appengine.applications.get",
+	}
+
 	emptyPolicyBindings = []*cloudresourcemanager.Binding{}
 
 	testValidPolicyBindings = []*cloudresourcemanager.Binding{
@@ -88,11 +94,12 @@ func TestCredentialsRequestGCPReconcile(t *testing.T) {
 	}
 
 	tests := []struct {
-		name              string
-		existing          []runtime.Object
-		expectErr         bool
-		mockRootGCPClient func(mockCtrl *gomock.Controller) *mockgcp.MockClient
-		validate          func(client.Client, *testing.T)
+		name                        string
+		existing                    []runtime.Object
+		expectErr                   bool
+		mockRootGCPClient           func(mockCtrl *gomock.Controller) *mockgcp.MockClient
+		mockCredRequestSecretClient func(mockCtrl *gomock.Controller) *mockgcp.MockClient
+		validate                    func(client.Client, *testing.T)
 		// Expected conditions on the credentials request:
 		expectedConditions []ExpectedCondition
 		// Expected conditions on the credentials cluster operator:
@@ -111,9 +118,16 @@ func TestCredentialsRequestGCPReconcile(t *testing.T) {
 			mockRootGCPClient: func(mockCtrl *gomock.Controller) *mockgcp.MockClient {
 				mockGCPClient := mockgcp.NewMockClient(mockCtrl)
 				mockGetProjectName(mockGCPClient)
+
+				// needsupdate
+				mockGetRole(mockGCPClient)
+				mockListServicesEnabled(mockGCPClient)
+
+				// create service account
+				mockGetRole(mockGCPClient)
+				mockListServicesEnabled(mockGCPClient)
 				mockGetServiceAccount(mockGCPClient)
 				mockGetProjectIamPolicy(mockGCPClient, nil)
-				mockGetRole(mockGCPClient)
 				mockSetProjectIamPolicy(mockGCPClient)
 				mockListServiceAccountKeysEmpty(mockGCPClient)
 				mockCreateServiceAccountKey(mockGCPClient, "")
@@ -159,9 +173,16 @@ func TestCredentialsRequestGCPReconcile(t *testing.T) {
 			mockRootGCPClient: func(mockCtrl *gomock.Controller) *mockgcp.MockClient {
 				mockGCPClient := mockgcp.NewMockClient(mockCtrl)
 				mockGetProjectName(mockGCPClient)
+
+				// needs update
+				mockGetRole(mockGCPClient)
+				mockListServicesEnabled(mockGCPClient)
+
+				// create serviceaccount
+				mockGetRole(mockGCPClient)
+				mockListServicesEnabled(mockGCPClient)
 				mockGetServiceAccount(mockGCPClient)
 				mockGetProjectIamPolicy(mockGCPClient, nil)
-				mockGetRole(mockGCPClient)
 				mockSetProjectIamPolicy(mockGCPClient)
 				mockListServiceAccountKeysEmpty(mockGCPClient)
 				mockCreateServiceAccountKey(mockGCPClient, "")
@@ -238,9 +259,16 @@ func TestCredentialsRequestGCPReconcile(t *testing.T) {
 			mockRootGCPClient: func(mockCtrl *gomock.Controller) *mockgcp.MockClient {
 				mockGCPClient := mockgcp.NewMockClient(mockCtrl)
 				mockGetProjectName(mockGCPClient)
+
+				// needs update
+				mockGetRole(mockGCPClient)
+				mockListServicesEnabled(mockGCPClient)
+
+				// new service account key
+				mockGetRole(mockGCPClient)
+				mockListServicesEnabled(mockGCPClient)
 				mockGetServiceAccount(mockGCPClient)
 				mockGetProjectIamPolicy(mockGCPClient, testValidPolicyBindings)
-				mockGetRole(mockGCPClient)
 				mockListServiceAccountKeys(mockGCPClient, testServiceAccountKeyName)
 				mockDeleteServiceAccountKey(mockGCPClient, testServiceAccountKeyName)
 				mockCreateServiceAccountKey(mockGCPClient, "NEW PRIVATE DATA")
@@ -272,9 +300,16 @@ func TestCredentialsRequestGCPReconcile(t *testing.T) {
 			mockRootGCPClient: func(mockCtrl *gomock.Controller) *mockgcp.MockClient {
 				mockGCPClient := mockgcp.NewMockClient(mockCtrl)
 				mockGetProjectName(mockGCPClient)
+
+				// needs update
+				mockGetRole(mockGCPClient)
+				mockListServicesEnabled(mockGCPClient)
+
+				// create service account key
+				mockGetRole(mockGCPClient)
+				mockListServicesEnabled(mockGCPClient)
 				mockGetServiceAccount(mockGCPClient)
 				mockGetProjectIamPolicy(mockGCPClient, testValidPolicyBindings)
-				mockGetRole(mockGCPClient)
 				mockListServiceAccountKeysEmpty(mockGCPClient)
 				mockCreateServiceAccountKey(mockGCPClient, "NEW AUTH KEY DATA")
 
@@ -329,6 +364,14 @@ func TestCredentialsRequestGCPReconcile(t *testing.T) {
 			mockRootGCPClient: func(mockCtrl *gomock.Controller) *mockgcp.MockClient {
 				mockGCPClient := mockgcp.NewMockClient(mockCtrl)
 				mockGetProjectName(mockGCPClient)
+
+				// needs update
+				mockGetRole(mockGCPClient)
+				mockListServicesEnabled(mockGCPClient)
+
+				// create service account
+				mockGetRole(mockGCPClient)
+				mockListServicesEnabled(mockGCPClient)
 				mockGetServiceAccountFailed(mockGCPClient)
 
 				return mockGCPClient
@@ -367,6 +410,157 @@ func TestCredentialsRequestGCPReconcile(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "new cred passthrough",
+			existing: []runtime.Object{
+				createTestNamespace(testSecretNamespace),
+				testGCPCredentialsRequest(t),
+				testGCPCredsSecretPassthrough("kube-system", gcpconst.GCPCloudCredSecretName, testRootGCPAuth),
+				testClusterVersion(),
+				testInfrastructure(testInfraName),
+			},
+			mockRootGCPClient: func(mockCtrl *gomock.Controller) *mockgcp.MockClient {
+				mockGCPClient := mockgcp.NewMockClient(mockCtrl)
+				mockGetProjectName(mockGCPClient)
+
+				mockGetRole(mockGCPClient)
+				mockListServicesEnabled(mockGCPClient)
+
+				mockGetRole(mockGCPClient)
+				mockTestIamPermissions(mockGCPClient)
+
+				return mockGCPClient
+			},
+			validate: func(c client.Client, t *testing.T) {
+				targetSecret := getCredRequestTargetSecret(c)
+				if assert.NotNil(t, targetSecret, "expected non-empty target secret to exist") {
+					assert.Equal(t, testRootGCPAuth, string(targetSecret.Data[gcpconst.GCPAuthJSONKey]))
+				}
+				cr := getCredRequest(c)
+				assert.NotNil(t, cr)
+				assert.True(t, cr.Status.Provisioned)
+				assert.Equal(t, int64(testCRGeneration), int64(cr.Status.LastSyncGeneration))
+				assert.NotNil(t, cr.Status.LastSyncTimestamp)
+			},
+		},
+		{
+			name: "new cred passthrough fail permissions",
+			existing: []runtime.Object{
+				createTestNamespace(testSecretNamespace),
+				testGCPCredentialsRequest(t),
+				testGCPCredsSecretPassthrough("kube-system", gcpconst.GCPCloudCredSecretName, testRootGCPAuth),
+				testClusterVersion(),
+				testInfrastructure(testInfraName),
+			},
+			mockRootGCPClient: func(mockCtrl *gomock.Controller) *mockgcp.MockClient {
+				mockGCPClient := mockgcp.NewMockClient(mockCtrl)
+				mockGetProjectName(mockGCPClient)
+
+				mockGetRole(mockGCPClient)
+				mockListServicesEnabled(mockGCPClient)
+
+				mockGetRole(mockGCPClient)
+				mockTestIamPermissionsFail(mockGCPClient)
+
+				return mockGCPClient
+			},
+			expectErr: true,
+			validate: func(c client.Client, t *testing.T) {
+				targetSecret := getCredRequestTargetSecret(c)
+				assert.Nil(t, targetSecret)
+				cr := getCredRequest(c)
+				assert.False(t, cr.Status.Provisioned)
+			},
+			expectedConditions: []ExpectedCondition{
+				{
+					conditionType: minterv1.CredentialsProvisionFailure,
+					reason:        "CredentialsProvisionFailure",
+					status:        corev1.ConditionTrue,
+				},
+			},
+		},
+		{
+			name: "existing cr secret enough passthrough perms",
+			existing: []runtime.Object{
+				createTestNamespace(testSecretNamespace),
+				testGCPPassthroughCredentialsRequest(t),
+				testGCPCredsSecretPassthrough("kube-system", gcpconst.GCPCloudCredSecretName, testRootGCPAuth),
+				testGCPCredsSecret(testSecretNamespace, testSecretName, testOldPassthroughPrivateData),
+				testClusterVersion(),
+				testInfrastructure(testInfraName),
+			},
+			mockRootGCPClient: func(mockCtrl *gomock.Controller) *mockgcp.MockClient {
+				mockGCPClient := mockgcp.NewMockClient(mockCtrl)
+				mockGetProjectName(mockGCPClient)
+
+				mockGetRole(mockGCPClient)
+				mockListServicesEnabled(mockGCPClient)
+
+				return mockGCPClient
+			},
+			mockCredRequestSecretClient: func(mockCtrl *gomock.Controller) *mockgcp.MockClient {
+				mockGCPClient := mockgcp.NewMockClient(mockCtrl)
+				mockGetProjectName(mockGCPClient)
+
+				mockTestIamPermissions(mockGCPClient)
+				return mockGCPClient
+			},
+			validate: func(c client.Client, t *testing.T) {
+				targetSecret := getCredRequestTargetSecret(c)
+				if assert.NotNil(t, targetSecret, "expected non-empty target secret to exist") {
+					assert.Equal(t, testOldPassthroughPrivateData, string(targetSecret.Data[gcpconst.GCPAuthJSONKey]))
+				}
+				cr := getCredRequest(c)
+				assert.NotNil(t, cr)
+				assert.True(t, cr.Status.Provisioned)
+				assert.Equal(t, int64(testCRGeneration), int64(cr.Status.LastSyncGeneration))
+				assert.NotNil(t, cr.Status.LastSyncTimestamp)
+			},
+		},
+		{
+			name: "existing secret not enough passthrough perms",
+			existing: []runtime.Object{
+				createTestNamespace(testSecretNamespace),
+				testGCPPassthroughCredentialsRequest(t),
+				testGCPCredsSecretPassthrough("kube-system", gcpconst.GCPCloudCredSecretName, testRootGCPAuth),
+				testGCPCredsSecret(testSecretNamespace, testSecretName, testOldPassthroughPrivateData),
+				testClusterVersion(),
+				testInfrastructure(testInfraName),
+			},
+			mockRootGCPClient: func(mockCtrl *gomock.Controller) *mockgcp.MockClient {
+				mockGCPClient := mockgcp.NewMockClient(mockCtrl)
+				mockGetProjectName(mockGCPClient)
+
+				// needs update
+				mockGetRole(mockGCPClient)
+				mockListServicesEnabled(mockGCPClient)
+
+				// sync passthrough
+				mockGetRole(mockGCPClient)
+				mockTestIamPermissionsFail(mockGCPClient)
+
+				return mockGCPClient
+			},
+			expectErr: true,
+			mockCredRequestSecretClient: func(mockCtrl *gomock.Controller) *mockgcp.MockClient {
+				mockGCPClient := mockgcp.NewMockClient(mockCtrl)
+				mockGetProjectName(mockGCPClient)
+
+				mockTestIamPermissionsFail(mockGCPClient)
+				return mockGCPClient
+			},
+			validate: func(c client.Client, t *testing.T) {
+				targetSecret := getCredRequestTargetSecret(c)
+				if assert.NotNil(t, targetSecret, "expected non-empty target secret to exist") {
+					// existing secret has old/unchanged content
+					assert.Equal(t, testOldPassthroughPrivateData, string(targetSecret.Data[gcpconst.GCPAuthJSONKey]))
+				}
+				cr := getCredRequest(c)
+				assert.NotNil(t, cr)
+				// TODO: modify CCO to leave provisoned==true when previously successfully provisioned.
+				assert.False(t, cr.Status.Provisioned)
+			},
+		},
 	}
 
 	for _, test := range tests {
@@ -376,14 +570,23 @@ func TestCredentialsRequestGCPReconcile(t *testing.T) {
 
 			mockRootGCPClient := test.mockRootGCPClient(mockCtrl)
 
+			mockSecretClient := mockgcp.NewMockClient(mockCtrl)
+			if test.mockCredRequestSecretClient != nil {
+				mockSecretClient = test.mockCredRequestSecretClient(mockCtrl)
+			}
+
 			fakeClient := fake.NewFakeClient(test.existing...)
 			rcr := &ReconcileCredentialsRequest{
 				Client: fakeClient,
 				Actuator: &actuator.Actuator{
 					Client: fakeClient,
 					Codec:  codec,
 					GCPClientBuilder: func(jsonAUTH []byte) (mintergcp.Client, error) {
-						return mockRootGCPClient, nil
+						if string(jsonAUTH) == testRootGCPAuth {
+							return mockRootGCPClient, nil
+						} else {
+							return mockSecretClient, nil
+						}
 					},
 				},
 			}
@@ -497,6 +700,12 @@ func testGCPPassthroughCredentialsRequest(t *testing.T) *minterv1.CredentialsReq
 	}
 }
 
+func testGCPCredsSecretPassthrough(namespace, name, jsonAUTH string) *corev1.Secret {
+	s := testGCPCredsSecret(namespace, name, jsonAUTH)
+	s.Annotations[annotatorconst.AnnotationKey] = annotatorconst.PassthroughAnnotation
+	return s
+}
+
 func testGCPCredsSecret(namespace, name, jsonAUTH string) *corev1.Secret {
 	s := &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
@@ -507,7 +716,7 @@ func testGCPCredsSecret(namespace, name, jsonAUTH string) *corev1.Secret {
 			},
 		},
 		Data: map[string][]byte{
-			gcpconst.GCPAuthJSONKey: []byte(testRootGCPAuth),
+			gcpconst.GCPAuthJSONKey: []byte(jsonAUTH),
 		},
 	}
 	return s
@@ -540,7 +749,26 @@ func mockGetProjectIamPolicy(mockGCPClient *mockgcp.MockClient, bindings []*clou
 
 func mockGetRole(mockGCPClient *mockgcp.MockClient) {
 	mockGCPClient.EXPECT().GetRole(gomock.Any(), gomock.Any()).Return(&iamadminpb.Role{
-		Name: testRoleName,
+		Name:                testRoleName,
+		IncludedPermissions: testRolePermissions,
+	}, nil)
+}
+
+func mockListServicesEnabled(mockGCPClient *mockgcp.MockClient) {
+	mockGCPClient.EXPECT().ListServicesEnabled().Return(map[string]bool{
+		testServiceAPIName: true,
+	}, nil)
+}
+
+func mockTestIamPermissions(mockGCPClient *mockgcp.MockClient) {
+	mockGCPClient.EXPECT().TestIamPermissions(gomock.Any(), gomock.Any()).Return(&cloudresourcemanager.TestIamPermissionsResponse{
+		Permissions: testRolePermissions,
+	}, nil)
+}
+
+func mockTestIamPermissionsFail(mockGCPClient *mockgcp.MockClient) {
+	mockGCPClient.EXPECT().TestIamPermissions(gomock.Any(), gomock.Any()).Return(&cloudresourcemanager.TestIamPermissionsResponse{
+		Permissions: []string{"not.expected.permission"},
 	}, nil)
 }