Separate cloud/non-cloud tests

.github/workflows/build.yaml

@@ -112,6 +112,15 @@ jobs:
           test_group_name: 'Unit Tests'
           artifact_name: 'unit_test_results'
           result_directory: 'dist/unit_test/'
+      - name: Run functional tests (These do not require cloud resources)
+        run: |
+          # Ensure rad cli is in path before running tests.
+          export PATH=$GITHUB_WORKSPACE/bin:$PATH
+          cd $GITHUB_WORKSPACE
+
+          which rad || { echo "cannot find rad"; exit 1; }
+
+          make test-functional-${{ matrix.name }}
       - name: Copy cli binaries to release (unix-like)
         if: matrix.target_os != 'windows'
         run: |

.github/workflows/functional-test.yaml

@@ -482,7 +482,7 @@ jobs:
       - name: Publish Terraform test recipes
         run: |
           make publish-test-terraform-recipes
-      - name: Run functional tests
+      - name: Run functional tests that require cloud resources
         run: |
           # Ensure rad cli is in path before running tests.
           export PATH=$GITHUB_WORKSPACE/bin:$PATH
@@ -501,7 +501,7 @@ jobs:
           # AZURE_MSSQL_PASSWORD
           eval "export $(echo "${{ secrets.FUNCTEST_PREPROVISIONED_RESOURCE_JSON }}" | jq -r 'to_entries | map("\(.key)=\(.value)") | @sh')"
 
-          make test-functional-${{ matrix.name }}
+          make test-functional-${{ matrix.name }}-cloud
         env:
           DOCKER_REGISTRY: ${{ env.CONTAINER_REGISTRY }}
           TEST_TIMEOUT: ${{ env.FUNCTIONALTEST_TIMEOUT }}

build/test.mk

@@ -66,6 +66,9 @@ test-functional-kubernetes: ## Runs Kubernetes functional tests
 test-functional-shared: ## Runs shared functional tests
 	CGO_ENABLED=1 $(GOTEST_TOOL) ./test/functional/shared/... -timeout ${TEST_TIMEOUT} -v -parallel 10 $(GOTEST_OPTS)
 
+test-functional-shared-cloud: ## Runs shared functional tests that require cloud resources
+	CGO_ENABLED=1 $(GOTEST_TOOL) ./test/functional-cloud/shared/... -timeout ${TEST_TIMEOUT} -v -parallel 10 $(GOTEST_OPTS)
+
 test-functional-msgrp: ## Runs Messaging RP functional tests
 	CGO_ENABLED=1 $(GOTEST_TOOL) ./test/functional/messagingrp/... -timeout ${TEST_TIMEOUT} -v -parallel 2 $(GOTEST_OPTS)
 
@@ -75,12 +78,18 @@ test-functional-daprrp: ## Runs Dapr RP functional tests
 test-functional-datastoresrp: ## Runs Datastores RP functional tests
 	CGO_ENABLED=1 $(GOTEST_TOOL) ./test/functional/datastoresrp/... -timeout ${TEST_TIMEOUT} -v -parallel 3 $(GOTEST_OPTS)
 
+test-functional-datastoresrp-cloud: ## Runs Datastores RP functional tests that require cloud resources
+	CGO_ENABLED=1 $(GOTEST_TOOL) ./test/functional-cloud/datastoresrp/... -timeout ${TEST_TIMEOUT} -v -parallel 3 $(GOTEST_OPTS)
+
 test-functional-samples: ## Runs Samples functional tests
 	CGO_ENABLED=1 $(GOTEST_TOOL) ./test/functional/samples/... -timeout ${TEST_TIMEOUT} -v -parallel 5 $(GOTEST_OPTS)
 
 test-functional-ucp: ## Runs UCP functional tests
 	CGO_ENABLED=1 $(GOTEST_TOOL) ./test/functional/ucp/... -timeout ${TEST_TIMEOUT} -v -parallel 5 $(GOTEST_OPTS)
 
+test-functional-ucp-cloud: ## Runs UCP functional tests that require cloud resources
+	CGO_ENABLED=1 $(GOTEST_TOOL) ./test/functional-cloud/ucp/... -timeout ${TEST_TIMEOUT} -v -parallel 5 $(GOTEST_OPTS)
+
 test-validate-bicep: ## Validates that all .bicep files compile cleanly
 	BICEP_PATH="${HOME}/.rad/bin/rad-bicep" ./build/validate-bicep.sh
 

test/functional-cloud/README.md

@@ -0,0 +1,3 @@
+* [Writing functional tests](../../docs/contributing/contributing-code/contributing-code-tests/writing-functional-tests.md)
+
+* [Running functional tests](../../docs/contributing/contributing-code/contributing-code-tests/running-functional-tests.md)

test/functional-cloud/shared/resources/extender_test.go

@@ -0,0 +1,105 @@
+/*
+Copyright 2023 The Radius Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package resource_test
+
+import (
+	"fmt"
+	"testing"
+
+	"os"
+
+	"github.com/radius-project/radius/test/functional"
+	"github.com/radius-project/radius/test/functional/shared"
+	"github.com/radius-project/radius/test/step"
+	"github.com/radius-project/radius/test/validation"
+)
+
+func Test_Extender_RecipeAWS(t *testing.T) {
+	t.Skip("Skipping until we resolve https://github.com/radius-project/radius/issues/6535")
+	awsAccountID := os.Getenv("AWS_ACCOUNT_ID")
+	awsRegion := os.Getenv("AWS_REGION")
+	// Error the test if the required environment variables are not set
+	// for running locally set the environment variables
+	if awsAccountID == "" || awsRegion == "" {
+		t.Error("This test needs the env variables AWS_ACCOUNT_ID and AWS_REGION to be set")
+	}
+
+	template := "testdata/corerp-resources-extender-aws-s3-recipe.bicep"
+	name := "corerp-resources-extenders-aws-s3-recipe"
+	appName := "corerp-resources-extenders-aws-s3-recipe-app"
+	bucketName := functional.GenerateS3BucketName()
+	bucketID := fmt.Sprintf("/planes/aws/aws/accounts/%s/regions/%s/providers/AWS.S3/Bucket/%s", awsAccountID, awsRegion, bucketName)
+	creationTimestamp := functional.GetCreationTimestamp()
+
+	test := shared.NewRPTest(t, name, []shared.TestStep{
+		{
+			Executor: step.NewDeployExecutor(
+				template,
+				"bucketName="+bucketName,
+				"creationTimestamp="+creationTimestamp,
+				functional.GetAWSAccountId(),
+				functional.GetAWSRegion(),
+				functional.GetBicepRecipeRegistry(),
+				functional.GetBicepRecipeVersion(),
+			),
+			RPResources: &validation.RPResourceSet{
+				Resources: []validation.RPResource{
+					{
+						Name: "corerp-resources-extenders-aws-s3-recipe-env",
+						Type: validation.EnvironmentsResource,
+					},
+					{
+						Name: "corerp-resources-extenders-aws-s3-recipe-app",
+						Type: validation.ApplicationsResource,
+					},
+					{
+						Name: "corerp-resources-extenders-aws-s3-recipe",
+						Type: validation.ExtendersResource,
+						App:  appName,
+						OutputResources: []validation.OutputResourceResponse{
+							{
+								ID: bucketID,
+							},
+						},
+					},
+				},
+			},
+			AWSResources: &validation.AWSResourceSet{
+				Resources: []validation.AWSResource{
+					{
+						Name:       bucketName,
+						Type:       validation.AWSS3BucketResourceType,
+						Identifier: bucketName,
+						Properties: map[string]any{
+							"BucketName": bucketName,
+							"Tags": []any{
+								map[string]any{
+									"Key":   "RadiusCreationTimestamp",
+									"Value": creationTimestamp,
+								},
+							},
+						},
+						SkipDeletion: true, // will be deleted by the recipe
+					},
+				},
+			},
+			SkipObjectValidation: true,
+		},
+	})
+
+	test.Test(t)
+}

test/functional-cloud/shared/resources/recipe_terraform_test.go

@@ -0,0 +1,134 @@
+/*
+Copyright 2023 The Radius Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package resource_test
+
+// This file contains tests for Terraform recipes functionality - covering general behaviors that should
+// be consistent across all resource types. These tests mostly use the extender resource type and mostly
+// avoid cloud resources to avoid unnecessary coupling and reliability issues.
+//
+// Tests in this file should only use cloud resources if absolutely necessary.
+//
+// Tests in this file should be kept *roughly* in sync with recipe_bicep_test and any other drivers.
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/radius-project/radius/pkg/recipes"
+	"github.com/radius-project/radius/pkg/recipes/terraform/config/backends"
+	"github.com/radius-project/radius/test/functional"
+	"github.com/radius-project/radius/test/functional/shared"
+	"github.com/radius-project/radius/test/step"
+	"github.com/radius-project/radius/test/validation"
+)
+
+var (
+	secretNamespace = "radius-system"
+	secretPrefix    = "tfstate-default-"
+)
+
+// Test_TerraformRecipe_Redis covers the following terraform recipe scenario:
+//
+// - Create an extender resource using a Terraform recipe that deploys Redis on Kubernetes.
+// - The recipe deployment creates a Kubernetes deployment and a Kubernetes service.
+
+// Test_TerraformRecipe_AzureStorage creates an Extender resource consuming a Terraform recipe that deploys an Azure blob storage instance.
+func Test_TerraformRecipe_AzureStorage(t *testing.T) {
+	template := "testdata/corerp-resources-terraform-azurestorage.bicep"
+	name := "corerp-resources-terraform-azstorage"
+	appName := "corerp-resources-terraform-azstorage-app"
+	envName := "corerp-resources-terraform-azstorage-env"
+
+	test := shared.NewRPTest(t, name, []shared.TestStep{
+		{
+			Executor: step.NewDeployExecutor(template, functional.GetTerraformRecipeModuleServerURL(), "appName="+appName),
+			RPResources: &validation.RPResourceSet{
+				Resources: []validation.RPResource{
+					{
+						Name: envName,
+						Type: validation.EnvironmentsResource,
+					},
+					{
+						Name: appName,
+						Type: validation.ApplicationsResource,
+					},
+					{
+						Name: name,
+						Type: validation.ExtendersResource,
+						App:  appName,
+					},
+				},
+			},
+			SkipObjectValidation: true,
+			PostStepVerify: func(ctx context.Context, t *testing.T, test shared.RPTest) {
+				resourceID := "/planes/radius/local/resourcegroups/kind-radius/providers/Applications.Core/extenders/" + name
+				secretSuffix, err := getSecretSuffix(resourceID, envName, appName)
+				require.NoError(t, err)
+
+				secret, err := test.Options.K8sClient.CoreV1().Secrets(secretNamespace).
+					Get(ctx, secretPrefix+secretSuffix, metav1.GetOptions{})
+				require.NoError(t, err)
+				require.Equal(t, secretNamespace, secret.Namespace)
+				require.Equal(t, secretPrefix+secretSuffix, secret.Name)
+			},
+		},
+	})
+
+	test.PostDeleteVerify = func(ctx context.Context, t *testing.T, test shared.RPTest) {
+		resourceID := "/planes/radius/local/resourcegroups/kind-radius/providers/Applications.Core/extenders/" + name
+		testSecretDeletion(t, ctx, test, appName, envName, resourceID)
+	}
+
+	test.Test(t)
+}
+
+func testSecretDeletion(t *testing.T, ctx context.Context, test shared.RPTest, appName, envName, resourceID string) {
+	secretSuffix, err := getSecretSuffix(resourceID, envName, appName)
+	require.NoError(t, err)
+
+	secret, err := test.Options.K8sClient.CoreV1().Secrets(secretNamespace).
+		Get(ctx, secretPrefix+secretSuffix, metav1.GetOptions{})
+	require.Error(t, err)
+	require.True(t, apierrors.IsNotFound(err))
+	require.Equal(t, secret, &corev1.Secret{})
+}
+
+func getSecretSuffix(resourceID, envName, appName string) (string, error) {
+	envID := "/planes/radius/local/resourcegroups/kind-radius/providers/Applications.Core/environments/" + envName
+	appID := "/planes/radius/local/resourcegroups/kind-radius/providers/Applications.Core/applications/" + appName
+
+	resourceRecipe := recipes.ResourceMetadata{
+		EnvironmentID: envID,
+		ApplicationID: appID,
+		ResourceID:    resourceID,
+		Parameters:    nil,
+	}
+
+	backend := backends.NewKubernetesBackend(nil)
+	secretMap, err := backend.BuildBackend(&resourceRecipe)
+	if err != nil {
+		return "", err
+	}
+	kubernetes := secretMap["kubernetes"].(map[string]any)
+
+	return kubernetes["secret_suffix"].(string), nil
+}

test/functional-cloud/shared/resources/testdata/corerp-azure-connection-database-service.bicep

@@ -0,0 +1,31 @@
+import radius as radius
+
+param magpieimage string
+
+param environment string
+
+param cosmosmongodbresourceid string
+
+resource app 'Applications.Core/applications@2023-10-01-preview' = {
+  name: 'corerp-azure-connection-database-service'
+  location: 'global'
+  properties: {
+    environment: environment
+  }
+}
+
+resource store 'Applications.Core/containers@2023-10-01-preview' = {
+  name: 'db-service'
+  location: 'global'
+  properties: {
+    application: app.id
+    container: {
+      image: magpieimage
+    }
+    connections: {
+      databaseresource: {
+        source: cosmosmongodbresourceid
+      }
+    }
+  }
+}