[mq] working branch - merge dacbe46 on top of main at c438dc2

{"baseBranch":"main","baseCommit":"c438dc206a70a0d75469656ce66076985672e457","createdAt":"2024-05-28T13:31:45.282110Z","headSha":"dacbe46f94bda94ca5d43e9dbc93a7bc6ae28fa0","id":"36b397b0-20d6-4d4f-80b9-7f193de200f4","priority":"200","pullRequestNumber":"23153","queuedAt":"2024-05-28T13:31:45.186389Z","status":"STATUS_QUEUED","triggeredBy":"kevin.fairise@datadoghq.com"}

.gitlab/common/test_infra_version.yml

@@ -4,4 +4,4 @@ variables:
   # and check the job creating the image to make sure you have the right SHA prefix
   TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: ""
   # Make sure to update test-infra-definitions version in go.mod as well
-  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 65fbab1e1eaf
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 63d1aae5f639

test/new-e2e/examples/kind_local_test.go

@@ -0,0 +1,39 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016-present Datadog, Inc.
+
+package examples
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e"
+	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments"
+	localkubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/local/kubernetes"
+	"github.com/stretchr/testify/assert"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type myLocalKindSuite struct {
+	e2e.BaseSuite[environments.Kubernetes]
+}
+
+func TestMyLocalKindSuite(t *testing.T) {
+	e2e.Run(t, &myLocalKindSuite{}, e2e.WithProvisioner(localkubernetes.Provisioner()))
+}
+
+func (v *myLocalKindSuite) TestClusterAgentInstalled() {
+	res, _ := v.Env().KubernetesCluster.Client().CoreV1().Pods("datadog").List(context.TODO(), v1.ListOptions{})
+	containsClusterAgent := false
+	for _, pod := range res.Items {
+		if strings.Contains(pod.Name, "cluster-agent") {
+			containsClusterAgent = true
+			break
+		}
+	}
+	assert.True(v.T(), containsClusterAgent, "Cluster Agent not found")
+	assert.Equal(v.T(), v.Env().Agent.InstallNameLinux, "dda-linux")
+}

test/new-e2e/examples/kind_test.go

@@ -32,7 +32,7 @@ func TestMyKindSuite(t *testing.T) {
 		awskubernetes.KindProvisioner(
 			awskubernetes.WithoutFakeIntake(),
 			awskubernetes.WithWorkloadApp(func(e config.Env, kubeProvider *kubernetes.Provider) (*compkube.Workload, error) {
-				return nginx.K8sAppDefinition(e, kubeProvider, "nginx", "", nil)
+				return nginx.K8sAppDefinition(e, kubeProvider, "nginx", "", false, nil)
 			}),
 			awskubernetes.WithWorkloadApp(func(e config.Env, kubeProvider *kubernetes.Provider) (*compkube.Workload, error) {
 				return dogstatsd.K8sAppDefinition(e, kubeProvider, "dogstatsd", 8125, "/var/run/datadog/dsd.socket")
@@ -50,5 +50,5 @@ func (v *myKindSuite) TestClusterAgentInstalled() {
 		}
 	}
 	assert.True(v.T(), containsClusterAgent, "Cluster Agent not found")
-	assert.Equal(v.T(), v.Env().Agent.InstallNameLinux, "dda")
+	assert.Equal(v.T(), v.Env().Agent.InstallNameLinux, "dda-linux")
 }

test/new-e2e/go.mod

@@ -29,7 +29,7 @@ require (
 	// `TEST_INFRA_DEFINITIONS_BUILDIMAGES` matches the commit sha in the module version
 	// Example: 	github.com/DataDog/test-infra-definitions v0.0.0-YYYYMMDDHHmmSS-0123456789AB
 	// => TEST_INFRA_DEFINITIONS_BUILDIMAGES: 0123456789AB
-	github.com/DataDog/test-infra-definitions v0.0.0-20240507122412-65fbab1e1eaf
+	github.com/DataDog/test-infra-definitions v0.0.0-20240527153959-63d1aae5f639
 	github.com/aws/aws-sdk-go-v2 v1.26.1
 	github.com/aws/aws-sdk-go-v2/config v1.27.11
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.138.1

test/new-e2e/pkg/environments/aws/kubernetes/eks.go

@@ -177,7 +177,7 @@ func EKSRunFunc(ctx *pulumi.Context, env *environments.AwsKubernetes, params *Pr
 		// Create configuration for POD subnets if any
 		workloadDeps := make([]pulumi.Resource, 0)
 		if podSubnets := awsEnv.EKSPODSubnets(); len(podSubnets) > 0 {
-			eniConfigs, err := localEks.NewENIConfigs(awsEnv, comp.KubeProvider, podSubnets, awsEnv.DefaultSecurityGroups())
+			eniConfigs, err := localEks.NewENIConfigs(awsEnv, podSubnets, awsEnv.DefaultSecurityGroups(), pulumi.Provider(eksKubeProvider))
 			if err != nil {
 				return err
 			}
@@ -259,7 +259,7 @@ func EKSRunFunc(ctx *pulumi.Context, env *environments.AwsKubernetes, params *Pr
 
 		// Create unmanaged node groups
 		if params.eksWindowsNodeGroup {
-			_, err := localEks.NewWindowsUnmanagedNodeGroup(awsEnv, cluster, windowsNodeRole)
+			_, err := localEks.NewWindowsNodeGroup(awsEnv, cluster, windowsNodeRole)
 			if err != nil {
 				return err
 			}

test/new-e2e/pkg/environments/local/kubernetes/kind.go

@@ -0,0 +1,185 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016-present Datadog, Inc.
+
+// Package localkubernetes contains the provisioner for the local Kubernetes based environments
+package localkubernetes
+
+import (
+	"fmt"
+
+	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e"
+	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments"
+	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner"
+	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional"
+
+	"github.com/DataDog/test-infra-definitions/components/datadog/agent"
+	"github.com/DataDog/test-infra-definitions/resources/local"
+
+	fakeintakeComp "github.com/DataDog/test-infra-definitions/components/datadog/fakeintake"
+	"github.com/DataDog/test-infra-definitions/components/datadog/kubernetesagentparams"
+	kubeComp "github.com/DataDog/test-infra-definitions/components/kubernetes"
+	"github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake"
+
+	"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes"
+	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
+)
+
+const (
+	provisionerBaseID = "aws-kind-"
+	defaultVMName     = "kind"
+)
+
+// ProvisionerParams contains all the parameters needed to create the environment
+type ProvisionerParams struct {
+	name              string
+	agentOptions      []kubernetesagentparams.Option
+	fakeintakeOptions []fakeintake.Option
+	extraConfigParams runner.ConfigMap
+}
+
+func newProvisionerParams() *ProvisionerParams {
+	return &ProvisionerParams{
+		name:              defaultVMName,
+		agentOptions:      []kubernetesagentparams.Option{},
+		fakeintakeOptions: []fakeintake.Option{},
+		extraConfigParams: runner.ConfigMap{},
+	}
+}
+
+// ProvisionerOption is a function that modifies the ProvisionerParams
+type ProvisionerOption func(*ProvisionerParams) error
+
+// WithName sets the name of the provisioner
+func WithName(name string) ProvisionerOption {
+	return func(params *ProvisionerParams) error {
+		params.name = name
+		return nil
+	}
+}
+
+// WithAgentOptions adds options to the agent
+func WithAgentOptions(opts ...kubernetesagentparams.Option) ProvisionerOption {
+	return func(params *ProvisionerParams) error {
+		params.agentOptions = opts
+		return nil
+	}
+}
+
+// WithoutFakeIntake removes the fake intake
+func WithoutFakeIntake() ProvisionerOption {
+	return func(params *ProvisionerParams) error {
+		params.fakeintakeOptions = nil
+		return nil
+	}
+}
+
+// WithoutAgent removes the agent
+func WithoutAgent() ProvisionerOption {
+	return func(params *ProvisionerParams) error {
+		params.agentOptions = nil
+		return nil
+	}
+}
+
+// WithExtraConfigParams adds extra config parameters to the environment
+func WithExtraConfigParams(configMap runner.ConfigMap) ProvisionerOption {
+	return func(params *ProvisionerParams) error {
+		params.extraConfigParams = configMap
+		return nil
+	}
+}
+
+// Provisioner creates a new provisioner
+func Provisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Kubernetes] {
+	// We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times.
+	// and it's easy to forget about it, leading to hard to debug issues.
+	params := newProvisionerParams()
+	_ = optional.ApplyOptions(params, opts)
+
+	provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Kubernetes) error {
+		// We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times.
+		// and it's easy to forget about it, leading to hard to debug issues.
+		params := newProvisionerParams()
+		_ = optional.ApplyOptions(params, opts)
+
+		return KindRunFunc(ctx, env, params)
+	}, params.extraConfigParams)
+
+	return provisioner
+}
+
+// KindRunFunc is the Pulumi run function that runs the provisioner
+func KindRunFunc(ctx *pulumi.Context, env *environments.Kubernetes, params *ProvisionerParams) error {
+
+	localEnv, err := local.NewEnvironment(ctx)
+	if err != nil {
+		return err
+	}
+
+	kindCluster, err := kubeComp.NewLocalKindCluster(&localEnv, localEnv.CommonNamer().ResourceName("kind"), params.name, localEnv.KubernetesVersion())
+	if err != nil {
+		return err
+	}
+
+	err = kindCluster.Export(ctx, &env.KubernetesCluster.ClusterOutput)
+	if err != nil {
+		return err
+	}
+
+	kubeProvider, err := kubernetes.NewProvider(ctx, localEnv.CommonNamer().ResourceName("k8s-provider"), &kubernetes.ProviderArgs{
+		EnableServerSideApply: pulumi.Bool(true),
+		Kubeconfig:            kindCluster.KubeConfig,
+	})
+	if err != nil {
+		return err
+	}
+
+	if params.fakeintakeOptions != nil {
+		fakeintakeOpts := []fakeintake.Option{fakeintake.WithLoadBalancer()}
+		params.fakeintakeOptions = append(fakeintakeOpts, params.fakeintakeOptions...)
+		fakeIntake, err := fakeintakeComp.NewLocalDockerFakeintake(&localEnv, "fakeintake")
+		if err != nil {
+			return err
+		}
+		err = fakeIntake.Export(ctx, &env.FakeIntake.FakeintakeOutput)
+		if err != nil {
+			return err
+		}
+
+		if params.agentOptions != nil {
+			newOpts := []kubernetesagentparams.Option{kubernetesagentparams.WithFakeintake(fakeIntake)}
+			params.agentOptions = append(newOpts, params.agentOptions...)
+		}
+	} else {
+		env.FakeIntake = nil
+	}
+
+	if params.agentOptions != nil {
+		kindClusterName := ctx.Stack()
+		helmValues := fmt.Sprintf(`
+datadog:
+  kubelet:
+    tlsVerify: false
+  clusterName: "%s"
+agents:
+  useHostNetwork: true
+`, kindClusterName)
+
+		newOpts := []kubernetesagentparams.Option{kubernetesagentparams.WithHelmValues(helmValues)}
+		params.agentOptions = append(newOpts, params.agentOptions...)
+		agent, err := agent.NewKubernetesAgent(&localEnv, kindClusterName, kubeProvider, params.agentOptions...)
+		if err != nil {
+			return err
+		}
+		err = agent.Export(ctx, &env.Agent.KubernetesAgentOutput)
+		if err != nil {
+			return err
+		}
+	} else {
+		env.Agent = nil
+	}
+
+	return nil
+}

test/new-e2e/test-infra-definition/kind_test.go

@@ -31,7 +31,7 @@ func TestMyKindSuite(t *testing.T) {
 		awskubernetes.KindProvisioner(
 			awskubernetes.WithoutFakeIntake(),
 			awskubernetes.WithWorkloadApp(func(e config.Env, kubeProvider *kubernetes.Provider) (*compkube.Workload, error) {
-				return nginx.K8sAppDefinition(e, kubeProvider, "nginx", "", nil)
+				return nginx.K8sAppDefinition(e, kubeProvider, "nginx", "", false, nil)
 			}),
 		)))
 }
@@ -46,5 +46,5 @@ func (v *myKindSuite) TestClusterAgentInstalled() {
 		}
 	}
 	assert.True(v.T(), containsClusterAgent, "Cluster Agent not found")
-	assert.Equal(v.T(), v.Env().Agent.InstallNameLinux, "dda")
+	assert.Equal(v.T(), v.Env().Agent.InstallNameLinux, "dda-linux")
 }

test/new-e2e/tests/orchestrator/apply.go

@@ -40,7 +40,7 @@ func Apply(ctx *pulumi.Context) error {
 
 	// Deploy testing workload
 	if awsEnv.TestingWorkloadDeploy() {
-		if _, err := redis.K8sAppDefinition(awsEnv, kindKubeProvider, "workload-redis", agentDependency); err != nil {
+		if _, err := redis.K8sAppDefinition(awsEnv, kindKubeProvider, "workload-redis", true, agentDependency); err != nil {
 			return fmt.Errorf("failed to install redis: %w", err)
 		}
 	}