From 7bfe97fbd3cf6b59c71ef765817bd7aa19e0217d Mon Sep 17 00:00:00 2001 From: Kevin Fairise Date: Mon, 26 Feb 2024 17:39:00 +0100 Subject: [PATCH 01/19] Create local kind cluster install --- components/kubernetes/kind.go | 66 +++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/components/kubernetes/kind.go b/components/kubernetes/kind.go index 0e569d010..5bc64783a 100644 --- a/components/kubernetes/kind.go +++ b/components/kubernetes/kind.go @@ -3,6 +3,8 @@ package kubernetes import ( _ "embed" "fmt" + + "os/user" "regexp" "strings" @@ -108,3 +110,67 @@ func NewKindCluster(env config.CommonEnvironment, vm *remote.Host, resourceName, return nil }, opts...) } + +func NewLocalKindCluster(env config.CommonEnvironment, resourceName, kindClusterName string, kubeVersion string, opts ...pulumi.ResourceOption) (*Cluster, error) { + return components.NewComponent(env, resourceName, func(clusterComp *Cluster) error { + opts = utils.MergeOptions[pulumi.ResourceOption](opts, pulumi.Parent(clusterComp)) + commonEnvironment := env + + user, err := user.Current() + + kindVersionConfig, err := getKindVersionConfig(kubeVersion) + if err != nil { + return err + } + + runner := command.NewLocalRunner(env, command.LocalRunnerArgs{ + User: user.Username, + OSCommand: command.NewUnixOSCommand(), + }) + + clusterConfigFilePath := fmt.Sprintf("/tmp/kind-cluster-%s.yaml", kindClusterName) + clusterConfig, err := runner.Command("kind-config", &command.Args{ + Create: pulumi.Sprintf("cat - | tee %s > /dev/null", clusterConfigFilePath), + Delete: pulumi.Sprintf("rm -f %s", clusterConfigFilePath), + Stdin: pulumi.String(kindClusterConfig), + }, opts...) + if err != nil { + return err + } + + nodeImage := fmt.Sprintf("%s:%s", kindNodeImageRegistry, kindVersionConfig.nodeImageVersion) + createCluster, err := runner.Command( + commonEnvironment.CommonNamer.ResourceName("kind-create-cluster", resourceName), + &command.Args{ + Create: pulumi.Sprintf("kind create cluster --name %s --config %s --image %s --wait %s", kindClusterName, clusterConfigFilePath, nodeImage, kindReadinessWait), + Delete: pulumi.Sprintf("kind delete cluster --name %s", kindClusterName), + Triggers: pulumi.Array{pulumi.String(kindClusterConfig)}, + }, + utils.MergeOptions(opts, utils.PulumiDependsOn(clusterConfig), pulumi.DeleteBeforeReplace(true))..., + ) + if err != nil { + return err + } + + kubeConfigCmd, err := runner.Command( + commonEnvironment.CommonNamer.ResourceName("kind-kubeconfig", resourceName), + &command.Args{ + Create: pulumi.Sprintf("kind get kubeconfig --name %s", kindClusterName), + }, + utils.MergeOptions(opts, utils.PulumiDependsOn(createCluster))..., + ) + if err != nil { + return err + } + + // Patch Kubeconfig based on private IP output + // Also add skip tls + clusterComp.KubeConfig = pulumi.All(kubeConfigCmd.Stdout, "localhost").ApplyT(func(args []interface{}) string { + allowInsecure := regexp.MustCompile("certificate-authority-data:.+").ReplaceAllString(args[0].(string), "insecure-skip-tls-verify: true") + return strings.ReplaceAll(allowInsecure, "0.0.0.0", args[1].(string)) + }).(pulumi.StringOutput) + clusterComp.ClusterName = pulumi.String(kindClusterName).ToStringOutput() + + return nil + }, opts...) +} From 8e479893af2c0d72c35978cccc2f4c78543fb2d2 Mon Sep 17 00:00:00 2001 From: Kevin Fairise Date: Thu, 4 Apr 2024 09:41:22 +0200 Subject: [PATCH 02/19] Add API on Kubernetes Agent to be able to set environment variables --- components/datadog/agent/kubernetes.go | 1 + components/datadog/agent/kubernetes_helm.go | 13 +++++++++++++ components/datadog/kubernetesagentparams/params.go | 14 +++++++++++++- 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/components/datadog/agent/kubernetes.go b/components/datadog/agent/kubernetes.go index 72c6ea9f9..9d8bf9576 100644 --- a/components/datadog/agent/kubernetes.go +++ b/components/datadog/agent/kubernetes.go @@ -41,6 +41,7 @@ func NewKubernetesAgent(e config.CommonEnvironment, resourceName string, kubePro DeployWindows: params.DeployWindows, Namespace: params.Namespace, ValuesYAML: params.HelmValues, + EnvironmentVariables: params.EnvironmentVariables, Fakeintake: params.FakeIntake, AgentFullImagePath: params.AgentFullImagePath, ClusterAgentFullImagePath: params.ClusterAgentFullImagePath, diff --git a/components/datadog/agent/kubernetes_helm.go b/components/datadog/agent/kubernetes_helm.go index 0ffa8f65b..c7998a1de 100644 --- a/components/datadog/agent/kubernetes_helm.go +++ b/components/datadog/agent/kubernetes_helm.go @@ -25,6 +25,7 @@ type HelmInstallationArgs struct { ClusterAgentFullImagePath string KubeProvider *kubernetes.Provider Namespace string + EnvironmentVariables map[string]string ValuesYAML pulumi.AssetOrArchiveArrayInput Fakeintake *fakeintake.Fakeintake DeployWindows bool @@ -121,6 +122,7 @@ func NewHelmInstallation(e config.CommonEnvironment, args HelmInstallationArgs, values := buildLinuxHelmValues(installName, agentImagePath, agentImageTag, clusterAgentImagePath, clusterAgentImageTag, randomClusterAgentToken.Result) values.configureImagePullSecret(imgPullSecret) + values.configureEnvVars(args.EnvironmentVariables) values.configureFakeintake(e, args.Fakeintake) linux, err := helm.NewInstallation(e, helm.InstallArgs{ @@ -349,6 +351,17 @@ func (values HelmValues) configureImagePullSecret(secret *corev1.Secret) { } } +func (values HelmValues) configureEnvVars(envVars map[string]string) { + envMapArray := pulumi.MapArray{} + for k, v := range envVars { + envMapArray = append(envMapArray, pulumi.Map{ + "name": pulumi.String(k), + "value": pulumi.String(v), + }) + } + values["datadog"].(pulumi.Map)["env"] = envMapArray +} + func (values HelmValues) configureFakeintake(e config.CommonEnvironment, fakeintake *fakeintake.Fakeintake) { if fakeintake == nil { return diff --git a/components/datadog/kubernetesagentparams/params.go b/components/datadog/kubernetesagentparams/params.go index 179422bf8..68abc6526 100644 --- a/components/datadog/kubernetesagentparams/params.go +++ b/components/datadog/kubernetesagentparams/params.go @@ -39,6 +39,8 @@ type Params struct { Namespace string // HelmValues is the Helm values to use for the agent installation. HelmValues pulumi.AssetOrArchiveArray + // EnvironmentVariables is the environment variables to inject in the agents. + EnvironmentVariables map[string]string // PulumiDependsOn is a list of resources to depend on. PulumiResourceOptions []pulumi.ResourceOption // FakeIntake is the fake intake to use for the agent installation. @@ -51,7 +53,8 @@ type Option = func(*Params) error func NewParams(e config.CommonEnvironment, options ...Option) (*Params, error) { version := &Params{ - Namespace: defaultAgentNamespace, + Namespace: defaultAgentNamespace, + EnvironmentVariables: map[string]string{}, } if e.PipelineID() != "" && e.CommitSHA() != "" { @@ -78,6 +81,15 @@ func WithClusterAgentFullImagePath(fullImagePath string) func(*Params) error { } } +func WithEnvironmentVariables(env map[string]string) func(*Params) error { + return func(p *Params) error { + for k, v := range env { + p.EnvironmentVariables[k] = v + } + return nil + } +} + // WithNamespace sets the namespace to deploy the agent to. func WithNamespace(namespace string) func(*Params) error { return func(p *Params) error { From b952dac7474c119aa6f334015fd5cffcee8454d5 Mon Sep 17 00:00:00 2001 From: Kevin Fairise Date: Thu, 4 Apr 2024 09:42:54 +0200 Subject: [PATCH 03/19] Update fakeintake env vars to be able to not use ssl --- components/datadog/agent/kubernetes_helm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/datadog/agent/kubernetes_helm.go b/components/datadog/agent/kubernetes_helm.go index c7998a1de..38e6d9280 100644 --- a/components/datadog/agent/kubernetes_helm.go +++ b/components/datadog/agent/kubernetes_helm.go @@ -394,7 +394,7 @@ func (values HelmValues) configureFakeintake(e config.CommonEnvironment, fakeint }, pulumi.Map{ "name": pulumi.String("DD_LOGS_CONFIG_ADDITIONAL_ENDPOINTS"), - "value": pulumi.Sprintf(`[{"host": "%s"}]`, fakeintake.Host), + "value": pulumi.Sprintf(`[{"host": "%s", "port": %v, "use_ssl": "%t"}]`, fakeintake.Host, fakeintake.Port, fakeintake.Scheme == "https"), }, pulumi.Map{ "name": pulumi.String("DD_LOGS_CONFIG_USE_HTTP"), From 0fa8d536bb3d1836d2cb0b019721451968f4e8df Mon Sep 17 00:00:00 2001 From: Kevin Fairise Date: Thu, 4 Apr 2024 09:43:13 +0200 Subject: [PATCH 04/19] Local Fakeintake in Kubernetes --- components/datadog/fakeintake/component.go | 5 +- components/datadog/fakeintake/kubernetes.go | 78 +++++++++++++++++++++ components/kubernetes/kind-cluster.yaml | 5 ++ go.mod | 18 ++--- go.sum | 20 ++++++ 5 files changed, 117 insertions(+), 9 deletions(-) create mode 100644 components/datadog/fakeintake/kubernetes.go diff --git a/components/datadog/fakeintake/component.go b/components/datadog/fakeintake/component.go index 4b75ea41e..32cab4e25 100644 --- a/components/datadog/fakeintake/component.go +++ b/components/datadog/fakeintake/component.go @@ -9,7 +9,8 @@ import ( type FakeintakeOutput struct { // nolint:revive, We want to keep the name as Output components.JSONImporter - URL string `json:"url"` + ClientURL string `json:"clientURL"` + URL string `json:"url"` } type Fakeintake struct { @@ -21,6 +22,8 @@ type Fakeintake struct { Host pulumi.StringOutput Scheme string // Scheme is a string as it's known in code and is useful to check HTTP/HTTPS Port uint32 // Same for Port + // ClientURL is the URL used by the fakeintake client. If empty, URL should be used. + ClientURL pulumi.StringOutput `pulumi:"clientURL"` URL pulumi.StringOutput `pulumi:"url"` } diff --git a/components/datadog/fakeintake/kubernetes.go b/components/datadog/fakeintake/kubernetes.go new file mode 100644 index 000000000..023dbc0f8 --- /dev/null +++ b/components/datadog/fakeintake/kubernetes.go @@ -0,0 +1,78 @@ +package fakeintake + +import ( + "github.com/DataDog/test-infra-definitions/common/config" + "github.com/DataDog/test-infra-definitions/components" + "github.com/pulumi/pulumi-kubernetes/sdk/v3/go/kubernetes" + v1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/apps/v1" + corev1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1" + metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1" + "github.com/pulumi/pulumi/sdk/v3/go/pulumi" +) + +func NewLocalKubernetesFakeintake(e config.CommonEnvironment, resourceName string, kubeProvider *kubernetes.Provider) (*Fakeintake, error) { + return components.NewComponent(e, resourceName, func(comp *Fakeintake) error { + v1.NewDeployment(e.Ctx, resourceName, &v1.DeploymentArgs{ + Metadata: &metav1.ObjectMetaArgs{ + Name: pulumi.String(resourceName), + }, + Spec: &v1.DeploymentSpecArgs{ + Replicas: pulumi.Int(1), + Selector: &metav1.LabelSelectorArgs{ + MatchLabels: pulumi.StringMap{ + "app": pulumi.String(resourceName), + }, + }, + Template: corev1.PodTemplateSpecArgs{ + Metadata: &metav1.ObjectMetaArgs{ + Labels: pulumi.StringMap{ + "app": pulumi.String(resourceName), + }, + }, + Spec: &corev1.PodSpecArgs{ + Containers: corev1.ContainerArray{ + &corev1.ContainerArgs{ + Name: pulumi.String(resourceName), + Image: pulumi.String("public.ecr.aws/datadog/fakeintake:latest"), + ReadinessProbe: &corev1.ProbeArgs{ + HttpGet: &corev1.HTTPGetActionArgs{ + Path: pulumi.String("/fakeintake/health"), + Port: pulumi.Int(80), + }, + }, + }, + }, + }, + }, + }, + }, pulumi.Provider(kubeProvider)) + + corev1.NewService(e.Ctx, resourceName, &corev1.ServiceArgs{ + Metadata: &metav1.ObjectMetaArgs{ + Name: pulumi.String(resourceName), + }, + Spec: &corev1.ServiceSpecArgs{ + Selector: pulumi.StringMap{ + "app": pulumi.String(resourceName), + }, + Ports: corev1.ServicePortArray{ + &corev1.ServicePortArgs{ + NodePort: pulumi.Int(30080), + Port: pulumi.Int(80), + TargetPort: pulumi.Int(80), + }, + }, + Type: pulumi.String("NodePort"), + }, + }, pulumi.Provider(kubeProvider)) + + comp.Port = 80 + comp.Scheme = "http" + comp.Host = pulumi.Sprintf("%s.default.svc.cluster.local", resourceName) + comp.URL = pulumi.Sprintf("%s://%s:%d", comp.Scheme, comp.Host, comp.Port) + comp.ClientURL = pulumi.Sprintf("http://localhost:30080") + + return nil + + }) +} diff --git a/components/kubernetes/kind-cluster.yaml b/components/kubernetes/kind-cluster.yaml index d4260056e..15bc50d67 100644 --- a/components/kubernetes/kind-cluster.yaml +++ b/components/kubernetes/kind-cluster.yaml @@ -2,6 +2,10 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane + extraPortMappings: + - containerPort: 30080 + hostPort: 30080 + protocol: TCP containerdConfigPatches: - |- [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] @@ -9,3 +13,4 @@ containerdConfigPatches: networking: apiServerAddress: "0.0.0.0" apiServerPort: 8443 + diff --git a/go.mod b/go.mod index 85b093109..2a91c15fa 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/pulumi/pulumi-libvirt/sdk v0.4.0 github.com/pulumi/pulumi-random/sdk/v4 v4.13.4 github.com/pulumi/pulumi-tls/sdk/v4 v4.10.0 - github.com/pulumi/pulumi/sdk/v3 v3.99.0 + github.com/pulumi/pulumi/sdk/v3 v3.109.0 github.com/samber/lo v1.38.1 github.com/stretchr/testify v1.8.4 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa @@ -60,7 +60,7 @@ require ( github.com/charmbracelet/bubbletea v0.24.2 // indirect github.com/charmbracelet/lipgloss v0.7.1 // indirect github.com/cheggaaa/pb v1.0.29 // indirect - github.com/cloudflare/circl v1.3.3 // indirect + github.com/cloudflare/circl v1.3.7 // indirect github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -69,11 +69,12 @@ require ( github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect - github.com/go-git/go-git/v5 v5.9.0 // indirect + github.com/go-git/go-git/v5 v5.11.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/glog v1.1.1 // indirect + github.com/golang/glog v1.1.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/google/uuid v1.4.0 // indirect github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect @@ -103,12 +104,13 @@ require ( github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 // indirect github.com/pulumi/esc v0.6.2 // indirect github.com/pulumi/pulumi-docker/sdk/v3 v3.6.1 // indirect + github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.9.1 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect github.com/santhosh-tekuri/jsonschema/v5 v5.3.0 // indirect github.com/sergi/go-diff v1.3.1 // indirect - github.com/skeema/knownhosts v1.2.0 // indirect + github.com/skeema/knownhosts v1.2.1 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/cobra v1.7.0 // indirect github.com/spf13/pflag v1.0.5 // indirect @@ -121,14 +123,14 @@ require ( go.uber.org/atomic v1.11.0 // indirect golang.org/x/crypto v0.17.0 // indirect golang.org/x/mod v0.14.0 // indirect - golang.org/x/net v0.18.0 // indirect + golang.org/x/net v0.19.0 // indirect golang.org/x/sync v0.5.0 // indirect golang.org/x/sys v0.15.0 // indirect golang.org/x/term v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.15.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230731190214-cbb8c96f2d6d // indirect - google.golang.org/grpc v1.57.1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect + google.golang.org/grpc v1.59.0 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/go.sum b/go.sum index d25bd078f..c7b68ef92 100644 --- a/go.sum +++ b/go.sum @@ -98,6 +98,8 @@ github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuP github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= +github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 h1:q2hJAaP1k2wIvVRd/hEHD7lacgqrCPS+k8g1MndzfWY= @@ -158,6 +160,8 @@ github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f/go.mod github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc= github.com/go-git/go-git/v5 v5.9.0 h1:cD9SFA7sHVRdJ7AYck1ZaAa/yeuBvGPxwXDL8cxrObY= github.com/go-git/go-git/v5 v5.9.0/go.mod h1:RKIqga24sWdMGZF+1Ekv9kylsDz6LzdTSI2s/OsZWE0= +github.com/go-git/go-git/v5 v5.11.0 h1:XIZc1p+8YzypNr34itUfSvYJcv+eYdTnTvOZ2vD3cA4= +github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lKqXmCUiUCY= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= @@ -172,6 +176,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.1.1 h1:jxpi2eWoU84wbX9iIEyAeeoac3FLuifZpY9tcNUD9kw= github.com/golang/glog v1.1.1/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= +github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= +github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -202,6 +208,8 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -353,6 +361,8 @@ github.com/pulumi/pulumi-eks/sdk v1.0.4/go.mod h1:eSRoTIxvvu+uyc4tXo//TCsE9qD/DU github.com/pulumi/pulumi-kubernetes/sdk/v3 v3.17.0/go.mod h1:w+Y1d8uqc+gv7JYWLF4rfzvTsIIHR1SCL+GG6sX1xMM= github.com/pulumi/pulumi-kubernetes/sdk/v3 v3.30.2 h1:xJu48+RW+BHHnKtBni6Vj5vKqOEgCzdZAysGbh6tVM0= github.com/pulumi/pulumi-kubernetes/sdk/v3 v3.30.2/go.mod h1:7yCJFC/jnUwFs566f0FAY2iAzc4G1mQP8H6K+40FK4Y= +github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.9.1 h1:dgazi5bI3Vxz+aLuH+DxRqKxPWGaFIkT3fIepHr7h0g= +github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.9.1/go.mod h1:O8hanLXCEXiyzA8gIeME5o/SmJ39Vyy9wLcBYCFpOp0= github.com/pulumi/pulumi-libvirt/sdk v0.4.0 h1:wq1Ox8FRKQ1kc2DPq3m5DGQgZEhE7kp4mtG556HxJLs= github.com/pulumi/pulumi-libvirt/sdk v0.4.0/go.mod h1:tjjyDajp6Pb1pRCdaIugknIfzxw3Prev3o/k2nade+I= github.com/pulumi/pulumi-random/sdk/v4 v4.13.4 h1:g3jdktE5L5IDrOw4OiB+yhgxSw0okRPJnyV6PlIzTEQ= @@ -363,6 +373,8 @@ github.com/pulumi/pulumi/sdk/v3 v3.16.0/go.mod h1:252ou/zAU1g6E8iTwe2Y9ht7pb5BDl github.com/pulumi/pulumi/sdk/v3 v3.50.1/go.mod h1:tqQ4z9ocyM/UI2VQ7ZReWR3w6dF5ffEozoHipOMcDh4= github.com/pulumi/pulumi/sdk/v3 v3.99.0 h1:vsFoEEdweYg3Hm6/Jlj1sE2cLtauzoqAdVbLMcC7Cw8= github.com/pulumi/pulumi/sdk/v3 v3.99.0/go.mod h1:wFM/6iAMlidgLDSF9QU+p3P+B+vg/xloFyVeZrVwA1w= +github.com/pulumi/pulumi/sdk/v3 v3.109.0 h1:bcejf5DNcx2I66X1M2tYFKzZGvefPD44sGzOMr0UB00= +github.com/pulumi/pulumi/sdk/v3 v3.109.0/go.mod h1:5A6GHUwAJlRY1SSLZh84aDIbsBShcrfcmHzI50ecSBg= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= @@ -392,6 +404,8 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/skeema/knownhosts v1.2.0 h1:h9r9cf0+u7wSE+M183ZtMGgOJKiL96brpaz5ekfJCpM= github.com/skeema/knownhosts v1.2.0/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo= +github.com/skeema/knownhosts v1.2.1 h1:SHWdIUa82uGZz+F+47k8SY4QhhI291cXCpopT1lK2AQ= +github.com/skeema/knownhosts v1.2.1/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -508,6 +522,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -608,6 +624,8 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY google.golang.org/genproto v0.0.0-20200608115520-7c474a2e3482/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230731190214-cbb8c96f2d6d h1:pgIUhmqwKOUlnKna4r6amKdUngdL8DrkpFeV8+VBElY= google.golang.org/genproto/googleapis/rpc v0.0.0-20230731190214-cbb8c96f2d6d/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f h1:ultW7fxlIvee4HYrtnaRPon9HpEgFk5zYpmfMgtKB5I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f/go.mod h1:L9KNLi232K1/xB6f7AlSX692koaRnKaWSR0stBki0Yc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= @@ -616,6 +634,8 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.57.1 h1:upNTNqv0ES+2ZOOqACwVtS3Il8M12/+Hz41RCPzAjQg= google.golang.org/grpc v1.57.1/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From 5df1e3c6d31849030eea941ab085f6a6f3c66526 Mon Sep 17 00:00:00 2001 From: Kevin Fairise Date: Thu, 4 Apr 2024 09:58:33 +0200 Subject: [PATCH 05/19] Use const for localPort --- components/datadog/fakeintake/kubernetes.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/components/datadog/fakeintake/kubernetes.go b/components/datadog/fakeintake/kubernetes.go index 023dbc0f8..f685ab22a 100644 --- a/components/datadog/fakeintake/kubernetes.go +++ b/components/datadog/fakeintake/kubernetes.go @@ -10,6 +10,8 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) +const localPort = 300080 + func NewLocalKubernetesFakeintake(e config.CommonEnvironment, resourceName string, kubeProvider *kubernetes.Provider) (*Fakeintake, error) { return components.NewComponent(e, resourceName, func(comp *Fakeintake) error { v1.NewDeployment(e.Ctx, resourceName, &v1.DeploymentArgs{ @@ -57,7 +59,7 @@ func NewLocalKubernetesFakeintake(e config.CommonEnvironment, resourceName strin }, Ports: corev1.ServicePortArray{ &corev1.ServicePortArgs{ - NodePort: pulumi.Int(30080), + NodePort: pulumi.Int(localPort), Port: pulumi.Int(80), TargetPort: pulumi.Int(80), }, @@ -70,7 +72,7 @@ func NewLocalKubernetesFakeintake(e config.CommonEnvironment, resourceName strin comp.Scheme = "http" comp.Host = pulumi.Sprintf("%s.default.svc.cluster.local", resourceName) comp.URL = pulumi.Sprintf("%s://%s:%d", comp.Scheme, comp.Host, comp.Port) - comp.ClientURL = pulumi.Sprintf("http://localhost:30080") + comp.ClientURL = pulumi.Sprintf("http://localhost:%v", localPort) return nil From 02f7fb0626ef5f05851af457399bbeebf383461a Mon Sep 17 00:00:00 2001 From: Kevin Fairise Date: Thu, 4 Apr 2024 11:27:20 +0200 Subject: [PATCH 06/19] Fix lint --- components/datadog/fakeintake/kubernetes.go | 12 ++++++++++-- components/kubernetes/kind.go | 3 +++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/components/datadog/fakeintake/kubernetes.go b/components/datadog/fakeintake/kubernetes.go index f84d28e40..c912c023f 100644 --- a/components/datadog/fakeintake/kubernetes.go +++ b/components/datadog/fakeintake/kubernetes.go @@ -14,7 +14,8 @@ const localPort = 30080 func NewLocalKubernetesFakeintake(e config.CommonEnvironment, resourceName string, kubeProvider *kubernetes.Provider) (*Fakeintake, error) { return components.NewComponent(e, resourceName, func(comp *Fakeintake) error { - v1.NewDeployment(e.Ctx, resourceName, &v1.DeploymentArgs{ + + _, err := v1.NewDeployment(e.Ctx, resourceName, &v1.DeploymentArgs{ Metadata: &metav1.ObjectMetaArgs{ Name: pulumi.String(resourceName), }, @@ -49,7 +50,11 @@ func NewLocalKubernetesFakeintake(e config.CommonEnvironment, resourceName strin }, }, pulumi.Provider(kubeProvider)) - corev1.NewService(e.Ctx, resourceName, &corev1.ServiceArgs{ + if err != nil { + return err + } + + _, err = corev1.NewService(e.Ctx, resourceName, &corev1.ServiceArgs{ Metadata: &metav1.ObjectMetaArgs{ Name: pulumi.String(resourceName), }, @@ -67,6 +72,9 @@ func NewLocalKubernetesFakeintake(e config.CommonEnvironment, resourceName strin Type: pulumi.String("NodePort"), }, }, pulumi.Provider(kubeProvider)) + if err != nil { + return err + } comp.Port = 80 comp.Scheme = "http" diff --git a/components/kubernetes/kind.go b/components/kubernetes/kind.go index 5e50312fa..70daae161 100644 --- a/components/kubernetes/kind.go +++ b/components/kubernetes/kind.go @@ -117,6 +117,9 @@ func NewLocalKindCluster(env config.CommonEnvironment, resourceName, kindCluster commonEnvironment := env user, err := user.Current() + if err != nil { + return err + } kindVersionConfig, err := getKindVersionConfig(kubeVersion) if err != nil { From fde677bce90b67842e880d90b1c8fdd0a878cb1e Mon Sep 17 00:00:00 2001 From: Kevin Fairise Date: Wed, 10 Apr 2024 16:44:37 +0200 Subject: [PATCH 07/19] Do not install the fakeintake in Kubernetes, rather use a docker version that can be used with other local provider --- common/config/providers.go | 6 ++ components/command/runner.go | 6 ++ components/datadog/fakeintake/component.go | 5 +- components/datadog/fakeintake/docker.go | 52 ++++++++++++ components/datadog/fakeintake/kubernetes.go | 88 --------------------- components/kubernetes/kind-cluster.yaml | 4 - components/kubernetes/kind.go | 8 +- 7 files changed, 66 insertions(+), 103 deletions(-) create mode 100644 components/datadog/fakeintake/docker.go delete mode 100644 components/datadog/fakeintake/kubernetes.go diff --git a/common/config/providers.go b/common/config/providers.go index df9f0441c..e013f2183 100644 --- a/common/config/providers.go +++ b/common/config/providers.go @@ -6,6 +6,7 @@ import ( "github.com/pulumi/pulumi-awsx/sdk/v2/go/awsx" "github.com/pulumi/pulumi-command/sdk/go/command" + "github.com/pulumi/pulumi-docker/sdk/v4/go/docker" "github.com/pulumi/pulumi-eks/sdk/v2/go/eks" "github.com/pulumi/pulumi-random/sdk/v4/go/random" "github.com/pulumi/pulumi-tls/sdk/v4/go/tls" @@ -22,6 +23,7 @@ const ( ProviderAWSX ProviderID = "awsx" ProviderEKS ProviderID = "eks" ProviderAzure ProviderID = "azure" + ProviderDocker ProviderID = "docker" ) func dummyProvidersFactory() map[ProviderID]func(ctx *pulumi.Context, name string) (pulumi.ProviderResource, error) { @@ -46,6 +48,10 @@ func dummyProvidersFactory() map[ProviderID]func(ctx *pulumi.Context, name strin provider, err := eks.NewProvider(ctx, name, nil) return provider, err }, + ProviderDocker: func(ctx *pulumi.Context, name string) (pulumi.ProviderResource, error) { + provider, err := docker.NewProvider(ctx, name, nil) + return provider, err + }, } } diff --git a/components/command/runner.go b/components/command/runner.go index c84f58be6..23e455254 100644 --- a/components/command/runner.go +++ b/components/command/runner.go @@ -53,6 +53,9 @@ type runnerConfiguration struct { connection remote.ConnectionInput } +type RunnerInterface interface { + Command(name string, args *Args, opts ...pulumi.ResourceOption) (*remote.Command, error) +} type Runner struct { e config.CommonEnvironment namer namer.Namer @@ -143,6 +146,9 @@ func NewLocalRunner(e config.CommonEnvironment, args LocalRunnerArgs) *LocalRunn return localRunner } +type Command struct { +} + func (r *LocalRunner) Command(name string, args *Args, opts ...pulumi.ResourceOption) (*local.Command, error) { opts = utils.MergeOptions[pulumi.ResourceOption](opts, r.e.WithProviders(config.ProviderCommand)) return local.NewCommand(r.e.Ctx, r.namer.ResourceName("cmd", name), args.toLocalCommandArgs(r.config, r.osCommand), opts...) diff --git a/components/datadog/fakeintake/component.go b/components/datadog/fakeintake/component.go index 32cab4e25..4b75ea41e 100644 --- a/components/datadog/fakeintake/component.go +++ b/components/datadog/fakeintake/component.go @@ -9,8 +9,7 @@ import ( type FakeintakeOutput struct { // nolint:revive, We want to keep the name as Output components.JSONImporter - ClientURL string `json:"clientURL"` - URL string `json:"url"` + URL string `json:"url"` } type Fakeintake struct { @@ -22,8 +21,6 @@ type Fakeintake struct { Host pulumi.StringOutput Scheme string // Scheme is a string as it's known in code and is useful to check HTTP/HTTPS Port uint32 // Same for Port - // ClientURL is the URL used by the fakeintake client. If empty, URL should be used. - ClientURL pulumi.StringOutput `pulumi:"clientURL"` URL pulumi.StringOutput `pulumi:"url"` } diff --git a/components/datadog/fakeintake/docker.go b/components/datadog/fakeintake/docker.go new file mode 100644 index 000000000..c8d2d2ec0 --- /dev/null +++ b/components/datadog/fakeintake/docker.go @@ -0,0 +1,52 @@ +package fakeintake + +import ( + "log" + "net" + + "github.com/DataDog/test-infra-definitions/common/config" + "github.com/DataDog/test-infra-definitions/components" + "github.com/pulumi/pulumi-docker/sdk/v4/go/docker" + "github.com/pulumi/pulumi/sdk/v3/go/pulumi" +) + +const hostPort = 30080 + +func NewLocalDockerFakeintake(e config.CommonEnvironment, resourceName string) (*Fakeintake, error) { + return components.NewComponent(e, resourceName, func(comp *Fakeintake) error { + + _, err := docker.NewContainer(e.Ctx, e.CommonNamer.ResourceName("local-docker-container"), &docker.ContainerArgs{ + Image: pulumi.String("public.ecr.aws/datadog/fakeintake:latest"), + Ports: docker.ContainerPortArray{ + &docker.ContainerPortArgs{ + Internal: pulumi.Int(80), + External: pulumi.Int(hostPort), + }, + }, + }, e.WithProviders(config.ProviderDocker)) + if err != nil { + return err + } + + localIp := getLocalIp() + + comp.Host = pulumi.Sprintf(localIp.String()) + comp.Port = hostPort + comp.Scheme = "http" + comp.URL = pulumi.Sprintf("%s://%s:%d", comp.Scheme, comp.Host, comp.Port) + + return nil + }) +} + +func getLocalIp() net.IP { + conn, err := net.Dial("udp", "255.255.255.255:80") // initiate a connection to get the local address, the url does not need to exist + if err != nil { + log.Fatal(err) + } + defer conn.Close() + + localAddr := conn.LocalAddr().(*net.UDPAddr) + + return localAddr.IP +} diff --git a/components/datadog/fakeintake/kubernetes.go b/components/datadog/fakeintake/kubernetes.go deleted file mode 100644 index c912c023f..000000000 --- a/components/datadog/fakeintake/kubernetes.go +++ /dev/null @@ -1,88 +0,0 @@ -package fakeintake - -import ( - "github.com/DataDog/test-infra-definitions/common/config" - "github.com/DataDog/test-infra-definitions/components" - "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" - v1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/apps/v1" - corev1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1" - metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1" - "github.com/pulumi/pulumi/sdk/v3/go/pulumi" -) - -const localPort = 30080 - -func NewLocalKubernetesFakeintake(e config.CommonEnvironment, resourceName string, kubeProvider *kubernetes.Provider) (*Fakeintake, error) { - return components.NewComponent(e, resourceName, func(comp *Fakeintake) error { - - _, err := v1.NewDeployment(e.Ctx, resourceName, &v1.DeploymentArgs{ - Metadata: &metav1.ObjectMetaArgs{ - Name: pulumi.String(resourceName), - }, - Spec: &v1.DeploymentSpecArgs{ - Replicas: pulumi.Int(1), - Selector: &metav1.LabelSelectorArgs{ - MatchLabels: pulumi.StringMap{ - "app": pulumi.String(resourceName), - }, - }, - Template: corev1.PodTemplateSpecArgs{ - Metadata: &metav1.ObjectMetaArgs{ - Labels: pulumi.StringMap{ - "app": pulumi.String(resourceName), - }, - }, - Spec: &corev1.PodSpecArgs{ - Containers: corev1.ContainerArray{ - &corev1.ContainerArgs{ - Name: pulumi.String(resourceName), - Image: pulumi.String("public.ecr.aws/datadog/fakeintake:latest"), - ReadinessProbe: &corev1.ProbeArgs{ - HttpGet: &corev1.HTTPGetActionArgs{ - Path: pulumi.String("/fakeintake/health"), - Port: pulumi.Int(80), - }, - }, - }, - }, - }, - }, - }, - }, pulumi.Provider(kubeProvider)) - - if err != nil { - return err - } - - _, err = corev1.NewService(e.Ctx, resourceName, &corev1.ServiceArgs{ - Metadata: &metav1.ObjectMetaArgs{ - Name: pulumi.String(resourceName), - }, - Spec: &corev1.ServiceSpecArgs{ - Selector: pulumi.StringMap{ - "app": pulumi.String(resourceName), - }, - Ports: corev1.ServicePortArray{ - &corev1.ServicePortArgs{ - NodePort: pulumi.Int(localPort), - Port: pulumi.Int(80), - TargetPort: pulumi.Int(80), - }, - }, - Type: pulumi.String("NodePort"), - }, - }, pulumi.Provider(kubeProvider)) - if err != nil { - return err - } - - comp.Port = 80 - comp.Scheme = "http" - comp.Host = pulumi.Sprintf("%s.default.svc.cluster.local", resourceName) - comp.URL = pulumi.Sprintf("%s://%s:%d", comp.Scheme, comp.Host, comp.Port) - comp.ClientURL = pulumi.Sprintf("http://localhost:%v", localPort) - - return nil - - }) -} diff --git a/components/kubernetes/kind-cluster.yaml b/components/kubernetes/kind-cluster.yaml index 15bc50d67..83a971970 100644 --- a/components/kubernetes/kind-cluster.yaml +++ b/components/kubernetes/kind-cluster.yaml @@ -2,10 +2,6 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane - extraPortMappings: - - containerPort: 30080 - hostPort: 30080 - protocol: TCP containerdConfigPatches: - |- [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] diff --git a/components/kubernetes/kind.go b/components/kubernetes/kind.go index 70daae161..f42e9c5fd 100644 --- a/components/kubernetes/kind.go +++ b/components/kubernetes/kind.go @@ -4,7 +4,6 @@ import ( _ "embed" "fmt" - "os/user" "regexp" "strings" @@ -116,18 +115,13 @@ func NewLocalKindCluster(env config.CommonEnvironment, resourceName, kindCluster opts = utils.MergeOptions[pulumi.ResourceOption](opts, pulumi.Parent(clusterComp)) commonEnvironment := env - user, err := user.Current() - if err != nil { - return err - } - kindVersionConfig, err := getKindVersionConfig(kubeVersion) if err != nil { return err } runner := command.NewLocalRunner(env, command.LocalRunnerArgs{ - User: user.Username, + // User: user.Username, OSCommand: command.NewUnixOSCommand(), }) From 29982fd0b032063f379932be9564a3d3860509d9 Mon Sep 17 00:00:00 2001 From: Kevin Fairise Date: Wed, 10 Apr 2024 17:31:04 +0200 Subject: [PATCH 08/19] Remove runner interface try --- components/command/runner.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/components/command/runner.go b/components/command/runner.go index 23e455254..c84f58be6 100644 --- a/components/command/runner.go +++ b/components/command/runner.go @@ -53,9 +53,6 @@ type runnerConfiguration struct { connection remote.ConnectionInput } -type RunnerInterface interface { - Command(name string, args *Args, opts ...pulumi.ResourceOption) (*remote.Command, error) -} type Runner struct { e config.CommonEnvironment namer namer.Namer @@ -146,9 +143,6 @@ func NewLocalRunner(e config.CommonEnvironment, args LocalRunnerArgs) *LocalRunn return localRunner } -type Command struct { -} - func (r *LocalRunner) Command(name string, args *Args, opts ...pulumi.ResourceOption) (*local.Command, error) { opts = utils.MergeOptions[pulumi.ResourceOption](opts, r.e.WithProviders(config.ProviderCommand)) return local.NewCommand(r.e.Ctx, r.namer.ResourceName("cmd", name), args.toLocalCommandArgs(r.config, r.osCommand), opts...) From 7804b1efe6224d223b732ebbcdc3579c53b1b6f8 Mon Sep 17 00:00:00 2001 From: Kevin Fairise Date: Wed, 10 Apr 2024 17:41:53 +0200 Subject: [PATCH 09/19] Remove not needed env var configuration --- components/datadog/agent/kubernetes.go | 1 - components/datadog/agent/kubernetes_helm.go | 3 --- components/datadog/kubernetesagentparams/params.go | 14 +------------- 3 files changed, 1 insertion(+), 17 deletions(-) diff --git a/components/datadog/agent/kubernetes.go b/components/datadog/agent/kubernetes.go index 3959e9bbe..c8b8386a3 100644 --- a/components/datadog/agent/kubernetes.go +++ b/components/datadog/agent/kubernetes.go @@ -41,7 +41,6 @@ func NewKubernetesAgent(e config.CommonEnvironment, resourceName string, kubePro DeployWindows: params.DeployWindows, Namespace: params.Namespace, ValuesYAML: params.HelmValues, - EnvironmentVariables: params.EnvironmentVariables, Fakeintake: params.FakeIntake, AgentFullImagePath: params.AgentFullImagePath, ClusterAgentFullImagePath: params.ClusterAgentFullImagePath, diff --git a/components/datadog/agent/kubernetes_helm.go b/components/datadog/agent/kubernetes_helm.go index 39ff5360c..b9e8204e2 100644 --- a/components/datadog/agent/kubernetes_helm.go +++ b/components/datadog/agent/kubernetes_helm.go @@ -36,8 +36,6 @@ type HelmInstallationArgs struct { AgentFullImagePath string // ClusterAgentFullImagePath is used to specify the full image path for the cluster agent ClusterAgentFullImagePath string - // EnvironmentVariables is used to specify environment variables to inject in the agents - EnvironmentVariables map[string]string } type HelmComponent struct { @@ -131,7 +129,6 @@ func NewHelmInstallation(e config.CommonEnvironment, args HelmInstallationArgs, values := buildLinuxHelmValues(installName, agentImagePath, agentImageTag, clusterAgentImagePath, clusterAgentImageTag, randomClusterAgentToken.Result) values.configureImagePullSecret(imgPullSecret) - values.configureEnvVars(args.EnvironmentVariables) values.configureFakeintake(e, args.Fakeintake) linux, err := helm.NewInstallation(e, helm.InstallArgs{ diff --git a/components/datadog/kubernetesagentparams/params.go b/components/datadog/kubernetesagentparams/params.go index 2a2159013..41024a2c3 100644 --- a/components/datadog/kubernetesagentparams/params.go +++ b/components/datadog/kubernetesagentparams/params.go @@ -39,8 +39,6 @@ type Params struct { Namespace string // HelmValues is the Helm values to use for the agent installation. HelmValues pulumi.AssetOrArchiveArray - // EnvironmentVariables is the environment variables to inject in the agents. - EnvironmentVariables map[string]string // PulumiDependsOn is a list of resources to depend on. PulumiResourceOptions []pulumi.ResourceOption // FakeIntake is the fake intake to use for the agent installation. @@ -53,8 +51,7 @@ type Option = func(*Params) error func NewParams(e config.CommonEnvironment, options ...Option) (*Params, error) { version := &Params{ - Namespace: defaultAgentNamespace, - EnvironmentVariables: map[string]string{}, + Namespace: defaultAgentNamespace, } if e.PipelineID() != "" && e.CommitSHA() != "" { @@ -81,15 +78,6 @@ func WithClusterAgentFullImagePath(fullImagePath string) func(*Params) error { } } -func WithEnvironmentVariables(env map[string]string) func(*Params) error { - return func(p *Params) error { - for k, v := range env { - p.EnvironmentVariables[k] = v - } - return nil - } -} - // WithNamespace sets the namespace to deploy the agent to. func WithNamespace(namespace string) func(*Params) error { return func(p *Params) error { From 2e39a1d5adf3323bf751c152fcbf32bf0fe6a48b Mon Sep 17 00:00:00 2001 From: Kevin Fairise Date: Wed, 10 Apr 2024 18:03:08 +0200 Subject: [PATCH 10/19] Remove unneeded kubeconfig modif --- components/kubernetes/kind.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/components/kubernetes/kind.go b/components/kubernetes/kind.go index f42e9c5fd..766b80c85 100644 --- a/components/kubernetes/kind.go +++ b/components/kubernetes/kind.go @@ -160,12 +160,7 @@ func NewLocalKindCluster(env config.CommonEnvironment, resourceName, kindCluster return err } - // Patch Kubeconfig based on private IP output - // Also add skip tls - clusterComp.KubeConfig = pulumi.All(kubeConfigCmd.Stdout, "localhost").ApplyT(func(args []interface{}) string { - allowInsecure := regexp.MustCompile("certificate-authority-data:.+").ReplaceAllString(args[0].(string), "insecure-skip-tls-verify: true") - return strings.ReplaceAll(allowInsecure, "0.0.0.0", args[1].(string)) - }).(pulumi.StringOutput) + clusterComp.KubeConfig = kubeConfigCmd.Stdout clusterComp.ClusterName = pulumi.String(kindClusterName).ToStringOutput() return nil From a12d6bf8c3cbcd25464c27eb8328160ef8759982 Mon Sep 17 00:00:00 2001 From: Kevin Fairise Date: Wed, 10 Apr 2024 18:05:52 +0200 Subject: [PATCH 11/19] Remove configureEnvVars --- components/datadog/agent/kubernetes_helm.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/components/datadog/agent/kubernetes_helm.go b/components/datadog/agent/kubernetes_helm.go index b9e8204e2..9c1907e91 100644 --- a/components/datadog/agent/kubernetes_helm.go +++ b/components/datadog/agent/kubernetes_helm.go @@ -361,17 +361,6 @@ func (values HelmValues) configureImagePullSecret(secret *corev1.Secret) { } } -func (values HelmValues) configureEnvVars(envVars map[string]string) { - envMapArray := pulumi.MapArray{} - for k, v := range envVars { - envMapArray = append(envMapArray, pulumi.Map{ - "name": pulumi.String(k), - "value": pulumi.String(v), - }) - } - values["datadog"].(pulumi.Map)["env"] = envMapArray -} - func (values HelmValues) configureFakeintake(e config.CommonEnvironment, fakeintake *fakeintake.Fakeintake) { if fakeintake == nil { return From acafc9277246ae7e26dc3a4dab2c9f4168435f54 Mon Sep 17 00:00:00 2001 From: Kevin Fairise Date: Fri, 12 Apr 2024 14:28:52 +0200 Subject: [PATCH 12/19] Make linter happy --- components/datadog/fakeintake/docker.go | 6 +++--- components/remote/component.go | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/components/datadog/fakeintake/docker.go b/components/datadog/fakeintake/docker.go index c8d2d2ec0..b576d62c5 100644 --- a/components/datadog/fakeintake/docker.go +++ b/components/datadog/fakeintake/docker.go @@ -28,9 +28,9 @@ func NewLocalDockerFakeintake(e config.CommonEnvironment, resourceName string) ( return err } - localIp := getLocalIp() + localIP := getLocalIP() - comp.Host = pulumi.Sprintf(localIp.String()) + comp.Host = pulumi.Sprintf(localIP.String()) comp.Port = hostPort comp.Scheme = "http" comp.URL = pulumi.Sprintf("%s://%s:%d", comp.Scheme, comp.Host, comp.Port) @@ -39,7 +39,7 @@ func NewLocalDockerFakeintake(e config.CommonEnvironment, resourceName string) ( }) } -func getLocalIp() net.IP { +func getLocalIP() net.IP { conn, err := net.Dial("udp", "255.255.255.255:80") // initiate a connection to get the local address, the url does not need to exist if err != nil { log.Fatal(err) diff --git a/components/remote/component.go b/components/remote/component.go index 99992fe2d..667ca5d3f 100644 --- a/components/remote/component.go +++ b/components/remote/component.go @@ -17,6 +17,7 @@ type HostOutput struct { OSFamily os.Family `json:"osFamily"` OSFlavor os.Flavor `json:"osFlavor"` OSVersion string `json:"osVersion"` + AMI string `json:"ami"` } // Host represents a remote host (for instance, a VM) @@ -32,6 +33,7 @@ type Host struct { OSFamily pulumi.IntOutput `pulumi:"osFamily"` OSFlavor pulumi.IntOutput `pulumi:"osFlavor"` OSVersion pulumi.StringOutput `pulumi:"osVersion"` + AMI pulumi.StringOutput `pulumi:"ami"` } func (h *Host) Export(ctx *pulumi.Context, out *HostOutput) error { From f77c11b166e7a6e4a8c419ef6522514be4582754 Mon Sep 17 00:00:00 2001 From: Kevin Fairise Date: Mon, 22 Apr 2024 16:32:18 +0200 Subject: [PATCH 13/19] Remove AMI --- components/remote/component.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/remote/component.go b/components/remote/component.go index 667ca5d3f..99992fe2d 100644 --- a/components/remote/component.go +++ b/components/remote/component.go @@ -17,7 +17,6 @@ type HostOutput struct { OSFamily os.Family `json:"osFamily"` OSFlavor os.Flavor `json:"osFlavor"` OSVersion string `json:"osVersion"` - AMI string `json:"ami"` } // Host represents a remote host (for instance, a VM) @@ -33,7 +32,6 @@ type Host struct { OSFamily pulumi.IntOutput `pulumi:"osFamily"` OSFlavor pulumi.IntOutput `pulumi:"osFlavor"` OSVersion pulumi.StringOutput `pulumi:"osVersion"` - AMI pulumi.StringOutput `pulumi:"ami"` } func (h *Host) Export(ctx *pulumi.Context, out *HostOutput) error { From 5542a3fc15beac6a2d104a5ee6bd277449d7ece6 Mon Sep 17 00:00:00 2001 From: Kevin Fairise <132568982+KevinFairise2@users.noreply.github.com> Date: Mon, 22 Apr 2024 18:47:07 +0200 Subject: [PATCH 14/19] Update components/datadog/fakeintake/docker.go Co-authored-by: pducolin <45568537+pducolin@users.noreply.github.com> --- components/datadog/fakeintake/docker.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/components/datadog/fakeintake/docker.go b/components/datadog/fakeintake/docker.go index b576d62c5..632fde25a 100644 --- a/components/datadog/fakeintake/docker.go +++ b/components/datadog/fakeintake/docker.go @@ -40,7 +40,10 @@ func NewLocalDockerFakeintake(e config.CommonEnvironment, resourceName string) ( } func getLocalIP() net.IP { - conn, err := net.Dial("udp", "255.255.255.255:80") // initiate a connection to get the local address, the url does not need to exist + // Open a connection to an external valid URL to + // get the local address from the connection instance + // The URL does not need to exist + conn, err := net.Dial("udp", "255.255.255.255:80") if err != nil { log.Fatal(err) } From 050bab2e2094826c8561357f4e51b9c3efcf1919 Mon Sep 17 00:00:00 2001 From: Kevin Fairise <132568982+KevinFairise2@users.noreply.github.com> Date: Mon, 22 Apr 2024 18:48:42 +0200 Subject: [PATCH 15/19] Update components/datadog/fakeintake/docker.go --- components/datadog/fakeintake/docker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/datadog/fakeintake/docker.go b/components/datadog/fakeintake/docker.go index 632fde25a..a377271c7 100644 --- a/components/datadog/fakeintake/docker.go +++ b/components/datadog/fakeintake/docker.go @@ -43,7 +43,7 @@ func getLocalIP() net.IP { // Open a connection to an external valid URL to // get the local address from the connection instance // The URL does not need to exist - conn, err := net.Dial("udp", "255.255.255.255:80") + conn, err := net.Dial("udp", "8.8.8.8:80") if err != nil { log.Fatal(err) } From 9f6baba2fcd8ece324592b599c46f061889c94a3 Mon Sep 17 00:00:00 2001 From: Kevin Fairise Date: Tue, 23 Apr 2024 09:45:44 +0200 Subject: [PATCH 16/19] gofmt --- components/datadog/fakeintake/docker.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/datadog/fakeintake/docker.go b/components/datadog/fakeintake/docker.go index a377271c7..bfc8244c8 100644 --- a/components/datadog/fakeintake/docker.go +++ b/components/datadog/fakeintake/docker.go @@ -40,10 +40,10 @@ func NewLocalDockerFakeintake(e config.CommonEnvironment, resourceName string) ( } func getLocalIP() net.IP { - // Open a connection to an external valid URL to + // Open a connection to an external valid URL to // get the local address from the connection instance // The URL does not need to exist - conn, err := net.Dial("udp", "8.8.8.8:80") + conn, err := net.Dial("udp", "8.8.8.8:80") if err != nil { log.Fatal(err) } From 8d100bc5470fcb93fdffb84f9f35abcf49adc647 Mon Sep 17 00:00:00 2001 From: Kevin Fairise Date: Tue, 23 Apr 2024 11:16:55 +0200 Subject: [PATCH 17/19] gomodtidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 3744780c4..59acd3dbb 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/pulumi/pulumi-azure-native-sdk/network/v2 v2.30.0 github.com/pulumi/pulumi-azure-native-sdk/v2 v2.30.0 github.com/pulumi/pulumi-command/sdk v0.9.2 + github.com/pulumi/pulumi-docker/sdk/v4 v4.5.1 github.com/pulumi/pulumi-eks/sdk/v2 v2.2.1 github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.9.0 github.com/pulumi/pulumi-libvirt/sdk v0.4.4 @@ -103,7 +104,6 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 // indirect github.com/pulumi/esc v0.8.2 // indirect - github.com/pulumi/pulumi-docker/sdk/v4 v4.5.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect From 9f6a94ea7aff5cb7e2429f18c2013d3541462304 Mon Sep 17 00:00:00 2001 From: Kevin Fairise Date: Mon, 13 May 2024 09:32:38 +0200 Subject: [PATCH 18/19] Handle error --- components/datadog/fakeintake/docker.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/components/datadog/fakeintake/docker.go b/components/datadog/fakeintake/docker.go index bfc8244c8..dc57db190 100644 --- a/components/datadog/fakeintake/docker.go +++ b/components/datadog/fakeintake/docker.go @@ -1,7 +1,6 @@ package fakeintake import ( - "log" "net" "github.com/DataDog/test-infra-definitions/common/config" @@ -28,7 +27,7 @@ func NewLocalDockerFakeintake(e config.CommonEnvironment, resourceName string) ( return err } - localIP := getLocalIP() + localIP, err := getLocalIP() comp.Host = pulumi.Sprintf(localIP.String()) comp.Port = hostPort @@ -39,17 +38,17 @@ func NewLocalDockerFakeintake(e config.CommonEnvironment, resourceName string) ( }) } -func getLocalIP() net.IP { +func getLocalIP() (net.IP, error) { // Open a connection to an external valid URL to // get the local address from the connection instance // The URL does not need to exist conn, err := net.Dial("udp", "8.8.8.8:80") if err != nil { - log.Fatal(err) + return nil, err } defer conn.Close() localAddr := conn.LocalAddr().(*net.UDPAddr) - return localAddr.IP + return localAddr.IP, nil } From df37cf188ca41947e6208031a0f885f2e31c9e5f Mon Sep 17 00:00:00 2001 From: Kevin Fairise Date: Mon, 13 May 2024 11:19:55 +0200 Subject: [PATCH 19/19] Use new Env interface --- components/datadog/fakeintake/docker.go | 7 +++++-- components/kubernetes/kind.go | 8 ++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/components/datadog/fakeintake/docker.go b/components/datadog/fakeintake/docker.go index dc57db190..a105edcd3 100644 --- a/components/datadog/fakeintake/docker.go +++ b/components/datadog/fakeintake/docker.go @@ -11,10 +11,10 @@ import ( const hostPort = 30080 -func NewLocalDockerFakeintake(e config.CommonEnvironment, resourceName string) (*Fakeintake, error) { +func NewLocalDockerFakeintake(e config.Env, resourceName string) (*Fakeintake, error) { return components.NewComponent(e, resourceName, func(comp *Fakeintake) error { - _, err := docker.NewContainer(e.Ctx, e.CommonNamer.ResourceName("local-docker-container"), &docker.ContainerArgs{ + _, err := docker.NewContainer(e.Ctx(), e.CommonNamer().ResourceName("local-docker-container"), &docker.ContainerArgs{ Image: pulumi.String("public.ecr.aws/datadog/fakeintake:latest"), Ports: docker.ContainerPortArray{ &docker.ContainerPortArgs{ @@ -28,6 +28,9 @@ func NewLocalDockerFakeintake(e config.CommonEnvironment, resourceName string) ( } localIP, err := getLocalIP() + if err != nil { + return err + } comp.Host = pulumi.Sprintf(localIP.String()) comp.Port = hostPort diff --git a/components/kubernetes/kind.go b/components/kubernetes/kind.go index aadcee0f2..40c213f13 100644 --- a/components/kubernetes/kind.go +++ b/components/kubernetes/kind.go @@ -110,7 +110,7 @@ func NewKindCluster(env config.Env, vm *remote.Host, resourceName, kindClusterNa }, opts...) } -func NewLocalKindCluster(env config.CommonEnvironment, resourceName, kindClusterName string, kubeVersion string, opts ...pulumi.ResourceOption) (*Cluster, error) { +func NewLocalKindCluster(env config.Env, resourceName, kindClusterName string, kubeVersion string, opts ...pulumi.ResourceOption) (*Cluster, error) { return components.NewComponent(env, resourceName, func(clusterComp *Cluster) error { opts = utils.MergeOptions[pulumi.ResourceOption](opts, pulumi.Parent(clusterComp)) commonEnvironment := env @@ -135,9 +135,9 @@ func NewLocalKindCluster(env config.CommonEnvironment, resourceName, kindCluster return err } - nodeImage := fmt.Sprintf("%s:%s", kindNodeImageRegistry, kindVersionConfig.nodeImageVersion) + nodeImage := fmt.Sprintf("%s/%s:%s", env.InfraOSDescriptor(), kindNodeImageName, kindVersionConfig.nodeImageVersion) createCluster, err := runner.Command( - commonEnvironment.CommonNamer.ResourceName("kind-create-cluster", resourceName), + commonEnvironment.CommonNamer().ResourceName("kind-create-cluster", resourceName), &command.Args{ Create: pulumi.Sprintf("kind create cluster --name %s --config %s --image %s --wait %s", kindClusterName, clusterConfigFilePath, nodeImage, kindReadinessWait), Delete: pulumi.Sprintf("kind delete cluster --name %s", kindClusterName), @@ -150,7 +150,7 @@ func NewLocalKindCluster(env config.CommonEnvironment, resourceName, kindCluster } kubeConfigCmd, err := runner.Command( - commonEnvironment.CommonNamer.ResourceName("kind-kubeconfig", resourceName), + commonEnvironment.CommonNamer().ResourceName("kind-kubeconfig", resourceName), &command.Args{ Create: pulumi.Sprintf("kind get kubeconfig --name %s", kindClusterName), },