Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

use Pull through cache #763

Open
wants to merge 12 commits into
base: main
Choose a base branch
from
Open

use Pull through cache #763

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
34272c3
use Pull through cache
KevinFairise2 Apr 15, 2024
e462c07
Fix typo
KevinFairise2 Apr 15, 2024
b2fbdac
Put dockerhub in image url
KevinFairise2 Apr 15, 2024
7296dbe
Create new method to get internal dockerhub registry
KevinFairise2 Apr 15, 2024
b056535
Fix method
KevinFairise2 Apr 15, 2024
aa0aa65
Default to public Dockerhub
KevinFairise2 Apr 16, 2024
0585474
WIP
KevinFairise2 Apr 16, 2024
6995fc2
Handle authentication on kind scenario
KevinFairise2 Apr 17, 2024
d414c64
Remove useless code
KevinFairise2 Apr 17, 2024
bbc1c49
Please be happy little linter
KevinFairise2 Apr 17, 2024
d6f794d
Update components/kubernetes/kind-cluster.yaml
KevinFairise2 Apr 17, 2024
b35bd0d
Update components/kubernetes/kind.go
KevinFairise2 Apr 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions common/config/environment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ type CommonEnvironment struct {

type CloudProviderEnvironment interface {
InternalRegistry() string
InternalDockerhubMirror() string
}

func NewCommonEnvironment(ctx *pulumi.Context, cloudProviderEnvironment CloudProviderEnvironment) (CommonEnvironment, error) {
Expand Down
4 changes: 3 additions & 1 deletion components/datadog/apps/mutatedbyadmissioncontroller/k8s.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package mutatedbyadmissioncontroller

import (
"fmt"

"github.com/DataDog/test-infra-definitions/common/config"

componentskube "github.com/DataDog/test-infra-definitions/components/kubernetes"
Expand Down Expand Up @@ -131,7 +133,7 @@ func k8sDeploymentWithLibInjection(e config.CommonEnvironment, namespace string,
corev1.ContainerArgs{
Name: pulumi.String(name),
// Python is one of the languages supported by APM lib injection
Image: pulumi.String("python:3.12-slim"),
Image: pulumi.String(fmt.Sprintf("%s/library/python:3.12-slim", e.CloudProviderEnvironment.InternalDockerhubMirror())),
Command: pulumi.ToStringArray([]string{
"python", "-c", "while True: import time; time.sleep(60)",
}),
Expand Down
4 changes: 4 additions & 0 deletions components/kubernetes/kind-cluster.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@ kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
extraMounts:
- containerPath: /var/lib/kubelet/config.json
hostPath: /tmp/kind-config/config.json
propagation: HostToContainer
containerdConfigPatches:
- |-
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
Expand Down
5 changes: 5 additions & 0 deletions resources/aws/environment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ const (
DDInfraDefaultInstanceStorageSize = "aws/defaultInstanceStorageSize"
DDInfraDefaultShutdownBehavior = "aws/defaultShutdownBehavior"
DDInfraDefaultInternalRegistry = "aws/defaultInternalRegistry"
DDInfraDefaultInternalDockerhubMirror = "aws/defaultInternalDockerhubMirror"

// AWS ECS
DDInfraEcsExecKMSKeyID = "aws/ecs/execKMSKeyID"
Expand Down Expand Up @@ -124,6 +125,10 @@ func (e *Environment) InternalRegistry() string {
return e.GetStringWithDefault(e.InfraConfig, DDInfraDefaultInternalRegistry, e.envDefault.ddInfra.defaultInternalRegistry)
}

func (e *Environment) InternalDockerhubMirror() string {
return e.GetStringWithDefault(e.InfraConfig, DDInfraDefaultInternalDockerhubMirror, e.envDefault.ddInfra.defaultInternalDockerhubMirror)
}

// Common
func (e *Environment) Region() string {
return e.GetStringWithDefault(e.awsConfig, awsRegionParamName, e.envDefault.aws.region)
Expand Down
76 changes: 40 additions & 36 deletions resources/aws/environmentDefaults.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,16 @@ type awsProvider struct {
}

type ddInfra struct {
defaultVPCID string
defaultSubnets []string
defaultSecurityGroups []string
defaultInstanceType string
defaultInstanceProfileName string
defaultARMInstanceType string
defaultInstanceStorageSize int
defaultShutdownBehavior string
defaultInternalRegistry string
defaultVPCID string
defaultSubnets []string
defaultSecurityGroups []string
defaultInstanceType string
defaultInstanceProfileName string
defaultARMInstanceType string
defaultInstanceStorageSize int
defaultShutdownBehavior string
defaultInternalRegistry string
defaultInternalDockerhubMirror string

ecs ddInfraECS
eks ddInfraEKS
Expand Down Expand Up @@ -82,15 +83,16 @@ func sandboxDefault() environmentDefault {
region: string(aws.RegionUSEast1),
},
ddInfra: ddInfra{
defaultVPCID: "vpc-d1aac1a8",
defaultSubnets: []string{"subnet-b89e00e2", "subnet-8ee8b1c6", "subnet-3f5db45b"},
defaultSecurityGroups: []string{"sg-46506837", "sg-7fedd80a", "sg-0e952e295ab41e748"},
defaultInstanceType: "t3.medium",
defaultInstanceProfileName: "ec2InstanceRole",
defaultARMInstanceType: "t4g.medium",
defaultInstanceStorageSize: 200,
defaultShutdownBehavior: "stop",
defaultInternalRegistry: "669783387624.dkr.ecr.us-east-1.amazonaws.com",
defaultVPCID: "vpc-d1aac1a8",
defaultSubnets: []string{"subnet-b89e00e2", "subnet-8ee8b1c6", "subnet-3f5db45b"},
defaultSecurityGroups: []string{"sg-46506837", "sg-7fedd80a", "sg-0e952e295ab41e748"},
defaultInstanceType: "t3.medium",
defaultInstanceProfileName: "ec2InstanceRole",
defaultARMInstanceType: "t4g.medium",
defaultInstanceStorageSize: 200,
defaultShutdownBehavior: "stop",
defaultInternalRegistry: "669783387624.dkr.ecr.us-east-1.amazonaws.com",
defaultInternalDockerhubMirror: "669783387624.dkr.ecr.us-east-1.amazonaws.com/dockerhub",

ecs: ddInfraECS{
execKMSKeyID: "arn:aws:kms:us-east-1:601427279990:key/c84f93c2-a562-4a59-a326-918fbe7235c7",
Expand Down Expand Up @@ -123,15 +125,16 @@ func agentSandboxDefault() environmentDefault {
region: string(aws.RegionUSEast1),
},
ddInfra: ddInfra{
defaultVPCID: "vpc-029c0faf8f49dee8d",
defaultSubnets: []string{"subnet-0a15f3482cd3f9820", "subnet-091570395d476e9ce", "subnet-003831c49a10df3dd"},
defaultSecurityGroups: []string{"sg-038231b976eb13d44", "sg-05466e7ce253d21b1"},
defaultInstanceType: "t3.medium",
defaultInstanceProfileName: "ec2InstanceRole",
defaultARMInstanceType: "t4g.medium",
defaultInstanceStorageSize: 200,
defaultShutdownBehavior: "stop",
defaultInternalRegistry: "669783387624.dkr.ecr.us-east-1.amazonaws.com",
defaultVPCID: "vpc-029c0faf8f49dee8d",
defaultSubnets: []string{"subnet-0a15f3482cd3f9820", "subnet-091570395d476e9ce", "subnet-003831c49a10df3dd"},
defaultSecurityGroups: []string{"sg-038231b976eb13d44", "sg-05466e7ce253d21b1"},
defaultInstanceType: "t3.medium",
defaultInstanceProfileName: "ec2InstanceRole",
defaultARMInstanceType: "t4g.medium",
defaultInstanceStorageSize: 200,
defaultShutdownBehavior: "stop",
defaultInternalRegistry: "669783387624.dkr.ecr.us-east-1.amazonaws.com",
defaultInternalDockerhubMirror: "669783387624.dkr.ecr.us-east-1.amazonaws.com/dockerhub",

ecs: ddInfraECS{
execKMSKeyID: "arn:aws:kms:us-east-1:376334461865:key/1d1fe533-a4f1-44ee-99ec-225b44fcb9ed",
Expand Down Expand Up @@ -166,15 +169,16 @@ func agentQADefault() environmentDefault {
region: string(aws.RegionUSEast1),
},
ddInfra: ddInfra{
defaultVPCID: "vpc-0097b9307ec2c8139",
defaultSubnets: []string{"subnet-0f1ca3e929eb3fb8b", "subnet-03061a1647c63c3c3", "subnet-071213aedb0e1ae54"},
defaultSecurityGroups: []string{"sg-05e9573fcc582f22c", "sg-0498c960a173dff1e"},
defaultInstanceType: "t3.medium",
defaultInstanceProfileName: "ec2InstanceRole",
defaultARMInstanceType: "t4g.medium",
defaultInstanceStorageSize: 200,
defaultShutdownBehavior: "stop",
defaultInternalRegistry: "669783387624.dkr.ecr.us-east-1.amazonaws.com",
defaultVPCID: "vpc-0097b9307ec2c8139",
defaultSubnets: []string{"subnet-0f1ca3e929eb3fb8b", "subnet-03061a1647c63c3c3", "subnet-071213aedb0e1ae54"},
defaultSecurityGroups: []string{"sg-05e9573fcc582f22c", "sg-0498c960a173dff1e"},
defaultInstanceType: "t3.medium",
defaultInstanceProfileName: "ec2InstanceRole",
defaultARMInstanceType: "t4g.medium",
defaultInstanceStorageSize: 200,
defaultShutdownBehavior: "stop",
defaultInternalRegistry: "669783387624.dkr.ecr.us-east-1.amazonaws.com",
defaultInternalDockerhubMirror: "669783387624.dkr.ecr.us-east-1.amazonaws.com/dockerhub",

ecs: ddInfraECS{
execKMSKeyID: "arn:aws:kms:us-east-1:669783387624:key/384373bc-6d99-4d68-84b5-b76b756b0af3",
Expand Down
4 changes: 4 additions & 0 deletions resources/azure/environment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,10 @@ func (e *Environment) InternalRegistry() string {
return "none"
}

func (e *Environment) InternalDockerhubMirror() string {
return "registry-1.docker.io"
}

// Common
func (e *Environment) DefaultResourceGroup() string {
return e.GetStringWithDefault(e.InfraConfig, DDInfraDefaultResourceGroup, e.envDefault.ddInfra.defaultResourceGroup)
Expand Down
4 changes: 4 additions & 0 deletions resources/hyperv/environment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@ func (e *Environment) InternalRegistry() string {
return "none"
}

func (e *Environment) InternalDockerhubMirror() string {
return "registry-1.docker.io"
}

// Common
func (e *Environment) DefaultPublicKeyPath() string {
return e.InfraConfig.Get(DDInfraDefaultPublicKeyPath)
Expand Down
51 changes: 50 additions & 1 deletion scenarios/aws/kindvm/run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"

"github.com/DataDog/test-infra-definitions/common/utils"
"github.com/DataDog/test-infra-definitions/components/command"
"github.com/DataDog/test-infra-definitions/components/datadog/agent"
"github.com/DataDog/test-infra-definitions/components/datadog/apps/cpustress"
"github.com/DataDog/test-infra-definitions/components/datadog/apps/dogstatsd"
Expand All @@ -14,12 +15,16 @@ import (
"github.com/DataDog/test-infra-definitions/components/datadog/apps/tracegen"
dogstatsdstandalone "github.com/DataDog/test-infra-definitions/components/datadog/dogstatsd-standalone"
fakeintakeComp "github.com/DataDog/test-infra-definitions/components/datadog/fakeintake"
"github.com/DataDog/test-infra-definitions/components/docker"
localKubernetes "github.com/DataDog/test-infra-definitions/components/kubernetes"
"github.com/DataDog/test-infra-definitions/components/os"
"github.com/DataDog/test-infra-definitions/components/remote"
resAws "github.com/DataDog/test-infra-definitions/resources/aws"
"github.com/DataDog/test-infra-definitions/scenarios/aws/ec2"
"github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake"

goremote "github.com/pulumi/pulumi-command/sdk/go/command/remote"

"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
Expand All @@ -39,9 +44,20 @@ func Run(ctx *pulumi.Context) error {
return err
}

// Install docker if not installed yet, we need it to configure docker credentials
_, dockerInstallCmd, err := docker.NewManager(*awsEnv.CommonEnvironment, vm)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We need docker installed before running ConfigureECRCredentials otherwise docker login fails.
This is not ideal because the NewKindCluster already installs docker but it works because docker installation is idempotent.

We could do the installation of docker oustide of NewKindCluster or add a hook to configure the credentials after docker is installed. The latter option would be easier if all the Environment to implement a common interface (#688)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's fine, I am wondering if we should have it inside the NewKindCluster component, as it needs it. Can be done later

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would have been easier, but we do not want to put AWS specific logic in the NewKindCluster component. That could be used on any cloud provider

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We'll need to duplicate this code in datadog-agent with current code. However you could have a WithECRCredentials that implements this. Note the ExtraMount you're adding means that it will currently fail when not using any credential helper. You may need to always create a empty {} JSON file.

You also don't need docker login, what docker login does is very basic JSON gen. The way I see it the clean way would be to build a small Go code that imports https://github.com/awslabs/amazon-ecr-credential-helper/blob/main/ecr-login/ecr.go#L83-L110 and produces the config.json on stdout.

It's much smaller and faster to download then the AWS SDK and you have the option to package as a docker container and get the config.json with docker run --network host <your_helper_image> > config-tmp.json.

if err != nil {
return err
}
// Configure ECR credentials for use in Kind
ecrLoginCommand, err := ConfigureECRCredentials(awsEnv, vm, osDesc.Architecture, utils.PulumiDependsOn(dockerInstallCmd))
if err != nil {
return err
}

kindClusterName := ctx.Stack()

kindCluster, err := localKubernetes.NewKindCluster(*awsEnv.CommonEnvironment, vm, awsEnv.CommonNamer.ResourceName("kind"), kindClusterName, awsEnv.KubernetesVersion())
kindCluster, err := localKubernetes.NewKindCluster(*awsEnv.CommonEnvironment, vm, awsEnv.CommonNamer.ResourceName("kind"), kindClusterName, awsEnv.KubernetesVersion(), utils.PulumiDependsOn(ecrLoginCommand))
if err != nil {
return err
}
Expand Down Expand Up @@ -154,3 +170,36 @@ agents:

return nil
}

func ConfigureECRCredentials(e resAws.Environment, vm *remote.Host, arch os.Architecture, opts ...pulumi.ResourceOption) (*goremote.Command, error) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💬 suggestion
If this is meant to be used from datadog-agent, could be moved to docker or some helpers package, rather than scenarios.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since it is something specific to AWS I did not want to have it in components/docker that should remain cloud agnostic. But yes we can probably find a better place for it

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe resources/aws/ecr

architecture := "x86_64"
if arch == os.ARM64Arch {
architecture = "aarch64"
}

unzipInstallCommand, err := vm.OS.PackageManager().Ensure("unzip", nil, "", opts...)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💭 thought
Unrelated, noticing now that calls to Ensure do not pin versions, might be something we work on

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think replacing unzip by unzip==<version> could work. But would need to be tested

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am not sure there is an added value in pinning versions, for two reasons:

  • Versions are usually pinned by distro release (normally only minor/security changes).
  • We don't pin AMIs and in most cases we don't even pin OS version.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We do pin some AMIs, for example for the installer tests. We don't probably want to maintain all images and versions, but for folks who would like to, we might want to offer the option. No one ever asked for it, just noticing it now.

if err != nil {
return nil, err
}

awsCliInstallCommand, err := vm.OS.Runner().Command(
e.CommonNamer.ResourceName("aws-cli-install"),
&command.Args{
Create: pulumi.Sprintf("command -v aws || curl 'https://awscli.amazonaws.com/awscli-exe-linux-%s.zip' -o 'awscliv2.zip' && unzip awscliv2.zip && sudo ./aws/install", architecture),
},
utils.PulumiDependsOn(unzipInstallCommand),
)
if err != nil {
return nil, err
}

ecrLoginCommand, err := vm.OS.Runner().Command(
e.CommonNamer.ResourceName("ecr-login"),
&command.Args{
Create: pulumi.Sprintf("aws ecr get-login-password | docker --config /tmp/kind-config login --username AWS --password-stdin %s", e.CloudProviderEnvironment.InternalRegistry()),
},
utils.PulumiDependsOn(awsCliInstallCommand),
)
pducolin marked this conversation as resolved.
Show resolved Hide resolved

return ecrLoginCommand, err
}
Loading