Skip to content

Commit

Permalink
Get registry from v2prov even without a secret (rancher#42774)
Browse files Browse the repository at this point in the history 
* ger private from v2 even without a secret

* fix comment

* change names

* change names

* add test for v2 prov with upstream registry

* add cast to fallback to the upstream registry

* improve test name

* fix lint from AMD64

* fix lint

* rename tests and remove comments

* improve logic comments

* Apply suggestions from code review

Co-authored-by: Jake Hyde <33796120+jakefhyde@users.noreply.github.com>

---------

Co-authored-by: Jake Hyde <33796120+jakefhyde@users.noreply.github.com>
  • Loading branch information
@felipe-colussi @jakefhyde
felipe-colussi and jakefhyde committed Sep 19, 2023
1 parent efc48ac commit 385b097
Show file tree
Hide file tree
Showing 2 changed files with 113 additions and 27 deletions.
67 changes: 40 additions & 27 deletions pkg/cluster/private_registry.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,9 @@ func GetPrivateClusterLevelRegistry(cluster *v3.Cluster) *rketypes.PrivateRegist
// clusters, as the function will reassemble them anyway.
func GeneratePrivateRegistryEncodedDockerConfig(cluster *v3.Cluster, secretLister v1.SecretLister) (string, string, error) {
var err error
// Declare here so we don't need to check if the rkeClusterRegistryOrGlobalSystemDefault exists while working with v2prov
var rkeClusterURLOrGlobalSystemDefault string

if cluster == nil {
return "", "", nil
}
Expand All @@ -80,37 +83,39 @@ func GeneratePrivateRegistryEncodedDockerConfig(cluster *v3.Cluster, secretListe
return "", "", err
}

// The PrivateRegistrySecret have the same name both for v1 or v2 provisioning clusters despite having different structures
registrySecretName := cluster.GetSecret(v3.ClusterPrivateRegistrySecret)

// Private registry will only be defined on the cluster if it is an RKE1 cluster, mgmt clusters generated from
// provisioning clusters do not have a populated `RancherKubernetesEngineConfig`.
if registry := GetPrivateRegistry(cluster); registry != nil {
if rkeClusterRegistryOrGlobalSystemDefault := GetPrivateRegistry(cluster); rkeClusterRegistryOrGlobalSystemDefault != nil {
rkeClusterURLOrGlobalSystemDefault = rkeClusterRegistryOrGlobalSystemDefault.URL
// check for RKE1 ECR credentials first
if registry.ECRCredentialPlugin != nil {
if rkeClusterRegistryOrGlobalSystemDefault.ECRCredentialPlugin != nil {
// generate ecr authConfig
authConfig, err := util.ECRCredentialPlugin(registry.ECRCredentialPlugin, registry.URL)
authConfig, err := util.ECRCredentialPlugin(rkeClusterRegistryOrGlobalSystemDefault.ECRCredentialPlugin, rkeClusterRegistryOrGlobalSystemDefault.URL)
if err != nil {
return registry.URL, "", err
return rkeClusterRegistryOrGlobalSystemDefault.URL, "", err
}
encodedJSON, err := json.Marshal(authConfig)
if err != nil {
return registry.URL, "", err
return rkeClusterRegistryOrGlobalSystemDefault.URL, "", err
}
return registry.URL, base64.StdEncoding.EncodeToString(encodedJSON), nil
}

// no private registry secret, generate authconfig based on existing fields
if registrySecretName == "" {
return registry.URL, "", nil
return rkeClusterRegistryOrGlobalSystemDefault.URL, base64.StdEncoding.EncodeToString(encodedJSON), nil
}

// check for the RKE1 registry secret next
registrySecret, err := secretLister.Get(namespace.GlobalNamespace, registrySecretName)
if err == nil {
return registry.URL, base64.StdEncoding.EncodeToString(registrySecret.Data[corev1.DockerConfigJsonKey]), nil
}
if err != nil && !apierrors.IsNotFound(err) { // ignore secret not found errors as we need to check v2prov clusters
return registry.URL, "", err
// If we have a Secret we try to check the rke1 provisioning, otherwise we go directly to the v2prov check.
// This is done this way to always check if there is a downstream registry.
if registrySecretName != "" {
// check for the RKE1 registry secret, returning it if it exists.
registrySecret, err := secretLister.Get(namespace.GlobalNamespace, registrySecretName)
if err == nil {
return rkeClusterRegistryOrGlobalSystemDefault.URL, base64.StdEncoding.EncodeToString(registrySecret.Data[corev1.DockerConfigJsonKey]), nil
}
// If it doesn't exist (secret not found error) we need to check for a v2prov cluster.
if err != nil && !apierrors.IsNotFound(err) {
return rkeClusterRegistryOrGlobalSystemDefault.URL, "", err
}
}
}

Expand All @@ -119,25 +124,33 @@ func GeneratePrivateRegistryEncodedDockerConfig(cluster *v3.Cluster, secretListe
// For RKE2 with a cluster level registry configured, this is the
// only reference to the registry URL available on the v3.Cluster.
// Without it, we cannot generate the registry credentials (.dockerconfigjson)
registryURL := cluster.GetSecret(v3.ClusterPrivateRegistryURL)
if registryURL == "" {
return "", "", nil
v2ProvRegistryURL := cluster.GetSecret(v3.ClusterPrivateRegistryURL)
// At this point we know that we don't have a RKE1 registry with authentication
// if we don't get a v2ProvRegistryURL we can just return the image set on v1 Prov or the global system default one.
if v2ProvRegistryURL == "" {
return rkeClusterURLOrGlobalSystemDefault, "", nil
}

if registrySecretName == "" { // no private registry configured
return registryURL, "", nil
// If we reach this point we know that we have a registry URL set on the v2prov downstream cluster.
// If it is a rke1 cluster that requires an authorization, a rke1 cluster without authorization or a v2prov cluster
// without a registry URL the function would have already returned.
// This last check is to see if the registry requires an authorization, if it doesn't we just return the v2ProvRegistryURL.
if registrySecretName == "" {
return v2ProvRegistryURL, "", nil
}

// If we have a registrySecretName (registry requires authentication) and this function reached this point
// it is a v2 prov cluster. We need to decode that information to return it.
registrySecret, err := secretLister.Get(cluster.Spec.FleetWorkspaceName, registrySecretName)
if err != nil {
return registryURL, "", err
return v2ProvRegistryURL, "", err
}

username := string(registrySecret.Data["username"])
password := string(registrySecret.Data["password"])
authConfig := credentialprovider.DockerConfigJSON{
Auths: credentialprovider.DockerConfig{
registryURL: credentialprovider.DockerConfigEntry{
v2ProvRegistryURL: credentialprovider.DockerConfigEntry{
Username: username,
Password: password,
},
Expand All @@ -146,8 +159,8 @@ func GeneratePrivateRegistryEncodedDockerConfig(cluster *v3.Cluster, secretListe

registryJSON, err := json.Marshal(authConfig)
if err != nil {
return registryURL, "", err
return v2ProvRegistryURL, "", err
}

return registryURL, base64.StdEncoding.EncodeToString(registryJSON), nil
return v2ProvRegistryURL, base64.StdEncoding.EncodeToString(registryJSON), nil
}
73 changes: 73 additions & 0 deletions pkg/cluster/private_registry_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,79 @@ func TestGeneratePrivateRegistryDockerConfig(t *testing.T) {
},
},
},
{
name: "global system default registry and no cluster default registry with secret",
expectedUrl: "0123456789abcdef.dkr.ecr.us-east-1.amazonaws.com",
expectedConfig: base64.StdEncoding.EncodeToString([]byte(`{"auths":{"0123456789abcdef.dkr.ecr.us-east-1.amazonaws.com":{"username":"testuser","password":"password","auth":"dGVzdHVzZXI6cGFzc3dvcmQ="}}}`)),
expectedError: "",
cluster: &v3.Cluster{
Spec: v3.ClusterSpec{
ClusterSpecBase: v3.ClusterSpecBase{
RancherKubernetesEngineConfig: &rketypes.RancherKubernetesEngineConfig{
PrivateRegistries: []rketypes.PrivateRegistry{{
URL: "upstream-registry.com",
}},
},
ClusterSecrets: v3.ClusterSecrets{
PrivateRegistrySecret: "test-secret",
PrivateRegistryURL: "0123456789abcdef.dkr.ecr.us-east-1.amazonaws.com",
},
},
FleetWorkspaceName: "fleet-default",
},
},
secrets: []*corev1.Secret{
{
ObjectMeta: metav1.ObjectMeta{
Namespace: "fleet-default",
Name: "test-secret",
},
Data: map[string][]byte{
"username": []byte("testuser"),
"password": []byte("password"),
},
},
},
},
{
name: "global system default registry and cluster default registry without secret",
expectedUrl: "0123456789abcdef.dkr.ecr.us-east-1.amazonaws.com",
expectedConfig: "",
expectedError: "",
cluster: &v3.Cluster{
Spec: v3.ClusterSpec{
ClusterSpecBase: v3.ClusterSpecBase{
RancherKubernetesEngineConfig: &rketypes.RancherKubernetesEngineConfig{
PrivateRegistries: []rketypes.PrivateRegistry{{
URL: "upstream-registry.com",
}},
},
ClusterSecrets: v3.ClusterSecrets{
PrivateRegistryURL: "0123456789abcdef.dkr.ecr.us-east-1.amazonaws.com",
},
},
FleetWorkspaceName: "fleet-default",
},
},
},
{
name: "global system default registry and no cluster default registry",
expectedUrl: "0123456789abcdef.dkr.ecr.us-east-1.amazonaws.com",
expectedConfig: "",
expectedError: "",
cluster: &v3.Cluster{
Spec: v3.ClusterSpec{
ClusterSpecBase: v3.ClusterSpecBase{
RancherKubernetesEngineConfig: &rketypes.RancherKubernetesEngineConfig{
PrivateRegistries: []rketypes.PrivateRegistry{{
URL: "0123456789abcdef.dkr.ecr.us-east-1.amazonaws.com",
}},
},
},
FleetWorkspaceName: "fleet-default",
},
},
},
}

for _, tt := range tests {
Expand Down

0 comments on commit 385b097

Please sign in to comment.