Skip to content

Retry when getting the pod_environment_secret #1777

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Mar 1, 2022
Merged

Retry when getting the pod_environment_secret #1777

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8e362b8
Retry when getting the pod_environment_secret
snarlysodboxer Feb 10, 2022
6635227
Variablize testing values
snarlysodboxer Feb 28, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions docs/developer.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,13 @@ dlv connect 127.0.0.1:DLV_PORT

## Unit tests

Prerequisites:

```bash
make deps
make mocks
```

To run all unit tests, you can simply do:

```bash
Expand Down
31 changes: 26 additions & 5 deletions pkg/cluster/k8sres.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,13 @@ import (
"sort"
"strings"

"github.com/pkg/errors"
"github.com/sirupsen/logrus"

appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
policybeta1 "k8s.io/api/policy/v1beta1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
Expand All @@ -24,6 +26,7 @@ import (
"github.com/zalando/postgres-operator/pkg/util/config"
"github.com/zalando/postgres-operator/pkg/util/constants"
"github.com/zalando/postgres-operator/pkg/util/k8sutil"
"github.com/zalando/postgres-operator/pkg/util/retryutil"
batchv1 "k8s.io/api/batch/v1"
batchv1beta1 "k8s.io/api/batch/v1beta1"
"k8s.io/apimachinery/pkg/labels"
Expand Down Expand Up @@ -893,12 +896,30 @@ func (c *Cluster) getPodEnvironmentSecretVariables() ([]v1.EnvVar, error) {
return secretPodEnvVarsList, nil
}

secret, err := c.KubeClient.Secrets(c.Namespace).Get(
context.TODO(),
c.OpConfig.PodEnvironmentSecret,
metav1.GetOptions{})
secret := &v1.Secret{}
var notFoundErr error
err := retryutil.Retry(c.OpConfig.ResourceCheckInterval, c.OpConfig.ResourceCheckTimeout,
func() (bool, error) {
var err error
secret, err = c.KubeClient.Secrets(c.Namespace).Get(
context.TODO(),
c.OpConfig.PodEnvironmentSecret,
metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
notFoundErr = err
return false, nil
}
return false, err
}
return true, nil
},
)
if notFoundErr != nil && err != nil {
err = errors.Wrap(notFoundErr, err.Error())
}
if err != nil {
return nil, fmt.Errorf("could not read Secret PodEnvironmentSecretName: %v", err)
return nil, errors.Wrap(err, "could not read Secret PodEnvironmentSecretName")
}

for k := range secret.Data {
Expand Down
40 changes: 33 additions & 7 deletions pkg/cluster/k8sres_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"fmt"
"reflect"
"sort"
"time"

"testing"

Expand All @@ -21,8 +22,10 @@ import (
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
policyv1beta1 "k8s.io/api/policy/v1beta1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes/fake"
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
Expand Down Expand Up @@ -639,8 +642,12 @@ func TestSecretVolume(t *testing.T) {
}

const (
testPodEnvironmentConfigMapName = "pod_env_cm"
testPodEnvironmentSecretName = "pod_env_sc"
testPodEnvironmentConfigMapName = "pod_env_cm"
testPodEnvironmentSecretName = "pod_env_sc"
testPodEnvironmentObjectNotExists = "idonotexist"
testPodEnvironmentSecretNameAPIError = "pod_env_sc_apierror"
testResourceCheckInterval = 3
testResourceCheckTimeout = 10
)

type mockSecret struct {
Expand All @@ -652,8 +659,11 @@ type mockConfigMap struct {
}

func (c *mockSecret) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.Secret, error) {
if name == testPodEnvironmentSecretNameAPIError {
return nil, fmt.Errorf("Secret PodEnvironmentSecret API error")
}
if name != testPodEnvironmentSecretName {
return nil, fmt.Errorf("Secret PodEnvironmentSecret not found")
return nil, k8serrors.NewNotFound(schema.GroupResource{Group: "core", Resource: "secret"}, name)
}
secret := &v1.Secret{}
secret.Name = testPodEnvironmentSecretName
Expand Down Expand Up @@ -722,7 +732,7 @@ func TestPodEnvironmentConfigMapVariables(t *testing.T) {
opConfig: config.Config{
Resources: config.Resources{
PodEnvironmentConfigMap: spec.NamespacedName{
Name: "idonotexist",
Name: testPodEnvironmentObjectNotExists,
},
},
},
Expand Down Expand Up @@ -773,6 +783,7 @@ func TestPodEnvironmentConfigMapVariables(t *testing.T) {

// Test if the keys of an existing secret are properly referenced
func TestPodEnvironmentSecretVariables(t *testing.T) {
maxRetries := int(testResourceCheckTimeout / testResourceCheckInterval)
testName := "TestPodEnvironmentSecretVariables"
tests := []struct {
subTest string
Expand All @@ -788,16 +799,31 @@ func TestPodEnvironmentSecretVariables(t *testing.T) {
subTest: "Secret referenced by PodEnvironmentSecret does not exist",
opConfig: config.Config{
Resources: config.Resources{
PodEnvironmentSecret: "idonotexist",
PodEnvironmentSecret: testPodEnvironmentObjectNotExists,
ResourceCheckInterval: time.Duration(testResourceCheckInterval),
ResourceCheckTimeout: time.Duration(testResourceCheckTimeout),
},
},
err: fmt.Errorf("could not read Secret PodEnvironmentSecretName: still failing after %d retries: secret.core %q not found", maxRetries, testPodEnvironmentObjectNotExists),
},
{
subTest: "API error during PodEnvironmentSecret retrieval",
opConfig: config.Config{
Resources: config.Resources{
PodEnvironmentSecret: testPodEnvironmentSecretNameAPIError,
ResourceCheckInterval: time.Duration(testResourceCheckInterval),
ResourceCheckTimeout: time.Duration(testResourceCheckTimeout),
},
},
err: fmt.Errorf("could not read Secret PodEnvironmentSecretName: Secret PodEnvironmentSecret not found"),
err: fmt.Errorf("could not read Secret PodEnvironmentSecretName: Secret PodEnvironmentSecret API error"),
},
{
subTest: "Pod environment vars reference all keys from secret configured by PodEnvironmentSecret",
opConfig: config.Config{
Resources: config.Resources{
PodEnvironmentSecret: testPodEnvironmentSecretName,
PodEnvironmentSecret: testPodEnvironmentSecretName,
ResourceCheckInterval: time.Duration(testResourceCheckInterval),
ResourceCheckTimeout: time.Duration(testResourceCheckTimeout),
},
},
envVars: []v1.EnvVar{
Expand Down