Skip to content

Commit

Permalink
feat: support for dynamically fetching credentials from external comm…
Browse files Browse the repository at this point in the history 
…and (#149)

* feat: support issuing cluster credential from external command

Signed-off-by: Claudio Netto <nettinhorama@gmail.com>

* refactor: rename the dynamic credential type

Signed-off-by: Claudio Netto <nettinhorama@gmail.com>

* refactor: covering dynamic credentials w/ unit tests

Signed-off-by: Claudio Netto <nettinhorama@gmail.com>

* docs: adding an example of secret using Dynamic cluster credential

Signed-off-by: Claudio Netto <nettinhorama@gmail.com>

* test: covering code branch of failed to build cluster credentials

Signed-off-by: Claudio Netto <nettinhorama@gmail.com>

* refactor: return ClusterGateway w/ credential type Dynamic

Signed-off-by: Claudio Netto <nettinhorama@gmail.com>

* feat: store/fetch credential from in-memory cache

Signed-off-by: Claudio Netto <nettinhorama@gmail.com>

* test: fixing exec plugin tests

Signed-off-by: Claudio Netto <nettinhorama@gmail.com>

* test: increasing test coverage of exec package

Signed-off-by: Claudio Netto <nettinhorama@gmail.com>

* test: cover transport package with tests about dynamic credential

Signed-off-by: Claudio Netto <nettinhorama@gmail.com>

* test: covering w/ test an error-catching

Signed-off-by: Claudio Netto <nettinhorama@gmail.com>

---------

Signed-off-by: Claudio Netto <nettinhorama@gmail.com>
  • Loading branch information
@nettoclaudio
nettoclaudio committed Mar 12, 2024
1 parent adc35a6 commit 48259e0
Show file tree
Hide file tree
Showing 8 changed files with 789 additions and 12 deletions.
17 changes: 16 additions & 1 deletion docs/local-run.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,21 @@ data:
token: "..." # working jwt token
```

2.3. (Alternatively) Create a secret containing an exec config to dynamically fetch the cluster credential from an external command:

```yaml
apiVersion: v1
kind: Secret
metadata:
name: managed1
labels:
cluster.core.oam.dev/cluster-credential-type: Dynamic
type: Opaque # <--- Has to be opaque
data:
endpoint: "..." # ditto
exec: "..." # an exec config in JSON format; see ExecConfig (https://github.com/kubernetes/kubernetes/blob/2016fab3085562b4132e6d3774b6ded5ba9939fd/staging/src/k8s.io/client-go/tools/clientcmd/api/types.go#L206, https://kubernetes.io/docs/reference/access-authn-authz/authentication/#configuration)
```

3. Proxy to cluster `managed1`'s `/healthz` endpoint

```shell
Expand Down Expand Up @@ -195,4 +210,4 @@ KUBECONFIG=/tmp/hub-managed1.kubeconfig kubectl get ns

```shell
$ kind delete cluster --name tmp
```
```
4 changes: 4 additions & 0 deletions pkg/apis/cluster/v1alpha1/clustergateway_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,10 @@ const (
// CredentialTypeX509Certificate means the cluster is accessible via
// X509 certificate and key.
CredentialTypeX509Certificate CredentialType = "X509Certificate"
// CredentialTypeDynamic means that a credential will be issued before
// accessing the cluster. The generated credential can be either a service
// account token or X509 certificate and key.
CredentialTypeDynamic CredentialType = "Dynamic"
)

type ClusterEndpointType string
Expand Down
58 changes: 54 additions & 4 deletions pkg/apis/cluster/v1alpha1/clustergateway_types_secret.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,21 @@ package v1alpha1

import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"

"k8s.io/apimachinery/pkg/util/yaml"
"k8s.io/apiserver/pkg/registry/rest"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
"k8s.io/utils/pointer"

"github.com/oam-dev/cluster-gateway/pkg/common"
"github.com/oam-dev/cluster-gateway/pkg/config"
"github.com/oam-dev/cluster-gateway/pkg/featuregates"
"github.com/oam-dev/cluster-gateway/pkg/options"
"github.com/oam-dev/cluster-gateway/pkg/util/exec"
"github.com/oam-dev/cluster-gateway/pkg/util/singleton"

"github.com/pkg/errors"
Expand All @@ -22,7 +26,6 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/registry/rest"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/klog/v2"
clusterv1 "open-cluster-management.io/api/cluster/v1"
Expand Down Expand Up @@ -176,11 +179,11 @@ func getEndpointFromSecret(secret *v1.Secret) ([]byte, string, error) {
func convert(caData []byte, apiServerEndpoint string, insecure bool, secret *v1.Secret) (*ClusterGateway, error) {
c := &ClusterGateway{
ObjectMeta: metav1.ObjectMeta{
Name: secret.Name,
Name: secret.Name,
CreationTimestamp: secret.CreationTimestamp,
},
Spec: ClusterGatewaySpec{
Provider: "",
Access: ClusterAccess{},
Access: ClusterAccess{},
},
}

Expand Down Expand Up @@ -242,11 +245,21 @@ func convert(caData []byte, apiServerEndpoint string, insecure bool, secret *v1.
PrivateKey: secret.Data[v1.TLSPrivateKeyKey],
},
}

case CredentialTypeServiceAccountToken:
c.Spec.Access.Credential = &ClusterAccessCredential{
Type: CredentialTypeServiceAccountToken,
ServiceAccountToken: string(secret.Data[v1.ServiceAccountTokenKey]),
}

case CredentialTypeDynamic:
credential, err := buildCredentialFromExecConfig(secret)
if err != nil {
return nil, fmt.Errorf("failed to issue credential from external command: %s", err)
}

c.Spec.Access.Credential = credential

default:
return nil, fmt.Errorf("unrecognized secret credential type %v", credentialType)
}
Expand Down Expand Up @@ -278,3 +291,40 @@ func convert(caData []byte, apiServerEndpoint string, insecure bool, secret *v1.

return c, nil
}

func buildCredentialFromExecConfig(secret *v1.Secret) (*ClusterAccessCredential, error) {
execConfigRaw := secret.Data["exec"]
if len(execConfigRaw) == 0 {
return nil, errors.New("missing secret data key: exec")
}

var ec clientcmdapi.ExecConfig
if err := json.Unmarshal(execConfigRaw, &ec); err != nil {
return nil, fmt.Errorf("failed to decode exec config JSON from secret data: %v", err)
}

cred, err := exec.IssueClusterCredential(secret.Name, &ec)
if err != nil {
return nil, err
}

if token := cred.Status.Token; len(token) > 0 {
return &ClusterAccessCredential{
Type: CredentialTypeDynamic,
ServiceAccountToken: token,
}, nil
}

if cert, key := cred.Status.ClientCertificateData, cred.Status.ClientKeyData; len(cert) > 0 && len(key) > 0 {
return &ClusterAccessCredential{
Type: CredentialTypeDynamic,
X509: &X509{
Certificate: []byte(cert),
PrivateKey: []byte(key),
},
}, nil

}

return nil, fmt.Errorf("no credential type available")
}
201 changes: 194 additions & 7 deletions pkg/apis/cluster/v1alpha1/clustergateway_types_secret_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,29 @@ import (
)

var (
testNamespace = "foo"
testName = "bar"
testCAData = "caData"
testCertData = "certData"
testKeyData = "keyData"
testToken = "token"
testEndpoint = "https://localhost:443"
testNamespace = "foo"
testName = "bar"
testCAData = "caData"
testCertData = "certData"
testKeyData = "keyData"
testToken = "token"
testEndpoint = "https://localhost:443"
testExecConfigForToken = `{
"apiVersion": "client.authentication.k8s.io/v1beta1",
"kind": "ExecConfig",
"command": "echo",
"args": [
"{\"apiVersion\": \"client.authentication.k8s.io/v1beta1\", \"kind\": \"ExecCredential\", \"status\": {\"token\": \"token\"}}"
]
}`
testExecConfigForX509 = `{
"apiVersion": "client.authentication.k8s.io/v1beta1",
"kind": "ExecConfig",
"command": "echo",
"args": [
"{\"apiVersion\": \"client.authentication.k8s.io/v1beta1\", \"kind\": \"ExecCredential\", \"status\": {\"clientCertificateData\": \"certData\", \"clientKeyData\": \"keyData\"}}"
]
}`
)

func TestConvertSecretToGateway(t *testing.T) {
Expand Down Expand Up @@ -260,6 +276,101 @@ func TestConvertSecretToGateway(t *testing.T) {
},
},
},
{
name: "dynamic service account token issued from external command",
inputSecret: &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: testName,
Namespace: testNamespace,
Labels: map[string]string{
common.LabelKeyClusterCredentialType: string(CredentialTypeDynamic),
},
},
Data: map[string][]byte{
"endpoint": []byte(testEndpoint),
"ca.crt": []byte(testCAData),
"exec": []byte(testExecConfigForToken),
},
},
expected: &ClusterGateway{
ObjectMeta: metav1.ObjectMeta{
Name: testName,
},
Spec: ClusterGatewaySpec{
Access: ClusterAccess{
Credential: &ClusterAccessCredential{
Type: CredentialTypeDynamic,
ServiceAccountToken: testToken,
},
Endpoint: &ClusterEndpoint{
Type: ClusterEndpointTypeConst,
Const: &ClusterEndpointConst{
CABundle: []byte(testCAData),
Address: testEndpoint,
},
},
},
},
},
},
{
name: "dynamic x509 cert-key pair issued from external command",
inputSecret: &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: testName,
Namespace: testNamespace,
Labels: map[string]string{
common.LabelKeyClusterCredentialType: string(CredentialTypeDynamic),
},
},
Data: map[string][]byte{
"endpoint": []byte(testEndpoint),
"ca.crt": []byte(testCAData),
"exec": []byte(testExecConfigForX509),
},
},
expected: &ClusterGateway{
ObjectMeta: metav1.ObjectMeta{
Name: testName,
},
Spec: ClusterGatewaySpec{
Access: ClusterAccess{
Credential: &ClusterAccessCredential{
Type: CredentialTypeDynamic,
X509: &X509{
Certificate: []byte(testCertData),
PrivateKey: []byte(testKeyData),
},
},
Endpoint: &ClusterEndpoint{
Type: ClusterEndpointTypeConst,
Const: &ClusterEndpointConst{
CABundle: []byte(testCAData),
Address: testEndpoint,
},
},
},
},
},
},
{
name: "failed to fetch cluster credential from dynamic auth mode",
inputSecret: &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: testName,
Namespace: testNamespace,
Labels: map[string]string{
common.LabelKeyClusterCredentialType: string(CredentialTypeDynamic),
},
},
Data: map[string][]byte{
"endpoint": []byte(testEndpoint),
"ca.crt": []byte(testCAData),
"exec": []byte("invalid exec config format"),
},
},
expectedFailure: true,
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
Expand Down Expand Up @@ -524,3 +635,79 @@ func TestListHybridClusterGateway(t *testing.T) {
}
assert.Equal(t, expectedNames, actualNames)
}

func TestBuildCredentialFromExecConfig(t *testing.T) {
cases := []struct {
name string
secret func(s *corev1.Secret) *corev1.Secret
cluster func(ce *ClusterEndpoint) *ClusterEndpoint
expectedError string
expected *ClusterAccessCredential
}{
{
name: "missing exec config",
expectedError: "missing secret data key: exec",
},

{
name: "invalid exec config format",
secret: func(s *corev1.Secret) *corev1.Secret {
s.Data["exec"] = []byte("some invalid exec config")
return s
},
expectedError: "failed to decode exec config JSON from secret data: invalid character 's' looking for beginning of value",
},

{
name: "returns successfully a service account token",
secret: func(s *corev1.Secret) *corev1.Secret {
s.Data["exec"] = []byte(`{"apiVersion": "client.authentication.k8s.io/v1", "command": "echo", "args": ["{\"apiVersion\": \"client.authentication.k8s.io/v1\", \"status\": {\"token\": \"token\"}}"]}`)
return s
},
expected: &ClusterAccessCredential{
Type: CredentialTypeDynamic,
ServiceAccountToken: testToken,
},
},

{
name: "returns successfully a X509 client certificate",
secret: func(s *corev1.Secret) *corev1.Secret {
s.Data["exec"] = []byte(`{"apiVersion": "client.authentication.k8s.io/v1", "command": "echo", "args": ["{\"apiVersion\": \"client.authentication.k8s.io/v1\", \"status\": {\"clientCertificateData\": \"certData\", \"clientKeyData\": \"keyData\"}}"]}`)
return s
},
expected: &ClusterAccessCredential{
Type: CredentialTypeDynamic,
X509: &X509{
Certificate: []byte(testCertData),
PrivateKey: []byte(testKeyData),
},
},
},
}

for _, tt := range cases {
t.Run(tt.name, func(t *testing.T) {
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: testName,
Namespace: testNamespace,
},
Data: map[string][]byte{},
}
if tt.secret != nil {
secret = tt.secret(secret)
}

got, err := buildCredentialFromExecConfig(secret)
if tt.expectedError != "" {
assert.Error(t, err)
assert.EqualError(t, err, tt.expectedError)
return
}

assert.NoError(t, err)
assert.Equal(t, tt.expected, got)
})
}
}
11 changes: 11 additions & 0 deletions pkg/apis/cluster/v1alpha1/transport.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,8 +94,19 @@ func NewConfigFromCluster(ctx context.Context, c *ClusterGateway) (*restclient.C
}
// setting up credentials
switch c.Spec.Access.Credential.Type {
case CredentialTypeDynamic:
if token := c.Spec.Access.Credential.ServiceAccountToken; token != "" {
cfg.BearerToken = token
}

if c.Spec.Access.Credential.X509 != nil && len(c.Spec.Access.Credential.X509.Certificate) > 0 && len(c.Spec.Access.Credential.X509.PrivateKey) > 0 {
cfg.CertData = c.Spec.Access.Credential.X509.Certificate
cfg.KeyData = c.Spec.Access.Credential.X509.PrivateKey
}

case CredentialTypeServiceAccountToken:
cfg.BearerToken = c.Spec.Access.Credential.ServiceAccountToken

case CredentialTypeX509Certificate:
cfg.CertData = c.Spec.Access.Credential.X509.Certificate
cfg.KeyData = c.Spec.Access.Credential.X509.PrivateKey
Expand Down

0 comments on commit 48259e0

Please sign in to comment.