Skip to content

Commit

Permalink
test/e2e: add support for dynamic users using client certs
Browse files Browse the repository at this point in the history
  • Loading branch information
@s-urbaniak
s-urbaniak committed Jan 25, 2023
1 parent 7b3126c commit 57f8858
Show file tree
Hide file tree
Showing 9 changed files with 209 additions and 90 deletions.
18 changes: 17 additions & 1 deletion cmd/test-server/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (

genericapiserver "k8s.io/apiserver/pkg/server"

"github.com/kcp-dev/kcp/cmd/sharded-test-server/third_party/library-go/crypto"
shard "github.com/kcp-dev/kcp/cmd/test-server/kcp"
)

Expand Down Expand Up @@ -79,12 +80,27 @@ func start(shardFlags []string, quiet bool) error {
ctx, cancelFn := context.WithCancel(genericapiserver.SetupSignalContext())
defer cancelFn()

//create client CA and kcp-admin client cert to connect through front-proxy
_, err := crypto.MakeSelfSignedCA(
filepath.Join(".kcp", "/client-ca.crt"),
filepath.Join(".kcp", "/client-ca.key"),
filepath.Join(".kcp", "/client-ca-serial.txt"),
"kcp-client-ca",
365,
)
if err != nil {
return fmt.Errorf("failed to create client-ca: %w", err)
}

logFilePath := flag.Lookup("log-file-path").Value.String()
shard := shard.NewShard(
"kcp",
".kcp",
logFilePath,
append(shardFlags, "--audit-log-path", filepath.Join(filepath.Dir(logFilePath), "audit.log")),
append(shardFlags,
"--audit-log-path", filepath.Join(filepath.Dir(logFilePath), "audit.log"),
"--client-ca-file", filepath.Join(".kcp", "client-ca.crt"),
),
)
if err := shard.Start(ctx, quiet); err != nil {
return err
Expand Down
10 changes: 5 additions & 5 deletions test/e2e/apibinding/maximalpermissionpolicy_authorizer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ func TestMaximalPermissionPolicyAuthorizerSystemGroupProtection(t *testing.T) {
t.Parallel()

t.Logf("Creating a WorkspaceType as user-1")
userKcpClusterClient, err := kcpclientset.NewForConfig(framework.UserConfig("user-1", server.BaseConfig(t)))
userKcpClusterClient, err := kcpclientset.NewForConfig(framework.StaticTokenUserConfig("user-1", server.BaseConfig(t)))
require.NoError(t, err, "failed to construct kcp cluster client for user-1")
framework.Eventually(t, func() (bool, string) { // authz makes this eventually succeed
_, err = userKcpClusterClient.Cluster(orgPath).TenancyV1alpha1().WorkspaceTypes().Create(ctx, &tenancyv1alpha1.WorkspaceType{
Expand Down Expand Up @@ -114,7 +114,7 @@ func TestMaximalPermissionPolicyAuthorizerSystemGroupProtection(t *testing.T) {
t.Parallel()

t.Logf("Creating a APIExport as user-1")
userKcpClusterClient, err := kcpclientset.NewForConfig(framework.UserConfig("user-1", server.BaseConfig(t)))
userKcpClusterClient, err := kcpclientset.NewForConfig(framework.StaticTokenUserConfig("user-1", server.BaseConfig(t)))
require.NoError(t, err, "failed to construct kcp cluster client for user-1")
framework.Eventually(t, func() (bool, string) { // authz makes this eventually succeed
_, err := userKcpClusterClient.Cluster(orgPath).ApisV1alpha1().APIExports().Create(ctx, &apisv1alpha1.APIExport{
Expand Down Expand Up @@ -169,7 +169,7 @@ func TestMaximalPermissionPolicyAuthorizer(t *testing.T) {
kubeClusterClient, err := kcpkubernetesclientset.NewForConfig(cfg)
require.NoError(t, err, "failed to construct dynamic cluster client for server")

user3KcpClient, err := kcpclientset.NewForConfig(framework.UserConfig("user-3", rest.CopyConfig(cfg)))
user3KcpClient, err := kcpclientset.NewForConfig(framework.StaticTokenUserConfig("user-3", rest.CopyConfig(cfg)))
require.NoError(t, err, "failed to construct dynamic cluster client for server")

serviceProviderClusterNames := []logicalcluster.Path{rbacServiceProviderPath, serviceProvider2Workspace}
Expand Down Expand Up @@ -228,7 +228,7 @@ func TestMaximalPermissionPolicyAuthorizer(t *testing.T) {
t.Logf("Set up user-1 and user-3 as admin for the consumer workspace %q", consumer)
framework.AdmitWorkspaceAccess(ctx, t, kubeClusterClient, consumer, []string{"user-1", "user-3"}, nil, true)
bindConsumerToProvider(consumer, serviceProvider)
wildwestClusterClient, err := wildwestclientset.NewForConfig(framework.UserConfig("user-1", rest.CopyConfig(cfg)))
wildwestClusterClient, err := wildwestclientset.NewForConfig(framework.StaticTokenUserConfig("user-1", rest.CopyConfig(cfg)))
cowboyclient := wildwestClusterClient.WildwestV1alpha1().Cluster(consumer).Cowboys("default")
require.NoError(t, err)
testCRUDOperations(ctx, t, consumer, wildwestClusterClient)
Expand All @@ -255,7 +255,7 @@ func TestMaximalPermissionPolicyAuthorizer(t *testing.T) {
require.NoError(t, err)

framework.AdmitWorkspaceAccess(ctx, t, kubeClusterClient, consumer, []string{"user-2"}, nil, false)
user2Client, err := wildwestclientset.NewForConfig(framework.UserConfig("user-2", rest.CopyConfig(cfg)))
user2Client, err := wildwestclientset.NewForConfig(framework.StaticTokenUserConfig("user-2", rest.CopyConfig(cfg)))
require.NoError(t, err)

t.Logf("Make sure user 2 can list cowboys in consumer workspace %q", consumer)
Expand Down
10 changes: 5 additions & 5 deletions test/e2e/authorizer/authorizer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,13 +91,13 @@ func TestAuthorizer(t *testing.T) {
framework.AdmitWorkspaceAccess(ctx, t, kubeClusterClient, org2.Join("workspace2"), []string{"user-3"}, nil, false)
framework.AdmitWorkspaceAccess(ctx, t, kubeClusterClient, org2.Join("workspace2"), []string{"user-2"}, nil, true)

user1KubeClusterClient, err := kcpkubernetesclientset.NewForConfig(framework.UserConfig("user-1", cfg))
user1KubeClusterClient, err := kcpkubernetesclientset.NewForConfig(framework.StaticTokenUserConfig("user-1", cfg))
require.NoError(t, err)
user1KubeDiscoveryClient, err := kcpdiscovery.NewForConfig(framework.UserConfig("user-1", cfg))
user1KubeDiscoveryClient, err := kcpdiscovery.NewForConfig(framework.StaticTokenUserConfig("user-1", cfg))
require.NoError(t, err)
user2KubeClusterClient, err := kcpkubernetesclientset.NewForConfig(framework.UserConfig("user-2", cfg))
user2KubeClusterClient, err := kcpkubernetesclientset.NewForConfig(framework.StaticTokenUserConfig("user-2", cfg))
require.NoError(t, err)
user3KubeClusterClient, err := kcpkubernetesclientset.NewForConfig(framework.UserConfig("user-3", cfg))
user3KubeClusterClient, err := kcpkubernetesclientset.NewForConfig(framework.StaticTokenUserConfig("user-3", cfg))
require.NoError(t, err)

t.Logf("Priming the authorization cache")
Expand Down Expand Up @@ -169,7 +169,7 @@ func TestAuthorizer(t *testing.T) {
// create client talking directly to root shard to test wildcard requests
rootKubeClusterClient, err := kcpkubernetesclientset.NewForConfig(rootShardCfg)
require.NoError(t, err)
user1RootKubeClusterClient, err := kcpkubernetesclientset.NewForConfig(framework.UserConfig("user-1", rootShardCfg))
user1RootKubeClusterClient, err := kcpkubernetesclientset.NewForConfig(framework.StaticTokenUserConfig("user-1", rootShardCfg))
require.NoError(t, err)

_, err = rootKubeClusterClient.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
Expand Down
54 changes: 45 additions & 9 deletions test/e2e/framework/kcp.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ import (
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"

"github.com/kcp-dev/kcp/cmd/sharded-test-server/third_party/library-go/crypto"
"github.com/kcp-dev/kcp/pkg/embeddedetcd"
"github.com/kcp-dev/kcp/pkg/server"
"github.com/kcp-dev/kcp/pkg/server/options"
Expand All @@ -76,6 +77,12 @@ func TestServerWithAuditPolicyFile(auditPolicyFile string) []string {
}
}

func TestServerWithClientCAFile(clientCAFile string) []string {
return []string{
"--client-ca-file", clientCAFile,
}
}

// TestServerArgsWithTokenAuthFile returns the set of kcp args used to
// start a test server with the given token auth file.
func TestServerArgsWithTokenAuthFile(tokenAuthFile string) []string {
Expand Down Expand Up @@ -140,7 +147,7 @@ func SharedKcpServer(t *testing.T) RunningServer {
// Use a persistent server

t.Logf("shared kcp server will target configuration %q", kubeconfig)
server, err := newPersistentKCPServer(serverName, kubeconfig, TestConfig.RootShardKubeconfig())
server, err := newPersistentKCPServer(serverName, kubeconfig, TestConfig.RootShardKubeconfig(), filepath.Join(RepositoryDir(), ".kcp"))
require.NoError(t, err, "failed to create persistent server fixture")
return server
}
Expand All @@ -155,18 +162,33 @@ func SharedKcpServer(t *testing.T) RunningServer {
artifactDir, dataDir, err := ScratchDirs(t)
require.NoError(t, err, "failed to create scratch dirs: %v", err)

args := TestServerArgsWithTokenAuthFile(WriteTokenAuthFile(t))
args = append(args, TestServerWithAuditPolicyFile(WriteEmbedFile(t, "audit-policy.yaml"))...)
clientCADir, clientCAFile := CreateClientCA(t)
args = append(args, TestServerWithClientCAFile(clientCAFile)...)
f := newKcpFixture(t, kcpConfig{
Name: serverName,
Args: append(
TestServerArgsWithTokenAuthFile(WriteTokenAuthFile(t)),
TestServerWithAuditPolicyFile(WriteEmbedFile(t, "audit-policy.yaml"))...,
),
Name: serverName,
Args: args,
ArtifactDir: artifactDir,
ClientCADir: clientCADir,
DataDir: dataDir,
})
return f.Servers[serverName]
}

func CreateClientCA(t *testing.T) (string, string) {
clientCADir := t.TempDir()
_, err := crypto.MakeSelfSignedCA(
filepath.Join(filepath.Join(clientCADir, "client-ca.crt")),
filepath.Join(filepath.Join(clientCADir, "client-ca.key")),
filepath.Join(filepath.Join(clientCADir, "client-ca-serial.txt")),
"kcp-client-ca",
365,
)
require.NoError(t, err)
return clientCADir, filepath.Join(clientCADir, "client-ca.crt")
}

// Deprecated for use outside this package. Prefer PrivateKcpServer().
func newKcpFixture(t *testing.T, cfgs ...kcpConfig) *kcpFixture {
t.Helper()
Expand All @@ -183,7 +205,7 @@ func newKcpFixture(t *testing.T, cfgs ...kcpConfig) *kcpFixture {
if len(cfg.DataDir) == 0 {
panic(fmt.Sprintf("provided kcpConfig for %s is incorrect, missing DataDir", cfg.Name))
}
server, err := newKcpServer(t, cfg, cfg.ArtifactDir, cfg.DataDir)
server, err := newKcpServer(t, cfg, cfg.ArtifactDir, cfg.DataDir, cfg.ClientCADir)
require.NoError(t, err)

servers = append(servers, server)
Expand Down Expand Up @@ -245,6 +267,7 @@ type RunningServer interface {
BaseConfig(t *testing.T) *rest.Config
RootShardSystemMasterBaseConfig(t *testing.T) *rest.Config
Artifact(t *testing.T, producer func() (runtime.Object, error))
ClientCAUserConfig(t *testing.T, config *rest.Config, name string, groups ...string) *rest.Config
}

// KcpConfigOption a function that wish to modify a given kcp configuration.
Expand Down Expand Up @@ -275,6 +298,7 @@ type kcpConfig struct {
Args []string
ArtifactDir string
DataDir string
ClientCADir string

LogToConsole bool
RunInProcess bool
Expand All @@ -291,6 +315,7 @@ type kcpServer struct {
ctx context.Context //nolint:containedctx
dataDir string
artifactDir string
clientCADir string

lock *sync.Mutex
cfg clientcmd.ClientConfig
Expand All @@ -299,7 +324,7 @@ type kcpServer struct {
t *testing.T
}

func newKcpServer(t *testing.T, cfg kcpConfig, artifactDir, dataDir string) (*kcpServer, error) {
func newKcpServer(t *testing.T, cfg kcpConfig, artifactDir, dataDir, clientCADir string) (*kcpServer, error) {
t.Helper()

kcpListenPort, err := GetFreePort(t)
Expand Down Expand Up @@ -339,6 +364,7 @@ func newKcpServer(t *testing.T, cfg kcpConfig, artifactDir, dataDir string) (*kc
cfg.Args...),
dataDir: dataDir,
artifactDir: artifactDir,
clientCADir: clientCADir,
t: t,
lock: &sync.Mutex{},
}, nil
Expand Down Expand Up @@ -622,6 +648,10 @@ func (c *kcpServer) config(context string) (*rest.Config, error) {
return restConfig, nil
}

func (c *kcpServer) ClientCAUserConfig(t *testing.T, config *rest.Config, name string, groups ...string) *rest.Config {
return ClientCAUserConfig(t, config, c.clientCADir, name, groups...)
}

// BaseConfig returns a rest.Config for the "base" context. Client-side throttling is disabled (QPS=-1).
func (c *kcpServer) BaseConfig(t *testing.T) *rest.Config {
t.Helper()
Expand Down Expand Up @@ -817,14 +847,19 @@ type unmanagedKCPServer struct {
kubeconfigPath string
rootShardKubeconfigPath string
cfg, rootShardCfg clientcmd.ClientConfig
clientCADir string
}

func (s *unmanagedKCPServer) ClientCAUserConfig(t *testing.T, config *rest.Config, name string, groups ...string) *rest.Config {
return ClientCAUserConfig(t, config, s.clientCADir, name, groups...)
}

// newPersistentKCPServer returns a RunningServer for a kubeconfig
// pointing to a kcp instance not managed by the test run. Since the
// kubeconfig is expected to exist prior to running tests against it,
// the configuration can be loaded synchronously and no locking is
// required to subsequently access it.
func newPersistentKCPServer(name, kubeconfigPath, rootShardKubeconfigPath string) (RunningServer, error) {
func newPersistentKCPServer(name, kubeconfigPath, rootShardKubeconfigPath, clientCADir string) (RunningServer, error) {
cfg, err := LoadKubeConfig(kubeconfigPath, "base")
if err != nil {
return nil, err
Expand All @@ -841,6 +876,7 @@ func newPersistentKCPServer(name, kubeconfigPath, rootShardKubeconfigPath string
rootShardKubeconfigPath: rootShardKubeconfigPath,
cfg: cfg,
rootShardCfg: rootShardCfg,
clientCADir: clientCADir,
}, nil
}

Expand Down
66 changes: 65 additions & 1 deletion test/e2e/framework/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,18 @@ limitations under the License.
package framework

import (
"bytes"
"context"
cryptorand "crypto/rand"
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"embed"
"encoding/pem"
"fmt"
"io"
"math/big"
"math/rand"
"net"
"os"
Expand All @@ -47,6 +55,7 @@ import (
"k8s.io/client-go/rest"
"k8s.io/client-go/restmapper"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
"k8s.io/client-go/util/cert"
"sigs.k8s.io/yaml"

"github.com/kcp-dev/kcp/config/helpers"
Expand Down Expand Up @@ -351,7 +360,62 @@ func EventuallyReady(t *testing.T, getter func() (conditions.Getter, error), msg
}, wait.ForeverTestTimeout, 100*time.Millisecond, msgAndArgs...)
}

func UserConfig(username string, cfg *rest.Config) *rest.Config {
// ClientCAUserConfig returns a config based on a dynamically created client certificate.
// The returned client CA is signed by "test/e2e/framework/client-ca.crt".
func ClientCAUserConfig(t *testing.T, cfg *rest.Config, clientCAConfigDirectory, username string, groups ...string) *rest.Config {
t.Helper()
caBytes, err := os.ReadFile(filepath.Join(clientCAConfigDirectory, "client-ca.crt"))
require.NoError(t, err, "error reading CA file")
caKeyBytes, err := os.ReadFile(filepath.Join(clientCAConfigDirectory, "client-ca.key"))
require.NoError(t, err, "error reading CA key")
caCerts, err := cert.ParseCertsPEM(caBytes)
require.NoError(t, err, "error parsing CA certs")
caKeys, err := tls.X509KeyPair(caBytes, caKeyBytes)
require.NoError(t, err, "error parsing CA keys")
clientPublicKey, clientPrivateKey, err := newRSAKeyPair()
require.NoError(t, err, "error creating client keys")
currentTime := time.Now()
clientCert := &x509.Certificate{
Subject: pkix.Name{
CommonName: username,
Organization: groups,
},

SignatureAlgorithm: x509.SHA256WithRSA,

NotBefore: currentTime.Add(-1 * time.Second),
NotAfter: currentTime.Add(time.Hour * 2),
SerialNumber: big.NewInt(1),

KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
BasicConstraintsValid: true,
}
signedClientCertBytes, err := x509.CreateCertificate(cryptorand.Reader, clientCert, caCerts[0], clientPublicKey, caKeys.PrivateKey)
require.NoError(t, err, "error creating client certificate")
clientCertPEM := new(bytes.Buffer)
require.NoError(t, pem.Encode(clientCertPEM, &pem.Block{Type: "CERTIFICATE", Bytes: signedClientCertBytes}), "error encoding client cert")
clientKeyPEM := new(bytes.Buffer)
require.NoError(t, pem.Encode(clientKeyPEM, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(clientPrivateKey)}), "error encoding client private key")

cfgCopy := rest.CopyConfig(cfg)
cfgCopy.CertData = clientCertPEM.Bytes()
cfgCopy.KeyData = clientKeyPEM.Bytes()
cfgCopy.BearerToken = ""
return cfgCopy
}

func newRSAKeyPair() (*rsa.PublicKey, *rsa.PrivateKey, error) {
privateKey, err := rsa.GenerateKey(cryptorand.Reader, 2048)
if err != nil {
return nil, nil, err
}
return &privateKey.PublicKey, privateKey, nil
}

// StaticTokenUserConfig returns a user config based on static user tokens defined in "test/e2e/framework/auth-tokens.csv".
// The token being used is "[username]-token".
func StaticTokenUserConfig(username string, cfg *rest.Config) *rest.Config {
return ConfigWithToken(username+"-token", cfg)
}

Expand Down
Loading

0 comments on commit 57f8858

Please sign in to comment.