Skip to content

Commit

Permalink
Merge 4837e90 into 4f44b42
Browse files Browse the repository at this point in the history
  • Loading branch information
@cameronmeissner
cameronmeissner committed Apr 16, 2024
2 parents 4f44b42 + 4837e90 commit 2a94525
Show file tree
Hide file tree
Showing 10 changed files with 242 additions and 105 deletions.
19 changes: 11 additions & 8 deletions client/cmd/client/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ const (
flagAADResource = "aad-resource"
flagVerbose = "verbose"
flagKubeconfigPath = "kubeconfig"
flagCertFilePath = "cert-file"
flagKeyFilePath = "key-file"
flagInsecureSkipTLSVerify = "insecure-skip-tls-verify"
flagEnsureClientAuthentication = "ensure-client-authentication"
)
Expand All @@ -46,19 +48,18 @@ func main() {

func createBootstrapCommand() *cobra.Command {
var (
opts = &client.GetKubeletClientCredentialOpts{}
azureConfigPath string
clusterCAFilePath string
logFile string
format string
verbose bool
opts = &client.GetKubeletClientCredentialOpts{}
azureConfigPath string
logFile string
format string
verbose bool
)

cmd := &cobra.Command{
Use: "bootstrap",
Short: "generate a secure TLS bootstrap token to securely join an AKS cluster",
RunE: func(cmd *cobra.Command, args []string) error {
if err := opts.ValidateAndSet(azureConfigPath, clusterCAFilePath); err != nil {
if err := opts.ValidateAndSet(azureConfigPath); err != nil {
return fmt.Errorf("validating and setting opts for kubelet client credential generation: %w", err)
}

Expand Down Expand Up @@ -95,7 +96,6 @@ func createBootstrapCommand() *cobra.Command {

cmd.Flags().BoolVar(&verbose, flagVerbose, false, "Enable verbose logging.")
cmd.Flags().StringVar(&azureConfigPath, flagAzureConfigPath, "", "Path to the azure config file.")
cmd.Flags().StringVar(&clusterCAFilePath, flagClusterCAFilePath, "", "Path to the cluster CA file.")
cmd.Flags().StringVar(&logFile, flagLogFile, "", "Path to the file where logs will be written.")
cmd.Flags().StringVar(&format, flagLogFormat, "json", "Log format: json or console.")
cmd.Flags().BoolVar(&opts.InsecureSkipTLSVerify, flagInsecureSkipTLSVerify, false, "Skip TLS verification when connecting to the API server FQDN.")
Expand All @@ -105,6 +105,9 @@ func createBootstrapCommand() *cobra.Command {
cmd.Flags().StringVar(&opts.AADResource, flagAADResource, "", "Resource (audience) used to request JWT tokens from AAD for authentication.")
cmd.Flags().StringVar(&opts.NextProto, flagNextProto, "", "ALPN Next Protocol value to send within requests to the bootstrap server.")
cmd.Flags().StringVar(&opts.KubeconfigPath, flagKubeconfigPath, "", "Path to the kubeconfig file containing the generated kubelet client certificate.")
cmd.Flags().StringVar(&opts.ClusterCAFilePath, flagClusterCAFilePath, "", "Path to the cluster CA file.")
cmd.Flags().StringVar(&opts.CertFilePath, flagCertFilePath, "", "Path to the file which will contain the PEM-encoded client certificate, referenced by the generated kubeconfig.")
cmd.Flags().StringVar(&opts.KeyFilePath, flagKeyFilePath, "", "Path to the file which will contain the PEM-encoded client key, referenced by the generated kubeconfig.")
return cmd
}

Expand Down
44 changes: 22 additions & 22 deletions client/pkg/client/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,22 +20,24 @@ import (
)

type GetKubeletClientCredentialOpts struct {
ClusterCAData []byte
APIServerFQDN string
CustomClientID string
NextProto string
AADResource string
ClusterCAFilePath string
KubeconfigPath string
CertFilePath string
KeyFilePath string
InsecureSkipTLSVerify bool
EnsureClientAuthentication bool
AzureConfig *datamodel.AzureConfig
}

func (o *GetKubeletClientCredentialOpts) ValidateAndSet(azureConfigPath, clusterCAFilePath string) error {
func (o *GetKubeletClientCredentialOpts) ValidateAndSet(azureConfigPath string) error {
if azureConfigPath == "" {
return fmt.Errorf("azure config path must be specified")
}
if clusterCAFilePath == "" {
if o.ClusterCAFilePath == "" {
return fmt.Errorf("cluster CA file path must be specified")
}
if o.APIServerFQDN == "" {
Expand All @@ -48,8 +50,15 @@ func (o *GetKubeletClientCredentialOpts) ValidateAndSet(azureConfigPath, cluster
return fmt.Errorf("AAD resource must be specified")
}
if o.KubeconfigPath == "" {
return fmt.Errorf("kubeconfig must be specified")
return fmt.Errorf("kubeconfig path must be specified")
}
if o.CertFilePath == "" {
return fmt.Errorf("cert file path must be specified")
}
if o.KeyFilePath == "" {
return fmt.Errorf("key file path must be specified")
}

azureConfig := &datamodel.AzureConfig{}
azureConfigData, err := os.ReadFile(azureConfigPath)
if err != nil {
Expand All @@ -58,12 +67,8 @@ func (o *GetKubeletClientCredentialOpts) ValidateAndSet(azureConfigPath, cluster
if err = json.Unmarshal(azureConfigData, azureConfig); err != nil {
return fmt.Errorf("unmarshaling azure config data: %w", err)
}
clusterCAData, err := os.ReadFile(clusterCAFilePath)
if err != nil {
return fmt.Errorf("reading cluster CA data from %s: %w", clusterCAFilePath, err)
}
o.AzureConfig = azureConfig
o.ClusterCAData = clusterCAData

return nil
}

Expand Down Expand Up @@ -101,9 +106,9 @@ func (c *SecureTLSBootstrapClient) GetKubeletClientCredential(ctx context.Contex
c.logger.Info("generated JWT token for auth")

c.logger.Debug("creating GRPC connection and bootstrap service client...")
serviceClient, conn, err := c.serviceClientFactory(ctx, c.logger, serviceClientFactoryOpts{
serviceClient, conn, err := c.serviceClientFactory(ctx, c.logger, &serviceClientFactoryConfig{
fqdn: opts.APIServerFQDN,
clusterCAData: opts.ClusterCAData,
clusterCAFilePath: opts.ClusterCAFilePath,
insecureSkipTLSVerify: opts.InsecureSkipTLSVerify,
nextProto: opts.NextProto,
authToken: authToken,
Expand Down Expand Up @@ -142,15 +147,8 @@ func (c *SecureTLSBootstrapClient) GetKubeletClientCredential(ctx context.Contex
}
c.logger.Info("retrieved IMDS attested data")

c.logger.Debug("resolving hostname...")
hostname, err := os.Hostname()
if err != nil {
return nil, fmt.Errorf("failed to resolve own hostname for kubelet CSR creation: %w", err)
}
c.logger.Info("resolved hostname", zap.String("hostname", hostname))

c.logger.Debug("generating kubelet client CSR and associated private key...")
csrPEM, privateKey, err := makeKubeletClientCSR(hostname)
csrPEM, privateKey, err := makeKubeletClientCSR()
if err != nil {
return nil, fmt.Errorf("failed to create kubelet client CSR: %w", err)
}
Expand All @@ -177,9 +175,11 @@ func (c *SecureTLSBootstrapClient) GetKubeletClientCredential(ctx context.Contex
if err != nil {
return nil, fmt.Errorf("failed to decode cert data from bootstrap server")
}
kubeconfigData, err := kubeconfig.GenerateForCertAndKey(certPEM, privateKey, &kubeconfig.GenerateOpts{
APIServerFQDN: opts.APIServerFQDN,
ClusterCAData: opts.ClusterCAData,
kubeconfigData, err := kubeconfig.GenerateForCertAndKey(certPEM, privateKey, &kubeconfig.GenerationConfig{
APIServerFQDN: opts.APIServerFQDN,
ClusterCAFilePath: opts.ClusterCAFilePath,
CertFilePath: opts.CertFilePath,
KeyFilePath: opts.KeyFilePath,
})
if err != nil {
return nil, fmt.Errorf("failed to generate kubeconfig for new client cert and key: %w", err)
Expand Down
14 changes: 10 additions & 4 deletions client/pkg/client/client_suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
package client

import (
"os"
"path/filepath"
"testing"
"time"

Expand All @@ -14,8 +16,8 @@ import (
)

var (
logger *zap.Logger
clusterCACertPEM []byte
logger *zap.Logger
mockClusterCAFilePath string
)

func TestTLSBootstrapClient(t *testing.T) {
Expand All @@ -25,7 +27,11 @@ func TestTLSBootstrapClient(t *testing.T) {
}

var _ = BeforeSuite(func() {
var err error
clusterCACertPEM, _, err = testutil.GenerateCertPEMWithExpiration("hcp", "aks", time.Now().Add(time.Hour))
clusterCACertPEM, _, err := testutil.GenerateCertPEMWithExpiration("hcp", "aks", time.Now().Add(time.Hour))
Expect(err).To(BeNil())

tempDir := GinkgoT().TempDir()
mockClusterCAFilePath = filepath.Join(tempDir, "ca.crt")
err = os.WriteFile(mockClusterCAFilePath, clusterCACertPEM, os.ModePerm)
Expect(err).To(BeNil())
})
95 changes: 63 additions & 32 deletions client/pkg/client/client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,10 @@ var _ = Describe("SecureTLSBootstrapClient tests", func() {
bootstrapClient *SecureTLSBootstrapClient
)
defaultOpts := &GetKubeletClientCredentialOpts{
NextProto: "bootstrap",
ClusterCAData: clusterCACertPEM,
APIServerFQDN: defaultAPIServerFQDN,
KubeconfigPath: defaultKubeconfigPath,
NextProto: "bootstrap",
ClusterCAFilePath: mockClusterCAFilePath,
APIServerFQDN: defaultAPIServerFQDN,
KubeconfigPath: defaultKubeconfigPath,
AzureConfig: &datamodel.AzureConfig{
ClientID: "clientId",
ClientSecret: "clientSecret",
Expand Down Expand Up @@ -85,7 +85,7 @@ var _ = Describe("SecureTLSBootstrapClient tests", func() {
bootstrapClient.serviceClientFactory = func(
ctx context.Context,
logger *zap.Logger,
opts serviceClientFactoryOpts) (secureTLSBootstrapService.SecureTLSBootstrapServiceClient, *grpc.ClientConn, error) {
cfg *serviceClientFactoryConfig) (secureTLSBootstrapService.SecureTLSBootstrapServiceClient, *grpc.ClientConn, error) {
return serviceClient, nil, nil
}
})
Expand Down Expand Up @@ -318,7 +318,13 @@ var _ = Describe("SecureTLSBootstrapClient tests", func() {
})

When("bootstrap server can generate a credential", func() {
It("should return a new kubeconfig object with the credential embedded", func() {
It("should return a new kubeconfig object referencing the new credential", func() {
tempDir := GinkgoT().TempDir()
certPath := filepath.Join(tempDir, "client.crt")
keyPath := filepath.Join(tempDir, "client.key")
defaultOpts.CertFilePath = certPath
defaultOpts.KeyFilePath = keyPath

ctx, cancel := context.WithCancel(context.Background())
defer cancel()
clientCertPEM, _, err := testutil.GenerateCertPEMWithExpiration("system:node:node", "system:nodes", time.Now().Add(time.Hour))
Expand Down Expand Up @@ -359,19 +365,27 @@ var _ = Describe("SecureTLSBootstrapClient tests", func() {
Expect(kubeconfigData.Clusters).To(HaveKey("default-cluster"))
defaultCluster := kubeconfigData.Clusters["default-cluster"]
Expect(defaultCluster.Server).To(Equal("https://controlplane.azmk8s.io:443"))
Expect(defaultCluster.CertificateAuthorityData).To(Equal(defaultOpts.ClusterCAData))
Expect(defaultCluster.CertificateAuthority).To(Equal(defaultOpts.ClusterCAFilePath))

Expect(kubeconfigData.AuthInfos).To(HaveKey("default-auth"))
defaultAuth := kubeconfigData.AuthInfos["default-auth"]
Expect(defaultAuth.ClientCertificateData).To(Equal(clientCertPEM))
Expect(defaultAuth.ClientKeyData).ToNot(BeEmpty())
Expect(defaultAuth.ClientCertificate).To(Equal(certPath))
Expect(defaultAuth.ClientKey).To(Equal(keyPath))

Expect(kubeconfigData.Contexts).To(HaveKey("default-context"))
defaultContext := kubeconfigData.Contexts["default-context"]
Expect(defaultContext.Cluster).To(Equal("default-cluster"))
Expect(defaultContext.AuthInfo).To(Equal("default-auth"))

Expect(kubeconfigData.CurrentContext).To(Equal("default-context"))

certData, err := os.ReadFile(certPath)
Expect(err).To(BeNil())
Expect(certData).ToNot(BeEmpty())

keyData, err := os.ReadFile(keyPath)
Expect(err).To(BeNil())
Expect(keyData).ToNot(BeEmpty())
})
})
})
Expand All @@ -380,40 +394,42 @@ var _ = Describe("SecureTLSBootstrapClient tests", func() {
Context("ValidateAndSet", func() {
var opts *GetKubeletClientCredentialOpts
defaultAzureConfigPath := "path/to/azure.json"
defaultClusterCAFilePath := "path/to/ca.crt"

BeforeEach(func() {
opts = &GetKubeletClientCredentialOpts{
APIServerFQDN: "fqdn",
CustomClientID: "clientId",
NextProto: "alpn",
AADResource: "appID",
KubeconfigPath: "path",
APIServerFQDN: "fqdn",
CustomClientID: "clientId",
NextProto: "alpn",
AADResource: "appID",
ClusterCAFilePath: "path",
KubeconfigPath: "path",
CertFilePath: "path",
KeyFilePath: "path",
}
})

When("clusterCAFilePath is empty", func() {
When("azureConfigPath is empty", func() {
It("should return an error", func() {
emptyClusterCAFilePath := ""
err := opts.ValidateAndSet(defaultAzureConfigPath, emptyClusterCAFilePath)
emptyAzureConfigPath := ""
err := opts.ValidateAndSet(emptyAzureConfigPath)
Expect(err).ToNot(BeNil())
Expect(err.Error()).To(ContainSubstring("cluster CA file path must be specified"))
Expect(err.Error()).To(ContainSubstring("azure config path must be specified"))
})
})

When("azureConfigPath is empty", func() {
When("ClusterCAFilePath is empty", func() {
It("should return an error", func() {
emptyAzureConfigPath := ""
err := opts.ValidateAndSet(emptyAzureConfigPath, defaultClusterCAFilePath)
opts.ClusterCAFilePath = ""
err := opts.ValidateAndSet(defaultAzureConfigPath)
Expect(err).ToNot(BeNil())
Expect(err.Error()).To(ContainSubstring("azure config path must be specified"))
Expect(err.Error()).To(ContainSubstring("cluster CA file path must be specified"))
})
})

When("APIServerFQDN is empty", func() {
It("should return an error", func() {
opts.APIServerFQDN = ""
err := opts.ValidateAndSet(defaultAzureConfigPath, defaultClusterCAFilePath)
err := opts.ValidateAndSet(defaultAzureConfigPath)
Expect(err).ToNot(BeNil())
Expect(err.Error()).To(ContainSubstring("apiserver FQDN must be specified"))
})
Expand All @@ -422,7 +438,7 @@ var _ = Describe("SecureTLSBootstrapClient tests", func() {
When("NextProto is empty", func() {
It("should return an error", func() {
opts.NextProto = ""
err := opts.ValidateAndSet(defaultAzureConfigPath, defaultClusterCAFilePath)
err := opts.ValidateAndSet(defaultAzureConfigPath)
Expect(err).ToNot(BeNil())
Expect(err.Error()).To(ContainSubstring("next proto header value must be specified"))
})
Expand All @@ -431,7 +447,7 @@ var _ = Describe("SecureTLSBootstrapClient tests", func() {
When("AADResource is empty", func() {
It("should return an error", func() {
opts.AADResource = ""
err := opts.ValidateAndSet(defaultAzureConfigPath, defaultClusterCAFilePath)
err := opts.ValidateAndSet(defaultAzureConfigPath)
Expect(err).ToNot(BeNil())
Expect(err.Error()).To(ContainSubstring("AAD resource must be specified"))
})
Expand All @@ -440,25 +456,40 @@ var _ = Describe("SecureTLSBootstrapClient tests", func() {
When("KubeconfigPath is empty", func() {
It("should return an error", func() {
opts.KubeconfigPath = ""
err := opts.ValidateAndSet(defaultAzureConfigPath, defaultClusterCAFilePath)
err := opts.ValidateAndSet(defaultAzureConfigPath)
Expect(err).ToNot(BeNil())
Expect(err.Error()).To(ContainSubstring("kubeconfig path must be specified"))
})
})

When("CertFilePath is empty", func() {
It("should return an error", func() {
opts.CertFilePath = ""
err := opts.ValidateAndSet(defaultAzureConfigPath)
Expect(err).ToNot(BeNil())
Expect(err.Error()).To(ContainSubstring("kubeconfig must be specified"))
Expect(err.Error()).To(ContainSubstring("cert file path must be specified"))
})
})

When("KeyFilePath is empty", func() {
It("should return an error", func() {
opts.KeyFilePath = ""
err := opts.ValidateAndSet(defaultAzureConfigPath)
Expect(err).ToNot(BeNil())
Expect(err.Error()).To(ContainSubstring("key file path must be specified"))
})
})

When("client opts are valid", func() {
It("should validate without error", func() {
tempDir := GinkgoT().TempDir()
clusterCAFilePath := filepath.Join(tempDir, "ca.crt")
err := os.WriteFile(clusterCAFilePath, clusterCACertPEM, os.ModePerm)
Expect(err).To(BeNil())
azureConfigPath := filepath.Join(tempDir, "azure.json")
azureConfigBytes, err := json.Marshal(defaultOpts.AzureConfig)
Expect(err).To(BeNil())
err = os.WriteFile(azureConfigPath, azureConfigBytes, os.ModePerm)
Expect(err).To(BeNil())

err = opts.ValidateAndSet(azureConfigPath, clusterCAFilePath)
err = opts.ValidateAndSet(azureConfigPath)
Expect(err).To(BeNil())
})
})
Expand Down

0 comments on commit 2a94525

Please sign in to comment.