Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ require (
open-cluster-management.io/api v1.1.0
open-cluster-management.io/cluster-proxy v0.7.0
open-cluster-management.io/managed-serviceaccount v0.8.0
open-cluster-management.io/ocm v1.1.0
open-cluster-management.io/ocm v1.1.1-0.20251105064423-d80ec55608e7
open-cluster-management.io/sdk-go v1.1.0
sigs.k8s.io/apiserver-network-proxy v0.29.0
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -484,8 +484,8 @@ open-cluster-management.io/cluster-proxy v0.7.0 h1:qOok0BIBL6j4mLRArzJdz0gK5nyyn
open-cluster-management.io/cluster-proxy v0.7.0/go.mod h1:6cgnExpuprO7Le7aqf7bI3H7Nvu3YnXBJCIbJ7wsC0s=
open-cluster-management.io/managed-serviceaccount v0.8.0 h1:8+Z142IUqVT/enxXkyb0nzLUL7JaR7dBM2fDtlCA4pM=
open-cluster-management.io/managed-serviceaccount v0.8.0/go.mod h1:eTixwpLA6XkPQARDjze3k0KRjwn6N22eFOEFx8CpB0I=
open-cluster-management.io/ocm v1.1.0 h1:Gu5+LYMHMCNxE1StB2x9SyH1E1K1cNOTdUrp1Id++T8=
open-cluster-management.io/ocm v1.1.0/go.mod h1:vuIzDonz/ypkaLNqXvjOHSwMA29x+e25s01S8Z0j3gc=
open-cluster-management.io/ocm v1.1.1-0.20251105064423-d80ec55608e7 h1:wRA9v3BH1mfxZiMZVgc0Ert2JziNbekUcg4PpPQUrLk=
open-cluster-management.io/ocm v1.1.1-0.20251105064423-d80ec55608e7/go.mod h1:LlEIZdZrQQduPS6HqFKvdadtFjOfVJzBj80/Ur+exP8=
open-cluster-management.io/sdk-go v1.1.0 h1:vYGkoihIVetyVT4ICO7HjoUHsnh6Gf+Da4ZSmWCamhc=
open-cluster-management.io/sdk-go v1.1.0/go.mod h1:DH4EMNDMiousmaj+noHYQxm48T+dbogiAfALhDnrjMg=
oras.land/oras-go/v2 v2.6.0 h1:X4ELRsiGkrbeox69+9tzTu492FMUu7zJQW6eJU+I2oc=
Expand Down
8 changes: 5 additions & 3 deletions pkg/cmd/init/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ var example = `
%[1]s init

# Initialize the hub cluster with the type of authentication. Either or both of csr,awsirsa
%[1]s init --registration-drivers "awsirsa,csr"
%[1]s init --registration-drivers "awsirsa,csr,grpc"
--hubClusterArn arn:aws:eks:us-west-2:123456789012:cluster/hub-cluster1
--aws-resource-tags product:v1:tenant:app-name=My-App,product:v1:tenant:created-by=Team-1
--auto-approved-csr-identities="user1,user2"
Expand Down Expand Up @@ -89,7 +89,7 @@ func NewCmd(clusteradmFlags *genericclioptionsclusteradm.ClusteradmFlags, stream
o.Helm.AddFlags(singletonSet)
cmd.Flags().AddFlagSet(singletonSet)
cmd.Flags().StringSliceVar(&o.registrationDrivers, "registration-drivers", []string{},
"The type of authentication to use for registering and authenticating with hub. Only csr and awsirsa are accepted as valid inputs. This flag can be repeated to specify multiple authentication types.")
"The type of authentication to use for registering and authenticating with hub. Only csr, awsirsa and grpc are accepted as valid inputs. This flag can be repeated to specify multiple authentication types.")
cmd.Flags().StringVar(&o.hubClusterArn, "hub-cluster-arn", "",
"The hubCluster ARN to be passed if awsirsa is one of the registrationAuths and the cluster name in EKS kubeconfig doesn't contain hubClusterArn")
cmd.Flags().StringSliceVar(&o.awsResourceTags, "aws-resource-tags", []string{},
Expand All @@ -100,6 +100,8 @@ func NewCmd(clusteradmFlags *genericclioptionsclusteradm.ClusteradmFlags, stream
cmd.Flags().StringSliceVar(&o.autoApprovedARNPatterns, "auto-approved-arn-patterns", []string{},
"List of AWS EKS ARN patterns so any EKS clusters with these patterns will be auto accepted to join with hub cluster")
cmd.Flags().BoolVar(&o.enableSyncLabels, "enable-sync-labels", false, "If true, sync the labels from clustermanager to all hub resources.")

cmd.Flags().StringVar(&o.grpcServer, "grpc-server", "", "The gRPC server address of the hub")
cmd.Flags().StringSliceVar(&o.autoApprovedGRPCIdentities, "auto-approved-grpc-identities", []string{},
"List of users or identities that are accepted and whatever matches can be auto accepted to join hub for grpc clusters")
return cmd
}
63 changes: 52 additions & 11 deletions pkg/cmd/init/exec.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ var (
releaseName = "multicluster-controlplane"
)

var validRegistrationDriver = sets.New[string](operatorv1.CSRAuthType, operatorv1.AwsIrsaAuthType, operatorv1.GRPCAuthType)

func (o *Options) complete(cmd *cobra.Command, args []string) (err error) {
klog.V(1).InfoS("init options:", "dry-run", o.ClusteradmFlags.DryRun, "force", o.force, "output-file", o.outputFile)

Expand Down Expand Up @@ -95,6 +97,25 @@ func (o *Options) complete(cmd *cobra.Command, args []string) (err error) {
genericclioptionsclusteradm.HubMutableFeatureGate, ocmfeature.DefaultHubAddonManagerFeatureGates),
},
}
if sets.New[string](o.registrationDrivers...).Has(operatorv1.GRPCAuthType) {
if o.grpcServer == "" {
return fmt.Errorf("grpc server should not be empty if registration driver has grpc type")
}

o.clusterManagerChartConfig.ClusterManager.ServerConfiguration = operatorv1.ServerConfiguration{
EndpointsExposure: []operatorv1.EndpointExposure{
{
Protocol: operatorv1.GRPCAuthType,
GRPC: &operatorv1.Endpoint{
Type: operatorv1.EndpointTypeHostname,
Hostname: &operatorv1.HostnameConfig{
Host: o.grpcServer,
},
},
},
},
}
}
o.clusterManagerChartConfig.CreateBootstrapToken = o.useBootstrapToken

if o.imagePullCredFile != "" {
Expand Down Expand Up @@ -155,25 +176,29 @@ func (o *Options) validate() error {
return fmt.Errorf("registry should not be empty")
}

validRegistrationDriver := sets.New[string]("csr", "awsirsa")
for _, driver := range o.registrationDrivers {
if !validRegistrationDriver.Has(driver) {
return fmt.Errorf("only csr and awsirsa are valid drivers")
return fmt.Errorf("only csr,awsirsa and grpc are valid drivers")
}
}

if genericclioptionsclusteradm.HubMutableFeatureGate.Enabled("ManagedClusterAutoApproval") {
// If hub registration does not accept awsirsa, we stop user if they also pass in a list of patterns for AWS EKS ARN.

if len(o.autoApprovedARNPatterns) > 0 && !sets.New[string](o.registrationDrivers...).Has("awsirsa") {
if len(o.autoApprovedARNPatterns) > 0 && !sets.New[string](o.registrationDrivers...).Has(operatorv1.AwsIrsaAuthType) {
return fmt.Errorf("should not provide list of patterns for aws eks arn if not initializing hub with awsirsa registration")
}

// If hub registration does not accept csr, we stop user if they also pass in a list of users for CSR auto approval.
if len(o.autoApprovedCSRIdentities) > 0 && !sets.New[string](o.registrationDrivers...).Has("csr") {
if len(o.autoApprovedCSRIdentities) > 0 && !sets.New[string](o.registrationDrivers...).Has(operatorv1.CSRAuthType) {
return fmt.Errorf("should not provide list of users for csr to auto approve if not initializing hub with csr registration")
}
} else if len(o.autoApprovedARNPatterns) > 0 || len(o.autoApprovedCSRIdentities) > 0 {

if len(o.autoApprovedGRPCIdentities) > 0 && !sets.New[string](o.registrationDrivers...).Has(operatorv1.GRPCAuthType) {
return fmt.Errorf("should not provide list of users or identities for grpc cluster to auto approve if not initializing hub with grpc registration")
}

} else if len(o.autoApprovedARNPatterns) > 0 || len(o.autoApprovedCSRIdentities) > 0 || len(o.autoApprovedGRPCIdentities) > 0 {
return fmt.Errorf("should enable feature gate ManagedClusterAutoApproval before passing list of identities")
}

Expand Down Expand Up @@ -394,22 +419,38 @@ func (o *Options) deploySingletonControlplane(kubeClient kubernetes.Interface) e

func getRegistrationDrivers(o *Options) ([]operatorv1.RegistrationDriverHub, error) {
registrationDrivers := []operatorv1.RegistrationDriverHub{}
var registrationDriver operatorv1.RegistrationDriverHub

for _, driver := range o.registrationDrivers {
if driver == "csr" {
csr := &operatorv1.CSRConfig{AutoApprovedIdentities: o.autoApprovedCSRIdentities}
registrationDriver = operatorv1.RegistrationDriverHub{AuthType: driver, CSR: csr}
} else if driver == "awsirsa" {
var registrationDriver operatorv1.RegistrationDriverHub
switch driver {
case operatorv1.CSRAuthType:
registrationDriver = operatorv1.RegistrationDriverHub{AuthType: operatorv1.CSRAuthType}
if len(o.autoApprovedCSRIdentities) != 0 {
registrationDriver.CSR = &operatorv1.CSRConfig{
AutoApprovedIdentities: o.autoApprovedCSRIdentities,
}
}
case operatorv1.AwsIrsaAuthType:
hubClusterArn, err := getHubClusterArn(o)
if err != nil {
return registrationDrivers, err
}
awsirsa := &operatorv1.AwsIrsaConfig{HubClusterArn: hubClusterArn, Tags: o.awsResourceTags, AutoApprovedIdentities: o.autoApprovedARNPatterns}
registrationDriver = operatorv1.RegistrationDriverHub{AuthType: driver, AwsIrsa: awsirsa}
registrationDriver = operatorv1.RegistrationDriverHub{AuthType: operatorv1.AwsIrsaAuthType, AwsIrsa: awsirsa}
case operatorv1.GRPCAuthType:
registrationDriver = operatorv1.RegistrationDriverHub{AuthType: operatorv1.GRPCAuthType}
if len(o.autoApprovedGRPCIdentities) != 0 {
registrationDriver.GRPC = &operatorv1.GRPCRegistrationConfig{
AutoApprovedIdentities: o.autoApprovedGRPCIdentities,
}
}
default:
return registrationDrivers, fmt.Errorf("unknown registration-drivers type: %s", driver)
}

registrationDrivers = append(registrationDrivers, registrationDriver)
}

return registrationDrivers, nil
}

Expand Down
6 changes: 6 additions & 0 deletions pkg/cmd/init/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,12 @@ type Options struct {
awsResourceTags []string
// enableSyncLabels is to enable the feature which can sync the labels from clustermanager to all hub resources.
enableSyncLabels bool

// grpcServer is the gRPC server of the hub.
grpcServer string
// autoApprovedGRPCIdentities are a list of users or identities that are accepted and whatever matches can
// be auto accepted to join hub for grpc clusters.
autoApprovedGRPCIdentities []string
}

func newOptions(clusteradmFlags *genericclioptionsclusteradm.ClusteradmFlags, streams genericiooptions.IOStreams) *Options {
Expand Down
57 changes: 13 additions & 44 deletions test/e2e/clusteradm/joinhubscenario_grpc_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,58 +38,27 @@ var _ = ginkgo.Describe("test clusteradm join with grpc", ginkgo.Label("join-hub
ginkgo.By("init hub")
err = e2e.Clusteradm().Init(
"--context", e2e.Cluster().Hub().Context(),
"--registration-drivers", "csr,grpc",
"--grpc-server", "cluster-manager-grpc-server.open-cluster-management-hub.svc:8090",
"--feature-gates=ManagedClusterAutoApproval=true",
"--auto-approved-grpc-identities", "system:serviceaccount:open-cluster-management:agent-registration-bootstrap",
"--bundle-version=latest",
)
gomega.Expect(err).NotTo(gomega.HaveOccurred(), "clusteradm init error")
util.WaitClusterManagerApplied(operatorClient)

ginkgo.By("wait for cluster-manager CR to be created and update cluster-manager to enable grpc")
var clusterManager *operatorv1.ClusterManager
gomega.Eventually(func() error {
clusterManager, err = operatorClient.OperatorV1().ClusterManagers().Get(
context.TODO(), "cluster-manager", metav1.GetOptions{})
if err != nil {
return err
}

// Enable ManagedClusterAutoApproval feature gate
if clusterManager.Spec.RegistrationConfiguration == nil {
clusterManager.Spec.RegistrationConfiguration = &operatorv1.RegistrationHubConfiguration{}
}
clusterManager.Spec.RegistrationConfiguration.FeatureGates = append(
clusterManager.Spec.RegistrationConfiguration.FeatureGates,
operatorv1.FeatureGate{
Feature: "ManagedClusterAutoApproval",
Mode: operatorv1.FeatureGateModeTypeEnable,
},
)

// Add grpc authType in registrationDrivers
clusterManager.Spec.RegistrationConfiguration.RegistrationDrivers = []operatorv1.RegistrationDriverHub{
{
AuthType: operatorv1.GRPCAuthType,
GRPC: &operatorv1.GRPCRegistrationConfig{
AutoApprovedIdentities: []string{
"system:serviceaccount:open-cluster-management:agent-registration-bootstrap",
},
},
},
}

// Add serverConfiguration with grpc protocol
clusterManager.Spec.ServerConfiguration = &operatorv1.ServerConfiguration{
EndpointsExposure: []operatorv1.EndpointExposure{
{
Protocol: "grpc",
},
},
}

_, err = operatorClient.OperatorV1().ClusterManagers().Update(
context.TODO(), clusterManager, metav1.UpdateOptions{})
clusterManager, err = operatorClient.OperatorV1().ClusterManagers().Get(context.TODO(),
"cluster-manager", metav1.GetOptions{})
return err
}, time.Second*30, time.Second*2).Should(gomega.Succeed())
}, time.Second*60, time.Second*2).Should(gomega.Succeed())

util.WaitClusterManagerApplied(operatorClient)
gomega.Expect(len(clusterManager.Spec.RegistrationConfiguration.RegistrationDrivers)).To(
gomega.Equal(2), "should have 2 registration drivers")

gomega.Expect(clusterManager.Spec.ServerConfiguration.EndpointsExposure[0].Protocol).To(
gomega.Equal("grpc"), "server config endpoint exposure protocol should be grpc")

ginkgo.By(fmt.Sprintf("join hub as managedCluster %s with grpc", e2e.Cluster().Hub().Name()))
err = e2e.Clusteradm().Join(
Expand Down
4 changes: 3 additions & 1 deletion test/e2e/util/helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,9 @@ func WaitClustersDeleted(restcfg *rest.Config) error {
return err
}
}
return fmt.Errorf("wait all clusters are deleted: %v", clusterList.Items)

fmt.Printf("wait for all clusters to be deleted: %+v\n", clusterList.Items)
return fmt.Errorf("not all clusters are deleted")
}, time.Second*300, time.Second*2).Should(gomega.Succeed())

return nil
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/util/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ func initE2E() (*TestE2eConfig, error) {
return err
}

fmt.Println("unjoin managedcluster1...")
fmt.Println("unjoin managedCluster on the spoke cluster...")
err := e2eConf.Clusteradm().Unjoin(
"--context", e2eConf.Cluster().ManagedCluster1().Context(),
"--cluster-name", e2eConf.Cluster().ManagedCluster1().Name(),
Expand Down
2 changes: 1 addition & 1 deletion vendor/modules.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1270,7 +1270,7 @@ open-cluster-management.io/managed-serviceaccount/pkg/generated/clientset/versio
open-cluster-management.io/managed-serviceaccount/pkg/generated/clientset/versioned/scheme
open-cluster-management.io/managed-serviceaccount/pkg/generated/clientset/versioned/typed/authentication/v1alpha1
open-cluster-management.io/managed-serviceaccount/pkg/generated/clientset/versioned/typed/authentication/v1beta1
# open-cluster-management.io/ocm v1.1.0
# open-cluster-management.io/ocm v1.1.1-0.20251105064423-d80ec55608e7
## explicit; go 1.24.0
open-cluster-management.io/ocm/deploy/cluster-manager/chart
open-cluster-management.io/ocm/deploy/klusterlet/chart
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading