-
Notifications
You must be signed in to change notification settings - Fork 821
/
promote.go
496 lines (402 loc) · 17.1 KB
/
promote.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
package karmadactl
import (
"context"
"fmt"
"io"
"os"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/cli-runtime/pkg/genericclioptions"
"k8s.io/cli-runtime/pkg/printers"
"k8s.io/cli-runtime/pkg/resource"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/rest"
cmdutil "k8s.io/kubectl/pkg/cmd/util"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1"
workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2"
karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned"
"github.com/karmada-io/karmada/pkg/karmadactl/options"
"github.com/karmada-io/karmada/pkg/resourceinterpreter/defaultinterpreter/prune"
"github.com/karmada-io/karmada/pkg/util/gclient"
"github.com/karmada-io/karmada/pkg/util/restmapper"
)
var (
promoteShort = `Promote resources from legacy clusters to karmada control plane.`
promoteLong = `Promote resources from legacy clusters to karmada control plane. Requires the cluster be joined or registered.`
)
// NewCmdPromote defines the `promote` command that promote resources from legacy clusters
func NewCmdPromote(cmdOut io.Writer, karmadaConfig KarmadaConfig, parentCommand string) *cobra.Command {
opts := CommandPromoteOption{}
opts.JSONYamlPrintFlags = genericclioptions.NewJSONYamlPrintFlags()
cmd := &cobra.Command{
Use: "promote <RESOURCE_TYPE> <RESOURCE_NAME> -n <NAME_SPACE> -c <CLUSTER_NAME>",
Short: promoteShort,
Long: promoteLong,
Example: promoteExample(parentCommand),
SilenceUsage: true,
RunE: func(cmd *cobra.Command, args []string) error {
if err := opts.Complete(args); err != nil {
return err
}
if err := opts.Validate(); err != nil {
return err
}
if err := RunPromote(cmdOut, karmadaConfig, opts, args); err != nil {
return err
}
return nil
},
}
flag := cmd.Flags()
opts.AddFlags(flag)
return cmd
}
func promoteExample(parentCommand string) string {
example := `
# Promote deployment(default/nginx) from cluster1 to Karmada` + "\n" +
fmt.Sprintf("%s promote deployment nginx -n default -c cluster1", parentCommand) + `
# Promote deployment((default/nginx) with gvk from cluster1 to Karmada` + "\n" +
fmt.Sprintf("%s promote deployment.v1.apps nginx -n default -c cluster1", parentCommand) + `
# Dumps the artifacts but does not deploy them to Karmada, same as 'dry run'` + "\n" +
fmt.Sprintf("%s promote deployment nginx -n default -c cluster1 -o yaml|json", parentCommand) + `
# Promote secret(default/default-token) from cluster1 to Karmada` + "\n" +
fmt.Sprintf("%s promote secret default-token -n default -c cluster1", parentCommand) + `
# Support to use '--cluster-kubeconfig' to specify the configuration of member cluster` + "\n" +
fmt.Sprintf("%s promote deployment nginx -n default -c cluster1 --cluster-kubeconfig=<CLUSTER_KUBECONFIG_PATH>", parentCommand) + `
# Support to use '--cluster-kubeconfig' and '--cluster-context' to specify the configuration of member cluster` + "\n" +
fmt.Sprintf("%s promote deployment nginx -n default -c cluster1 --cluster-kubeconfig=<CLUSTER_KUBECONFIG_PATH> --cluster-context=<CLUSTER_CONTEXT>", parentCommand)
return example
}
// CommandPromoteOption holds all command options for promote
type CommandPromoteOption struct {
options.GlobalCommandOptions
// Cluster is the name of legacy cluster
Cluster string
// ClusterNamespace holds the namespace name where the member cluster objects are stored.
ClusterNamespace string
// Namespace is the namespace of legacy resource
Namespace string
// ClusterContext is the cluster's context that we are going to join with.
ClusterContext string
// ClusterKubeConfig is the cluster's kubeconfig path.
ClusterKubeConfig string
resource.FilenameOptions
JSONYamlPrintFlags *genericclioptions.JSONYamlPrintFlags
OutputFormat string
Printer func(*meta.RESTMapping, *bool, bool, bool) (printers.ResourcePrinterFunc, error)
TemplateFlags *genericclioptions.KubeTemplatePrintFlags
name string
gvk schema.GroupVersionKind
}
// AddFlags adds flags to the specified FlagSet.
func (o *CommandPromoteOption) AddFlags(flags *pflag.FlagSet) {
o.GlobalCommandOptions.AddFlags(flags)
flags.StringVarP(&o.OutputFormat, "output", "o", "", "Output format. One of: json|yaml")
flags.StringVarP(&o.Namespace, "namespace", "n", "default", "-n=namespace or -n namespace")
flags.StringVarP(&o.Cluster, "cluster", "c", "", "the name of legacy cluster (eg -c=member1)")
flags.StringVar(&o.ClusterNamespace, "cluster-namespace", options.DefaultKarmadaClusterNamespace, "Namespace in the control plane where member cluster are stored.")
flags.StringVar(&o.ClusterContext, "cluster-context", "",
"Context name of legacy cluster in kubeconfig. Only works when there are multiple contexts in the kubeconfig.")
flags.StringVar(&o.ClusterKubeConfig, "cluster-kubeconfig", "",
"Path of the legacy cluster's kubeconfig.")
}
// Complete ensures that options are valid and marshals them if necessary
func (o *CommandPromoteOption) Complete(args []string) error {
if len(args) != 2 {
return fmt.Errorf("incorrect command format, please use correct command format")
}
o.name = args[1]
if o.OutputFormat == "yaml" || o.OutputFormat == "json" {
o.Printer = func(mapping *meta.RESTMapping, outputObjects *bool, withNamespace bool, withKind bool) (printers.ResourcePrinterFunc, error) {
printer, err := o.JSONYamlPrintFlags.ToPrinter(o.OutputFormat)
if genericclioptions.IsNoCompatiblePrinterError(err) {
return nil, err
}
printer, err = printers.NewTypeSetter(gclient.NewSchema()).WrapToPrinter(printer, nil)
if err != nil {
return nil, err
}
return printer.PrintObj, nil
}
}
return nil
}
// Validate checks to the PromoteOptions to see if there is sufficient information run the command
func (o *CommandPromoteOption) Validate() error {
if o.Cluster == "" {
return fmt.Errorf("the cluster cannot be empty")
}
if o.OutputFormat != "" && o.OutputFormat != "yaml" && o.OutputFormat != "json" {
return fmt.Errorf("output format is only one of json and yaml")
}
// If '--cluster-context' not specified, take the cluster name as the context.
if len(o.ClusterContext) == 0 {
o.ClusterContext = o.Cluster
}
return nil
}
// RunPromote promote resources from legacy clusters
func RunPromote(_ io.Writer, karmadaConfig KarmadaConfig, opts CommandPromoteOption, args []string) error {
// Get control plane karmada-apiserver client
controlPlaneRestConfig, err := karmadaConfig.GetRestConfig(opts.KarmadaContext, opts.KubeConfig)
if err != nil {
return fmt.Errorf("failed to get control plane rest config. context: %s, kubeconfig: %s, error: %v",
opts.KarmadaContext, opts.KubeConfig, err)
}
clusterInfos := make(map[string]*ClusterInfo)
if err := getClusterInKarmada(controlPlaneRestConfig, clusterInfos); err != nil {
return fmt.Errorf("failed to get cluster info in karmada control plane. err: %v", err)
}
if _, exist := clusterInfos[opts.Cluster]; !exist {
return fmt.Errorf("cluster(%s) does't exist in karmada control plane", opts.Cluster)
}
var f cmdutil.Factory
if opts.ClusterKubeConfig != "" {
kubeConfigFlags := genericclioptions.NewConfigFlags(true).WithDeprecatedPasswordFlag()
kubeConfigFlags.KubeConfig = &opts.ClusterKubeConfig
kubeConfigFlags.Context = &opts.ClusterContext
f = cmdutil.NewFactory(kubeConfigFlags)
} else {
opts.setClusterProxyInfo(controlPlaneRestConfig, opts.Cluster, clusterInfos)
f = getFactory(opts.Cluster, clusterInfos)
}
objInfo, err := opts.getObjInfo(f, opts.Cluster, args)
if err != nil {
return fmt.Errorf("failed to get resource in cluster(%s). err: %v", opts.Cluster, err)
}
obj := objInfo.Info.Object.(*unstructured.Unstructured)
opts.gvk = obj.GetObjectKind().GroupVersionKind()
mapper, err := apiutil.NewDynamicRESTMapper(controlPlaneRestConfig)
if err != nil {
return fmt.Errorf("failed to create restmapper: %v", err)
}
gvr, err := restmapper.GetGroupVersionResource(mapper, opts.gvk)
if err != nil {
return fmt.Errorf("failed to get gvr from %q: %v", opts.gvk, err)
}
return promote(controlPlaneRestConfig, obj, gvr, opts)
}
func promote(controlPlaneRestConfig *rest.Config, obj *unstructured.Unstructured, gvr schema.GroupVersionResource, opts CommandPromoteOption) error {
if err := preprocessResource(obj); err != nil {
return fmt.Errorf("failed to preprocess resource %q(%s/%s) in control plane: %v", gvr, opts.Namespace, opts.name, err)
}
if opts.OutputFormat != "" {
// only print the resource template and Policy
err := printObjectAndPolicy(obj, gvr, opts)
return err
}
if opts.DryRun {
return nil
}
controlPlaneDynamicClient := dynamic.NewForConfigOrDie(controlPlaneRestConfig)
karmadaClient := karmadaclientset.NewForConfigOrDie(controlPlaneRestConfig)
if len(obj.GetNamespace()) == 0 {
_, err := controlPlaneDynamicClient.Resource(gvr).Get(context.TODO(), opts.name, metav1.GetOptions{})
if err == nil {
fmt.Printf("Resource %q(%s) already exist in karmada control plane, you can edit PropagationPolicy and OverridePolicy to propagate it\n",
gvr, opts.name)
return nil
}
if !apierrors.IsNotFound(err) {
return fmt.Errorf("failed to get resource %q(%s) in control plane: %v", gvr, opts.name, err)
}
_, err = controlPlaneDynamicClient.Resource(gvr).Create(context.TODO(), obj, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("failed to create resource %q(%s) in control plane: %v", gvr, opts.name, err)
}
err = createOrUpdateClusterPropagationPolicy(karmadaClient, gvr, opts)
if err != nil {
return err
}
fmt.Printf("Resource %q(%s) is promoted successfully\n", gvr, opts.name)
} else {
_, err := controlPlaneDynamicClient.Resource(gvr).Namespace(opts.Namespace).Get(context.TODO(), opts.name, metav1.GetOptions{})
if err == nil {
fmt.Printf("Resource %q(%s/%s) already exist in karmada control plane, you can edit PropagationPolicy and OverridePolicy to propagate it\n",
gvr, opts.Namespace, opts.name)
return nil
}
if !apierrors.IsNotFound(err) {
return fmt.Errorf("failed to get resource %q(%s/%s) in control plane: %v", gvr, opts.Namespace, opts.name, err)
}
_, err = controlPlaneDynamicClient.Resource(gvr).Namespace(opts.Namespace).Create(context.TODO(), obj, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("failed to create resource %q(%s/%s) in control plane: %v", gvr, opts.Namespace, opts.name, err)
}
err = createOrUpdatePropagationPolicy(karmadaClient, gvr, opts)
if err != nil {
return err
}
fmt.Printf("Resource %q(%s/%s) is promoted successfully\n", gvr, opts.Namespace, opts.name)
}
return nil
}
// getObjInfo get obj info in member cluster
func (o *CommandPromoteOption) getObjInfo(f cmdutil.Factory, cluster string, args []string) (*Obj, error) {
chunkSize := int64(500)
r := f.NewBuilder().
Unstructured().
NamespaceParam(o.Namespace).
FilenameParam(false, &o.FilenameOptions).
RequestChunksOf(chunkSize).
ResourceTypeOrNameArgs(true, args...).
ContinueOnError().
Latest().
Flatten().
Do()
r.IgnoreErrors(apierrors.IsNotFound)
infos, err := r.Infos()
if err != nil {
return nil, err
}
if len(infos) == 0 {
return nil, fmt.Errorf("the %s or %s don't exist in cluster(%s)", args[0], args[1], o.Cluster)
}
obj := &Obj{
Cluster: cluster,
Info: infos[0],
}
return obj, nil
}
// setClusterProxyInfo set proxy information of cluster
func (o *CommandPromoteOption) setClusterProxyInfo(karmadaRestConfig *rest.Config, name string, clusterInfos map[string]*ClusterInfo) {
clusterInfos[name].APIEndpoint = karmadaRestConfig.Host + fmt.Sprintf(proxyURL, name)
clusterInfos[name].KubeConfig = o.KubeConfig
clusterInfos[name].Context = o.KarmadaContext
if clusterInfos[name].KubeConfig == "" {
env := os.Getenv("KUBECONFIG")
if env != "" {
clusterInfos[name].KubeConfig = env
} else {
clusterInfos[name].KubeConfig = defaultKubeConfig
}
}
}
// printObjectAndPolicy print the converted resource
func printObjectAndPolicy(obj *unstructured.Unstructured, gvr schema.GroupVersionResource, opts CommandPromoteOption) error {
printer, err := opts.Printer(nil, nil, false, false)
if err != nil {
return fmt.Errorf("failed to initialize k8s printer. err: %v", err)
}
if err = printer.PrintObj(obj, os.Stdout); err != nil {
return fmt.Errorf("failed to print the resource template. err: %v", err)
}
if len(obj.GetNamespace()) == 0 {
cpp := buildClusterPropagationPolicy(gvr, opts)
if err = printer.PrintObj(cpp, os.Stdout); err != nil {
return fmt.Errorf("failed to print the ClusterPropagationPolicy. err: %v", err)
}
} else {
pp := buildPropagationPolicy(gvr, opts)
if err = printer.PrintObj(pp, os.Stdout); err != nil {
return fmt.Errorf("failed to print the PropagationPolicy. err: %v", err)
}
}
return nil
}
// preprocessResource delete redundant fields to convert resource as template
func preprocessResource(obj *unstructured.Unstructured) error {
// remove fields that generated by kube-apiserver and no need(or can't) propagate to member clusters.
if err := prune.RemoveIrrelevantField(obj); err != nil {
return err
}
addOverwriteAnnotation(obj)
return nil
}
// addOverwriteAnnotation add annotation "work.karmada.io/conflict-resolution" to the resource
func addOverwriteAnnotation(obj *unstructured.Unstructured) {
annotations := obj.DeepCopy().GetAnnotations()
if annotations == nil {
annotations = map[string]string{}
}
if _, exist := annotations[workv1alpha2.ResourceConflictResolutionAnnotation]; !exist {
annotations[workv1alpha2.ResourceConflictResolutionAnnotation] = workv1alpha2.ResourceConflictResolutionOverwrite
obj.SetAnnotations(annotations)
}
}
// createOrUpdatePropagationPolicy create PropagationPolicy in karmada control plane
func createOrUpdatePropagationPolicy(karmadaClient *karmadaclientset.Clientset, gvr schema.GroupVersionResource, opts CommandPromoteOption) error {
name := opts.name + "-propagation"
_, err := karmadaClient.PolicyV1alpha1().PropagationPolicies(opts.Namespace).Get(context.TODO(), name, metav1.GetOptions{})
if err != nil && apierrors.IsNotFound(err) {
pp := buildPropagationPolicy(gvr, opts)
_, err = karmadaClient.PolicyV1alpha1().PropagationPolicies(opts.Namespace).Create(context.TODO(), pp, metav1.CreateOptions{})
return err
}
if err != nil {
return fmt.Errorf("failed to get PropagationPolicy(%s/%s) in control plane: %v", opts.Namespace, name, err)
}
// PropagationPolicy already exists, not to create it
return fmt.Errorf("the PropagationPolicy(%s/%s) in control plane: %v", opts.Namespace, name, err)
}
// createOrUpdateClusterPropagationPolicy create ClusterPropagationPolicy in karmada control plane
func createOrUpdateClusterPropagationPolicy(karmadaClient *karmadaclientset.Clientset, gvr schema.GroupVersionResource, opts CommandPromoteOption) error {
name := opts.name + "-propagation"
_, err := karmadaClient.PolicyV1alpha1().ClusterPropagationPolicies().Get(context.TODO(), name, metav1.GetOptions{})
if err != nil && apierrors.IsNotFound(err) {
cpp := buildClusterPropagationPolicy(gvr, opts)
_, err = karmadaClient.PolicyV1alpha1().ClusterPropagationPolicies().Create(context.TODO(), cpp, metav1.CreateOptions{})
return err
}
if err != nil {
return fmt.Errorf("failed to get ClusterPropagationPolicy(%s) in control plane: %v", name, err)
}
// ClusterPropagationPolicy already exists, not to create it
return fmt.Errorf("the ClusterPropagationPolicy(%s) already exist, please edit it to propagate resource", name)
}
// buildPropagationPolicy build PropagationPolicy according to resource and cluster
func buildPropagationPolicy(gvr schema.GroupVersionResource, opts CommandPromoteOption) *policyv1alpha1.PropagationPolicy {
name := opts.name + "-propagation"
ns := opts.Namespace
pp := &policyv1alpha1.PropagationPolicy{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: ns,
},
Spec: policyv1alpha1.PropagationSpec{
ResourceSelectors: []policyv1alpha1.ResourceSelector{
{
APIVersion: gvr.GroupVersion().String(),
Kind: opts.gvk.Kind,
Name: opts.name,
},
},
Placement: policyv1alpha1.Placement{
ClusterAffinity: &policyv1alpha1.ClusterAffinity{
ClusterNames: []string{opts.Cluster},
},
},
},
}
return pp
}
// buildClusterPropagationPolicy build ClusterPropagationPolicy according to resource and cluster
func buildClusterPropagationPolicy(gvr schema.GroupVersionResource, opts CommandPromoteOption) *policyv1alpha1.ClusterPropagationPolicy {
name := opts.name + "-propagation"
cpp := &policyv1alpha1.ClusterPropagationPolicy{
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
Spec: policyv1alpha1.PropagationSpec{
ResourceSelectors: []policyv1alpha1.ResourceSelector{
{
APIVersion: gvr.GroupVersion().String(),
Kind: opts.gvk.Kind,
Name: opts.name,
},
},
Placement: policyv1alpha1.Placement{
ClusterAffinity: &policyv1alpha1.ClusterAffinity{
ClusterNames: []string{opts.Cluster},
},
},
},
}
return cpp
}