From ff6881e509112f4bc7cd60c957b9f4d3898c6f34 Mon Sep 17 00:00:00 2001 From: hejianpeng Date: Fri, 19 Aug 2022 15:36:59 +0800 Subject: [PATCH 1/2] support pixie vizier Signed-off-by: hejianpeng --- cmd/kurator/app/install/install.go | 2 + cmd/kurator/app/install/pixie/pixie.go | 39 +++ .../app/install/pixie/vizier/vizier.go | 60 ++++ manifests/profiles/components.yaml | 6 + pkg/client/client.go | 18 ++ pkg/client/helmclient.go | 92 +++++++ pkg/plugin/istio/install.go | 2 +- pkg/plugin/pixie/vizier/plugin.go | 256 ++++++++++++++++++ 8 files changed, 474 insertions(+), 1 deletion(-) create mode 100644 cmd/kurator/app/install/pixie/pixie.go create mode 100644 cmd/kurator/app/install/pixie/vizier/vizier.go create mode 100644 pkg/client/helmclient.go create mode 100644 pkg/plugin/pixie/vizier/plugin.go diff --git a/cmd/kurator/app/install/install.go b/cmd/kurator/app/install/install.go index 039770b0d..82df67c31 100644 --- a/cmd/kurator/app/install/install.go +++ b/cmd/kurator/app/install/install.go @@ -23,6 +23,7 @@ import ( "kurator.dev/kurator/cmd/kurator/app/install/istio" "kurator.dev/kurator/cmd/kurator/app/install/karmada" "kurator.dev/kurator/cmd/kurator/app/install/kubeedge" + "kurator.dev/kurator/cmd/kurator/app/install/pixie" "kurator.dev/kurator/cmd/kurator/app/install/prometheus" "kurator.dev/kurator/cmd/kurator/app/install/submariner" "kurator.dev/kurator/cmd/kurator/app/install/volcano" @@ -46,5 +47,6 @@ func NewCmd(opts *generic.Options) *cobra.Command { installCmd.AddCommand(prometheus.NewCmd(opts)) installCmd.AddCommand(submariner.NewCmd(opts)) installCmd.AddCommand(argocd.NewCmd(opts)) + installCmd.AddCommand(pixie.NewCmd(opts)) return installCmd } diff --git a/cmd/kurator/app/install/pixie/pixie.go b/cmd/kurator/app/install/pixie/pixie.go new file mode 100644 index 000000000..e0a4dca0f --- /dev/null +++ b/cmd/kurator/app/install/pixie/pixie.go @@ -0,0 +1,39 @@ +/* +Copyright Kurator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pixie + +import ( + "github.com/spf13/cobra" + + "kurator.dev/kurator/cmd/kurator/app/install/pixie/vizier" + "kurator.dev/kurator/pkg/generic" +) + +func NewCmd(opts *generic.Options) *cobra.Command { + pixieCmd := &cobra.Command{ + Use: "pixie", + Short: "Install pixie component", + DisableFlagsInUseLine: true, + FParseErrWhitelist: cobra.FParseErrWhitelist{ + UnknownFlags: true, + }, + } + + pixieCmd.AddCommand(vizier.NewCmd(opts)) + + return pixieCmd +} diff --git a/cmd/kurator/app/install/pixie/vizier/vizier.go b/cmd/kurator/app/install/pixie/vizier/vizier.go new file mode 100644 index 000000000..856534c80 --- /dev/null +++ b/cmd/kurator/app/install/pixie/vizier/vizier.go @@ -0,0 +1,60 @@ +/* +Copyright Kurator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vizier + +import ( + "fmt" + + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "kurator.dev/kurator/pkg/generic" + plugin "kurator.dev/kurator/pkg/plugin/pixie/vizier" +) + +const ( + communityCloudAddr = "withpixie.ai:443" + pxNamespace = "px" +) + +func NewCmd(opts *generic.Options) *cobra.Command { + pluginArgs := plugin.InstallArgs{} + vizierCmd := &cobra.Command{ + Use: "vizier", + Short: "Install vizier component", + RunE: func(c *cobra.Command, args []string) error { + plugin, err := plugin.NewPlugin(opts, &pluginArgs) + if err != nil { + logrus.Errorf("pixie vizier init error: %v", err) + return fmt.Errorf("pixie vizier init error: %v", err) + } + + if err := plugin.Execute(args, nil); err != nil { + logrus.Errorf("pixie vizier execute error: %v", err) + return fmt.Errorf("pixie vizier execute error: %v", err) + } + logrus.Info("pixie vizier install completed.") + return nil + }, + } + + vizierCmd.PersistentFlags().StringVar(&pluginArgs.PxNamespace, "px-namespace", pxNamespace, "The namespace use to install vizier.") + vizierCmd.PersistentFlags().StringVar(&pluginArgs.CloudAddress, "cloud-addr", communityCloudAddr, "The address of the Pixie cloud instance that the vizier should be connected to.") + vizierCmd.PersistentFlags().StringVar(&pluginArgs.DeployKey, "deploy-key", "", "The deploy key is used to link the deployed vizier to a specific user/project.") + + return vizierCmd +} diff --git a/manifests/profiles/components.yaml b/manifests/profiles/components.yaml index 6b560b685..7dca2ec85 100644 --- a/manifests/profiles/components.yaml +++ b/manifests/profiles/components.yaml @@ -19,8 +19,14 @@ components: version: 1.5.1 hub: docker.io/volcanosh releaseURLPrefix: "https://raw.githubusercontent.com/volcano-sh/volcano/" + - name: pixie + version: 0.0.30 - name: prometheus version: - name: argocd version: v2.4.8 releaseURLPrefix: "https://github.com/argoproj/argo-cd/releases/download/" + # FIXME: https://github.com/kurator-dev/kurator/issues/61 + - name: helm + version: v3.9.3 + releaseURLPrefix: "https://get.helm.sh/" diff --git a/pkg/client/client.go b/pkg/client/client.go index b0cc8195c..88851fc30 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -143,3 +143,21 @@ func (c *Client) NewClusterClientSet(clusterName string) (kubeclient.Interface, } return kubeclient.NewForConfig(clusterConfig) } + +func (c *Client) NewClusterCRDClientset(clusterName string) (crdclientset.Interface, error) { + clusterConfig, err := c.memberClusterConfig(clusterName) + if err != nil { + return nil, err + } + return crdclientset.NewForConfig(clusterConfig) +} + +func (c *Client) NewClusterHelmClient(clusterName string) (helmclient.Interface, error) { + clusterConfig, err := c.memberClusterConfig(clusterName) + if err != nil { + return nil, err + } + + clusterGetter := NewRESTClientGetter(clusterConfig) + return helmclient.New(clusterGetter), nil +} diff --git a/pkg/client/helmclient.go b/pkg/client/helmclient.go new file mode 100644 index 000000000..3e4fa089b --- /dev/null +++ b/pkg/client/helmclient.go @@ -0,0 +1,92 @@ +/* +Copyright Kurator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/client-go/discovery" + "k8s.io/client-go/discovery/cached/memory" + "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" + "k8s.io/client-go/tools/clientcmd" +) + +// RESTClientGetter defines the values of a helm REST client. +type RESTClientGetter struct { + restConfig *rest.Config +} + +// NewRESTClientGetter returns a RESTClientGetter using the provided 'restConfig'. +// +// source: https://github.com/helm/helm/issues/6910#issuecomment-601277026 +func NewRESTClientGetter(restConfig *rest.Config) *RESTClientGetter { + return &RESTClientGetter{ + restConfig: restConfig, + } +} + +// ToRESTConfig returns a REST config build from a given kubeconfig +func (c *RESTClientGetter) ToRESTConfig() (*rest.Config, error) { + if c.restConfig != nil { + return c.restConfig, nil + } + + return nil, fmt.Errorf("restconfig can not be empty") +} + +// ToDiscoveryClient returns a CachedDiscoveryInterface that can be used as a discovery client. +func (c *RESTClientGetter) ToDiscoveryClient() (discovery.CachedDiscoveryInterface, error) { + config, err := c.ToRESTConfig() + if err != nil { + return nil, err + } + + // The more API groups exist, the more discovery requests need to be made. + // Given 25 API groups with about one version each, discovery needs to make 50 requests. + // This setting is only used for discovery. + config.Burst = 100 + + discoveryClient, err := discovery.NewDiscoveryClientForConfig(config) + if err != nil { + return nil, err + } + return memory.NewMemCacheClient(discoveryClient), nil +} + +func (c *RESTClientGetter) ToRESTMapper() (meta.RESTMapper, error) { + discoveryClient, err := c.ToDiscoveryClient() + if err != nil { + return nil, err + } + + mapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient) + expander := restmapper.NewShortcutExpander(mapper, discoveryClient) + return expander, nil +} + +func (c *RESTClientGetter) ToRawKubeConfigLoader() clientcmd.ClientConfig { + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + // use the standard defaults for this client command + // DEPRECATED: remove and replace with something more accurate + loadingRules.DefaultClientConfig = &clientcmd.DefaultClientConfig + + overrides := &clientcmd.ConfigOverrides{ClusterDefaults: clientcmd.ClusterDefaults} + + return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, overrides) +} diff --git a/pkg/plugin/istio/install.go b/pkg/plugin/istio/install.go index 8d3eac3dc..afab3c842 100644 --- a/pkg/plugin/istio/install.go +++ b/pkg/plugin/istio/install.go @@ -434,7 +434,7 @@ func (p *IstioPlugin) installRemotes(remotePilotAddress string) error { defer wg.Done() err := waitIngressgatewayReady(p.Client, p.options, cluster) if err != nil { - _ = multierror.Append(multiErr, err) + multiErr = multierror.Append(multiErr, err) } }(remote) } diff --git a/pkg/plugin/pixie/vizier/plugin.go b/pkg/plugin/pixie/vizier/plugin.go new file mode 100644 index 000000000..9213fc633 --- /dev/null +++ b/pkg/plugin/pixie/vizier/plugin.go @@ -0,0 +1,256 @@ +/* +Copyright Kurator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vizier + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + + karmadautil "github.com/karmada-io/karmada/pkg/util" + "github.com/sirupsen/logrus" + helmclient "helm.sh/helm/v3/pkg/kube" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/resource" + + "kurator.dev/kurator/pkg/client" + "kurator.dev/kurator/pkg/generic" + "kurator.dev/kurator/pkg/moreos" + "kurator.dev/kurator/pkg/util" +) + +const ( + crdKind = "CustomResourceDefinition" + vizierCRDName = "viziers.px.dev" + + repoName = "pixie-operator" + repoAddress = "https://pixie-operator-charts.storage.googleapis.com" +) + +var helmBinary = filepath.Join("helm" + moreos.Exe) + +type InstallArgs struct { + PxNamespace string + CloudAddress string + DeployKey string +} + +type Plugin struct { + *client.Client + + args *InstallArgs + options *generic.Options + // use helm to install pixie, because of the cli of pixie need login and store token in local and karmada issues: + // https://github.com/karmada-io/karmada/issues/2393, https://github.com/karmada-io/karmada/issues/2392 + helm string +} + +func NewPlugin(s *generic.Options, args *InstallArgs) (*Plugin, error) { + plugin := &Plugin{ + options: s, + args: args, + } + rest := s.RESTClientGetter() + c, err := client.NewClient(rest) + if err != nil { + return nil, err + } + plugin.Client = c + plugin.helm = "helm" + + return plugin, nil +} + +func (p *Plugin) Execute(cmdArgs, environment []string) error { + if err := p.installHelm(); err != nil { + return err + } + + if err := p.addRepo(); err != nil { + return err + } + + clusters, err := p.allClusters() + if err != nil { + return err + } + + for _, c := range clusters { + clusterClient, err := p.Client.NewClusterClientSet(c) + if err != nil { + return err + } + + if _, err := karmadautil.EnsureNamespaceExist(clusterClient, p.args.PxNamespace, p.options.DryRun); err != nil { + return fmt.Errorf("failed to ensure namespace %s, %w", p.args.PxNamespace, err) + } + + clusterHelmClient, err := p.Client.NewClusterHelmClient(c) + if err != nil { + return err + } + + _, err = p.applyCrds(clusterHelmClient) + if err != nil { + return err + } + + clusterCRDClient, err := p.Client.NewClusterCRDClientset(c) + if err != nil { + return err + } + if err := util.WaitCRDReady(clusterCRDClient, vizierCRDName, p.options.WaitInterval, p.options.WaitTimeout); err != nil { + return fmt.Errorf("wait cluster %s CRD %s ready fail, %w", c, vizierCRDName, err) + } + + _, err = p.applyTemplates(clusterHelmClient, c) + if err != nil { + return err + } + } + + return nil +} + +func (p *Plugin) addRepo() error { + addArgs := []string{ + "repo", + "add", + repoName, + repoAddress, + } + addCmd := exec.Command(p.helm, addArgs...) + err := util.RunCommand(addCmd) + if err != nil { + return err + } + + cmd := exec.Command(p.helm, "repo", "update") + return util.RunCommand(cmd) +} + +func (p *Plugin) applyCrds(helmClient helmclient.Interface) (helmclient.ResourceList, error) { + args := []string{ + "show", + "crds", + fmt.Sprintf("%s/%s", repoName, "pixie-operator-chart"), + } + + cmd := exec.Command(p.helm, args...) + out, err := cmd.CombinedOutput() + if err != nil { + return nil, errors.New(string(out)) + } + + r, err := helmClient.Build(bytes.NewBuffer(out), false) + if err != nil { + logrus.Debugf("crds: %s", out) + return nil, fmt.Errorf("failed to build crds: %w", err) + } + + if _, err := helmClient.Create(r); err != nil { + return r, err + } + + return r, nil +} + +func (p *Plugin) applyTemplates(helmClient helmclient.Interface, cluster string) (helmclient.ResourceList, error) { + args := []string{ + "template", + "--namespace", p.args.PxNamespace, + fmt.Sprintf("%s/%s", repoName, "pixie-operator-chart"), + "--set", fmt.Sprintf("clusterName=%s", cluster), + "--set", fmt.Sprintf("cloudAddr=%s", p.args.CloudAddress), + "--set", fmt.Sprintf("deployKey=%s", p.args.DeployKey), + } + + logrus.Debugf("helm template with args: %v", args) + cmd := exec.Command(p.helm, args...) + out, err := cmd.CombinedOutput() + if err != nil { + return nil, errors.New(string(out)) + } + + r, err := helmClient.Build(bytes.NewBuffer(out), false) + if err != nil { + return nil, err + } + + r = r.Filter(func(r *resource.Info) bool { + // crd created in prev steps + return r.Mapping.GroupVersionKind.Kind != crdKind + }) + + if _, err := helmClient.Create(r); err != nil { + return r, err + } + + return r, nil +} + +func (p *Plugin) allClusters() ([]string, error) { + clusters, err := p.KarmadaClient().ClusterV1alpha1().Clusters().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return nil, err + } + + clusterNames := make([]string, 0, len(clusters.Items)) + for _, c := range clusters.Items { + clusterNames = append(clusterNames, c.Name) + } + + return clusterNames, nil +} + +func (p *Plugin) installHelm() error { + helmComponent := p.options.Components["helm"] + + // TODO: refactor all download code as https://github.com/kurator-dev/kurator/issues/61 + installPath := filepath.Join(p.options.HomeDir, helmComponent.Name, helmComponent.Version) + helmPath := filepath.Join(installPath, fmt.Sprintf("%s-%s", util.OSExt(), runtime.GOARCH), helmBinary) + _, err := os.Stat(helmPath) + if err == nil { + p.helm = helmPath + return nil + } + + if os.IsNotExist(err) { + if err = os.MkdirAll(installPath, 0o750); err != nil { + return fmt.Errorf("unable to create directory %q: %w", installPath, err) + } + // https://get.helm.sh/helm-v3.9.3-linux-amd64.tar.gz + url, _ := util.JoinUrlPath(helmComponent.ReleaseURLPrefix, + fmt.Sprintf("helm-%s-%s-%s.tar.gz", helmComponent.Version, util.OSExt(), runtime.GOARCH)) + if _, err := util.DownloadResource(url, installPath); err != nil { + return fmt.Errorf("unable to get helm binary %q: %w", installPath, err) + } + } + + b, err := util.VerifyExecutableBinary(helmPath) + if err != nil { + return err + } + + p.helm = b + return err +} From ff090d0f335c244373177781a0113b8b789cb465 Mon Sep 17 00:00:00 2001 From: hejianpeng Date: Tue, 23 Aug 2022 11:12:30 +0800 Subject: [PATCH 2/2] waitCRDReady should wait all CRDs Signed-off-by: hejianpeng --- cmd/kurator/app/install/pixie/pixie.go | 2 +- pkg/plugin/pixie/vizier/plugin.go | 41 +++++++++++++++++++------- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/cmd/kurator/app/install/pixie/pixie.go b/cmd/kurator/app/install/pixie/pixie.go index e0a4dca0f..6f3bffb5a 100644 --- a/cmd/kurator/app/install/pixie/pixie.go +++ b/cmd/kurator/app/install/pixie/pixie.go @@ -26,7 +26,7 @@ import ( func NewCmd(opts *generic.Options) *cobra.Command { pixieCmd := &cobra.Command{ Use: "pixie", - Short: "Install pixie component", + Short: "Install pixie component", DisableFlagsInUseLine: true, FParseErrWhitelist: cobra.FParseErrWhitelist{ UnknownFlags: true, diff --git a/pkg/plugin/pixie/vizier/plugin.go b/pkg/plugin/pixie/vizier/plugin.go index 9213fc633..73e376298 100644 --- a/pkg/plugin/pixie/vizier/plugin.go +++ b/pkg/plugin/pixie/vizier/plugin.go @@ -25,7 +25,9 @@ import ( "os/exec" "path/filepath" "runtime" + "sync" + "github.com/hashicorp/go-multierror" karmadautil "github.com/karmada-io/karmada/pkg/util" "github.com/sirupsen/logrus" helmclient "helm.sh/helm/v3/pkg/kube" @@ -39,8 +41,7 @@ import ( ) const ( - crdKind = "CustomResourceDefinition" - vizierCRDName = "viziers.px.dev" + crdKind = "CustomResourceDefinition" repoName = "pixie-operator" repoAddress = "https://pixie-operator-charts.storage.googleapis.com" @@ -109,18 +110,14 @@ func (p *Plugin) Execute(cmdArgs, environment []string) error { return err } - _, err = p.applyCrds(clusterHelmClient) + crds, err := p.applyCRDs(clusterHelmClient) if err != nil { return err } - clusterCRDClient, err := p.Client.NewClusterCRDClientset(c) - if err != nil { + if err := p.waitCRDReady(c, crds); err != nil { return err } - if err := util.WaitCRDReady(clusterCRDClient, vizierCRDName, p.options.WaitInterval, p.options.WaitTimeout); err != nil { - return fmt.Errorf("wait cluster %s CRD %s ready fail, %w", c, vizierCRDName, err) - } _, err = p.applyTemplates(clusterHelmClient, c) if err != nil { @@ -148,7 +145,7 @@ func (p *Plugin) addRepo() error { return util.RunCommand(cmd) } -func (p *Plugin) applyCrds(helmClient helmclient.Interface) (helmclient.ResourceList, error) { +func (p *Plugin) applyCRDs(helmClient helmclient.Interface) (helmclient.ResourceList, error) { args := []string{ "show", "crds", @@ -174,6 +171,30 @@ func (p *Plugin) applyCrds(helmClient helmclient.Interface) (helmclient.Resource return r, nil } +func (p *Plugin) waitCRDReady(cluster string, crdList helmclient.ResourceList) error { + clusterCRDClient, err := p.Client.NewClusterCRDClientset(cluster) + if err != nil { + return err + } + + var ( + wg = sync.WaitGroup{} + multiErr *multierror.Error + ) + for _, r := range crdList { + wg.Add(1) + go func(crd *resource.Info) { + defer wg.Done() + if err := util.WaitCRDReady(clusterCRDClient, crd.Name, p.options.WaitInterval, p.options.WaitTimeout); err != nil { + multiErr = multierror.Append(multiErr, fmt.Errorf("wait cluster %s CRD %s ready fail, %w", cluster, crd.Name, err)) + } + }(r) + } + wg.Wait() + + return multiErr.ErrorOrNil() +} + func (p *Plugin) applyTemplates(helmClient helmclient.Interface, cluster string) (helmclient.ResourceList, error) { args := []string{ "template", @@ -197,7 +218,7 @@ func (p *Plugin) applyTemplates(helmClient helmclient.Interface, cluster string) } r = r.Filter(func(r *resource.Info) bool { - // crd created in prev steps + // should not happen in helm3, just in case return r.Mapping.GroupVersionKind.Kind != crdKind })