Skip to content

Commit

Permalink
Merge pull request #2992 from nunnatsa/dump-kv-external
Browse files Browse the repository at this point in the history 
Dump kubevirt external infra clusters
  • Loading branch information
@openshift-merge-robot
openshift-merge-robot committed Sep 7, 2023
2 parents c9acc46 + 0db3aac commit 0232892
Show file tree
Hide file tree
Showing 2 changed files with 142 additions and 47 deletions.
161 changes: 125 additions & 36 deletions cmd/cluster/core/dump.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"math/rand"
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"time"
Expand Down Expand Up @@ -43,10 +44,38 @@ import (
"github.com/openshift/hypershift/cmd/log"
"github.com/openshift/hypershift/cmd/util"
"github.com/openshift/hypershift/hypershift-operator/controllers/manifests"
kvinfra "github.com/openshift/hypershift/kubevirtexternalinfra"
"github.com/openshift/hypershift/support/config"
supportutil "github.com/openshift/hypershift/support/util"
)

const (
kubevirtNamespace = "openshift-cnv"
)

var (
coreResources = []client.Object{
&appsv1.DaemonSet{},
&appsv1.Deployment{},
&appsv1.ReplicaSet{},
&appsv1.StatefulSet{},
&batchv1.Job{},
&corev1.ConfigMap{},
&corev1.Endpoints{},
&corev1.Event{},
&corev1.PersistentVolumeClaim{},
&corev1.Pod{},
&corev1.ReplicationController{},
&corev1.Service{},
}

kubevirtResources = []client.Object{
&cdiv1beta1.DataVolume{},
&kubevirtv1.VirtualMachine{},
&kubevirtv1.VirtualMachineInstance{},
}
)

type DumpOptions struct {
Namespace string
Name string
Expand All @@ -65,6 +94,12 @@ type DumpOptions struct {
Log logr.Logger
}

type kubevirtExtCluster struct {
uid string
nodePoolNS string
creds *hyperv1.KubevirtPlatformCredentials
}

func NewDumpCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "cluster",
Expand Down Expand Up @@ -279,26 +314,29 @@ func DumpCluster(ctx context.Context, opts *DumpOptions) error {
controlPlaneNamespace := manifests.HostedControlPlaneNamespace(opts.Namespace, opts.Name).Name

kubevirtInUse := false
var kubevirtInExternalInfras []kubevirtExtCluster

for _, np := range nodePools {
if np.Spec.Platform.Type == hyperv1.KubevirtPlatform {
kubevirtInUse = true
break
if np.Status.Platform != nil &&
np.Status.Platform.KubeVirt != nil &&
np.Status.Platform.KubeVirt.Credentials != nil &&
np.Status.Platform.KubeVirt.Credentials.InfraKubeConfigSecret != nil {

extInfra := kubevirtExtCluster{
uid: string(np.UID),
nodePoolNS: np.Namespace,
creds: np.Status.Platform.KubeVirt.Credentials,
}

kubevirtInExternalInfras = append(kubevirtInExternalInfras, extInfra)
} else {
kubevirtInUse = true
}
}
}

resources := []client.Object{
&appsv1.DaemonSet{},
&appsv1.Deployment{},
&appsv1.ReplicaSet{},
&appsv1.StatefulSet{},
&batchv1.Job{},
&corev1.ConfigMap{},
&corev1.Endpoints{},
&corev1.Event{},
&corev1.PersistentVolumeClaim{},
&corev1.Pod{},
&corev1.ReplicationController{},
&corev1.Service{},
resources := append(coreResources,
&capiv1.Cluster{},
&capiv1.MachineDeployment{},
&capiv1.Machine{},
Expand All @@ -317,14 +355,10 @@ func DumpCluster(ctx context.Context, opts *DumpOptions) error {
&routev1.Route{},
&imagev1.ImageStream{},
&networkingv1.NetworkPolicy{},
}
)

if kubevirtInUse {
resources = append(resources,
&cdiv1beta1.DataVolume{},
&kubevirtv1.VirtualMachine{},
&kubevirtv1.VirtualMachineInstance{},
)
resources = append(resources, kubevirtResources...)
}

resourceList := strings.Join(resourceTypes(resources), ",")
Expand All @@ -340,7 +374,7 @@ func DumpCluster(ctx context.Context, opts *DumpOptions) error {
}

if kubevirtInUse {
cmd.WithNamespace("openshift-cnv").Run(ctx, resourceList)
cmd.WithNamespace(kubevirtNamespace).Run(ctx, resourceList)
}

podList := &corev1.PodList{}
Expand All @@ -362,6 +396,15 @@ func DumpCluster(ctx context.Context, opts *DumpOptions) error {
}
}

for _, infra := range kubevirtInExternalInfras {
destDit := path.Join(opts.ArtifactDir, "external-infra-clusters", "nodepool-"+infra.uid)
opts.Log.Info("dumping external infra cluster into " + destDit)

if err = dumpKubevirtExternalCluster(ctx, c, infra.creds, infra.nodePoolNS, destDit, opts.Log); err != nil {
opts.Log.Error(err, "Failed to dump infra cluster")
}
}

if opts.ArchiveDump {
if err = CreateArchive(ctx, opts); err != nil {
return err
Expand All @@ -386,25 +429,14 @@ func DumpGuestCluster(ctx context.Context, log logr.Logger, kubeconfig string, d
kubeconfig: kubeconfig,
log: log,
}
resources := []client.Object{

resources := append(coreResources,
&apiextensionsv1.CustomResourceDefinition{},
&appsv1.ControllerRevision{},
&appsv1.DaemonSet{},
&appsv1.Deployment{},
&appsv1.ReplicaSet{},
&appsv1.StatefulSet{},
&batchv1.Job{},
&configv1.ClusterOperator{},
&corev1.ConfigMap{},
&corev1.Endpoints{},
&corev1.Event{},
&corev1.Namespace{},
&corev1.Node{},
&corev1.PersistentVolume{},
&corev1.PersistentVolumeClaim{},
&corev1.Pod{},
&corev1.ReplicationController{},
&corev1.Service{},
&rbacv1.ClusterRole{},
&rbacv1.ClusterRoleBinding{},
&rbacv1.Role{},
Expand All @@ -418,7 +450,8 @@ func DumpGuestCluster(ctx context.Context, log logr.Logger, kubeconfig string, d
// https://github.com/openshift/api/blob/2bde012f248a5172dcde2f7104caf0726cf6d93a/config/v1/types_cluster_version.go#L266-L270
&snapshotv1.VolumeSnapshotClass{},
&snapshotv1.VolumeSnapshotContent{},
}
)

resourceList := strings.Join(resourceTypes(resources), ",")
cmd.Run(ctx, resourceList)
dumpWorkerNodeLogsCmd := OCAdmNodeLogs{
Expand Down Expand Up @@ -636,3 +669,59 @@ func CreateArchive(ctx context.Context, opts *DumpOptions) error {

return nil
}

var (
kvExtResources = append(coreResources, kubevirtResources...)
kvExtResourcesList = strings.Join(resourceTypes(kvExtResources), ",")
)

func dumpKubevirtExternalCluster(ctx context.Context, mngmtCl client.Client, creds *hyperv1.KubevirtPlatformCredentials, npNS string, destDir string, log logr.Logger) error {
kubeconfig, err := kvinfra.GetKubeConfig(ctx, mngmtCl, npNS, creds.InfraKubeConfigSecret.Name)
if err != nil {
return fmt.Errorf("failed to get KubeVirt external infra-cluster: %w", err)
}

kubeconfigFile, err := writeKubeconfigToFile(kubeconfig)
if err != nil {
return err
}

defer func() {
_ = os.Remove(kubeconfigFile)
}()

ocCommand, err := exec.LookPath("oc")
if err != nil || len(ocCommand) == 0 {
return fmt.Errorf("cannot find oc command")
}
cmd := OCAdmInspect{
oc: ocCommand,
artifactDir: destDir,
kubeconfig: kubeconfigFile,
log: log,
}

namespaces := []string{creds.InfraNamespace, kubevirtNamespace}
for _, ns := range namespaces {
cmd.WithNamespace(ns).Run(ctx, kvExtResourcesList)
}

return nil
}

func writeKubeconfigToFile(data []byte) (string, error) {
kubeconfigFile, err := os.CreateTemp(os.TempDir(), "kubeconfig-")
if err != nil {
return "", fmt.Errorf("failed to get create kubeconfig file for KubeVirt external infra-cluster: %w", err)
}
_, err = kubeconfigFile.Write(data)
if err != nil {
return "", fmt.Errorf("failed to get create kubeconfig file for KubeVirt external infra-cluster: %w", err)
}
err = kubeconfigFile.Close()
if err != nil {
return "", fmt.Errorf("failed to get create kubeconfig file for KubeVirt external infra-cluster: %w", err)
}

return kubeconfigFile.Name(), err
}
28 changes: 17 additions & 11 deletions kubevirtexternalinfra/externalinfra.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,17 +82,7 @@ func (k *kubevirtInfraClientMapImp) Delete(key string) {
}

func generateKubevirtInfraClusterClient(ctx context.Context, cpClient client.Client, credentials *hyperv1.KubevirtPlatformCredentials, secretNamespace string) (client.Client, error) {
infraKubeconfigSecret := &corev1.Secret{}

infraKubeconfigSecretKey := client.ObjectKey{Namespace: secretNamespace, Name: credentials.InfraKubeConfigSecret.Name}
if err := cpClient.Get(ctx, infraKubeconfigSecretKey, infraKubeconfigSecret); err != nil {
return nil, fmt.Errorf("failed to fetch infra kubeconfig secret %s/%s: %w", secretNamespace, credentials.InfraKubeConfigSecret.Name, err)
}

kubeConfig, ok := infraKubeconfigSecret.Data["kubeconfig"]
if !ok {
return nil, errors.New("failed to retrieve infra kubeconfig from secret: 'kubeconfig' key is missing")
}
kubeConfig, err := GetKubeConfig(ctx, cpClient, secretNamespace, credentials.InfraKubeConfigSecret.Name)

clientConfig, err := clientcmd.NewClientConfigFromBytes(kubeConfig)
if err != nil {
Expand All @@ -112,3 +102,19 @@ func generateKubevirtInfraClusterClient(ctx context.Context, cpClient client.Cli

return infraClusterClient, nil
}

func GetKubeConfig(ctx context.Context, cl client.Client, secretNamespace, secretName string) ([]byte, error) {
infraKubeconfigSecret := &corev1.Secret{}

infraKubeconfigSecretKey := client.ObjectKey{Namespace: secretNamespace, Name: secretName}
if err := cl.Get(ctx, infraKubeconfigSecretKey, infraKubeconfigSecret); err != nil {
return nil, fmt.Errorf("failed to fetch infra kubeconfig secret %s/%s: %w", secretNamespace, secretName, err)
}

kubeConfig, ok := infraKubeconfigSecret.Data["kubeconfig"]
if !ok {
return nil, errors.New("failed to retrieve infra kubeconfig from secret: 'kubeconfig' key is missing")
}

return kubeConfig, nil
}

0 comments on commit 0232892

Please sign in to comment.