From e471235974d786113aad1efb3b211b9b02bed57f Mon Sep 17 00:00:00 2001 From: "kevin.qiao" Date: Sat, 25 Feb 2023 12:49:16 +0800 Subject: [PATCH] write logs to std without log PV mounted (#180) --- apis/apps/v1alpha1/nebulacluster_common.go | 2 +- apis/apps/v1alpha1/nebulacluster_graphd.go | 15 ++++ apis/apps/v1alpha1/nebulacluster_metad.go | 74 +++++++++++-------- apis/apps/v1alpha1/nebulacluster_storaged.go | 63 +++++++++------- .../component/reclaimer/meta_reconciler.go | 15 +++- 5 files changed, 111 insertions(+), 58 deletions(-) diff --git a/apis/apps/v1alpha1/nebulacluster_common.go b/apis/apps/v1alpha1/nebulacluster_common.go index 1761dc42..29e0b904 100644 --- a/apis/apps/v1alpha1/nebulacluster_common.go +++ b/apis/apps/v1alpha1/nebulacluster_common.go @@ -42,7 +42,7 @@ const ( AgentInitContainerName = "ng-init-agent" DefaultAgentPortGRPC = 8888 agentPortNameGRPC = "grpc" - defaultAgentImage = "megabyte875/nebula-agent:latest" + defaultAgentImage = "vesoft/nebula-agent:latest" defaultExporterImage = "vesoft/nebula-stats-exporter" ) diff --git a/apis/apps/v1alpha1/nebulacluster_graphd.go b/apis/apps/v1alpha1/nebulacluster_graphd.go index 18e32a2a..245e4368 100644 --- a/apis/apps/v1alpha1/nebulacluster_graphd.go +++ b/apis/apps/v1alpha1/nebulacluster_graphd.go @@ -90,6 +90,9 @@ func (c *graphdComponent) GetLogStorageClass() *string { } func (c *graphdComponent) GetLogStorageResources() *corev1.ResourceRequirements { + if c.nc.Spec.Graphd.LogVolumeClaim == nil { + return nil + } return c.nc.Spec.Graphd.LogVolumeClaim.Resources.DeepCopy() } @@ -217,6 +220,10 @@ func (c *graphdComponent) GenerateContainerPorts() []corev1.ContainerPort { } func (c *graphdComponent) GenerateVolumeMounts() []corev1.VolumeMount { + if c.nc.Spec.Graphd.LogVolumeClaim == nil { + return nil + } + componentType := c.Type().String() return []corev1.VolumeMount{ { @@ -228,6 +235,10 @@ func (c *graphdComponent) GenerateVolumeMounts() []corev1.VolumeMount { } func (c *graphdComponent) GenerateVolumes() []corev1.Volume { + if c.nc.Spec.Graphd.LogVolumeClaim == nil { + return nil + } + componentType := c.Type().String() return []corev1.Volume{ { @@ -242,6 +253,10 @@ func (c *graphdComponent) GenerateVolumes() []corev1.Volume { } func (c *graphdComponent) GenerateVolumeClaim() ([]corev1.PersistentVolumeClaim, error) { + if c.nc.Spec.Graphd.LogVolumeClaim == nil { + return nil, nil + } + componentType := c.Type().String() logSC, logRes := c.GetLogStorageClass(), c.GetLogStorageResources() storageRequest, err := parseStorageRequest(logRes.Requests) diff --git a/apis/apps/v1alpha1/nebulacluster_metad.go b/apis/apps/v1alpha1/nebulacluster_metad.go index 6ff3a844..2d1cadf6 100644 --- a/apis/apps/v1alpha1/nebulacluster_metad.go +++ b/apis/apps/v1alpha1/nebulacluster_metad.go @@ -101,10 +101,16 @@ func (c *metadComponent) GetDataStorageClass() *string { } func (c *metadComponent) GetLogStorageResources() *corev1.ResourceRequirements { + if c.nc.Spec.Metad.LogVolumeClaim == nil { + return nil + } return c.nc.Spec.Metad.LogVolumeClaim.Resources.DeepCopy() } func (c *metadComponent) GetDataStorageResources() (*corev1.ResourceRequirements, error) { + if c.nc.Spec.Metad.DataVolumeClaim == nil { + return nil, nil + } return c.nc.Spec.Metad.DataVolumeClaim.Resources.DeepCopy(), nil } @@ -230,11 +236,6 @@ func (c *metadComponent) GenerateContainerPorts() []corev1.ContainerPort { func (c *metadComponent) GenerateVolumeMounts() []corev1.VolumeMount { componentType := c.Type().String() mounts := []corev1.VolumeMount{ - { - Name: logVolume(componentType), - MountPath: "/usr/local/nebula/logs", - SubPath: "logs", - }, { Name: dataVolume(componentType), MountPath: "/usr/local/nebula/data", @@ -242,6 +243,14 @@ func (c *metadComponent) GenerateVolumeMounts() []corev1.VolumeMount { }, } + if c.nc.Spec.Metad.LogVolumeClaim != nil { + mounts = append(mounts, corev1.VolumeMount{ + Name: logVolume(componentType), + MountPath: "/usr/local/nebula/logs", + SubPath: "logs", + }) + } + if c.nc.Spec.Metad.License != nil { mounts = append(mounts, corev1.VolumeMount{ Name: "nebula-license", @@ -258,21 +267,24 @@ func (c *metadComponent) GenerateVolumes() []corev1.Volume { componentType := c.Type().String() volumes := []corev1.Volume{ { - Name: logVolume(componentType), + Name: dataVolume(componentType), VolumeSource: corev1.VolumeSource{ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: logVolume(componentType), + ClaimName: dataVolume(componentType), }, }, }, - { - Name: dataVolume(componentType), + } + + if c.nc.Spec.Metad.LogVolumeClaim != nil { + volumes = append(volumes, corev1.Volume{ + Name: logVolume(componentType), VolumeSource: corev1.VolumeSource{ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: dataVolume(componentType), + ClaimName: logVolume(componentType), }, }, - }, + }) } if c.nc.Spec.Metad.License != nil { @@ -297,11 +309,7 @@ func (c *metadComponent) GenerateVolumes() []corev1.Volume { func (c *metadComponent) GenerateVolumeClaim() ([]corev1.PersistentVolumeClaim, error) { componentType := c.Type().String() - logSC, logRes := c.GetLogStorageClass(), c.GetLogStorageResources() - logReq, err := parseStorageRequest(logRes.Requests) - if err != nil { - return nil, fmt.Errorf("cannot parse storage request for %s log volume, error: %v", componentType, err) - } + claims := make([]corev1.PersistentVolumeClaim, 0) dataRes, err := c.GetDataStorageResources() if err != nil { @@ -313,8 +321,25 @@ func (c *metadComponent) GenerateVolumeClaim() ([]corev1.PersistentVolumeClaim, return nil, fmt.Errorf("cannot parse storage request for %s data volume, error: %v", componentType, err) } - claims := []corev1.PersistentVolumeClaim{ - { + claims = append(claims, corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: dataVolume(componentType), + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: dataReq, + StorageClassName: dataSC, + }, + }) + + if c.nc.Spec.Metad.LogVolumeClaim != nil { + logSC, logRes := c.GetLogStorageClass(), c.GetLogStorageResources() + logReq, err := parseStorageRequest(logRes.Requests) + if err != nil { + return nil, fmt.Errorf("cannot parse storage request for %s log volume, error: %v", componentType, err) + } + + claims = append(claims, corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: logVolume(componentType), }, @@ -323,18 +348,9 @@ func (c *metadComponent) GenerateVolumeClaim() ([]corev1.PersistentVolumeClaim, Resources: logReq, StorageClassName: logSC, }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: dataVolume(componentType), - }, - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - Resources: dataReq, - StorageClassName: dataSC, - }, - }, + }) } + return claims, nil } diff --git a/apis/apps/v1alpha1/nebulacluster_storaged.go b/apis/apps/v1alpha1/nebulacluster_storaged.go index 72e2064a..9b9dfb34 100644 --- a/apis/apps/v1alpha1/nebulacluster_storaged.go +++ b/apis/apps/v1alpha1/nebulacluster_storaged.go @@ -96,6 +96,9 @@ func (c *storagedComponent) GetDataStorageClass() *string { } func (c *storagedComponent) GetLogStorageResources() *corev1.ResourceRequirements { + if c.nc.Spec.Storaged.LogVolumeClaim == nil { + return nil + } return c.nc.Spec.Storaged.LogVolumeClaim.Resources.DeepCopy() } @@ -241,13 +244,8 @@ func (c *storagedComponent) GenerateContainerPorts() []corev1.ContainerPort { func (c *storagedComponent) GenerateVolumeMounts() []corev1.VolumeMount { componentType := c.Type().String() - mounts := []corev1.VolumeMount{ - { - Name: logVolume(componentType), - MountPath: "/usr/local/nebula/logs", - SubPath: "logs", - }, - } + mounts := make([]corev1.VolumeMount, 0) + for i := range c.nc.Spec.Storaged.DataVolumeClaims { volumeName := storageDataVolume(componentType, i) mountPath := "/usr/local/nebula/data" @@ -265,21 +263,21 @@ func (c *storagedComponent) GenerateVolumeMounts() []corev1.VolumeMount { } mounts = append(mounts, mount) } + + if c.nc.Spec.Storaged.LogVolumeClaim != nil { + mounts = append(mounts, corev1.VolumeMount{ + Name: logVolume(componentType), + MountPath: "/usr/local/nebula/logs", + SubPath: "logs", + }) + } + return mounts } func (c *storagedComponent) GenerateVolumes() []corev1.Volume { componentType := c.Type().String() - volumes := []corev1.Volume{ - { - Name: logVolume(componentType), - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: logVolume(componentType), - }, - }, - }, - } + volumes := make([]corev1.Volume, 0) for i := range c.nc.Spec.Storaged.DataVolumeClaims { volumeName := storageDataVolume(componentType, i) @@ -294,24 +292,38 @@ func (c *storagedComponent) GenerateVolumes() []corev1.Volume { volumes = append(volumes, volume) } + if c.nc.Spec.Storaged.LogVolumeClaim != nil { + volumes = append(volumes, corev1.Volume{ + Name: logVolume(componentType), + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: logVolume(componentType), + }, + }, + }) + } + return volumes } func (c *storagedComponent) GenerateVolumeClaim() ([]corev1.PersistentVolumeClaim, error) { componentType := c.Type().String() - logSC, logRes := c.GetLogStorageClass(), c.GetLogStorageResources() - logReq, err := parseStorageRequest(logRes.Requests) - if err != nil { - return nil, fmt.Errorf("cannot parse storage request for %s log volume, error: %v", componentType, err) - } + claims := make([]corev1.PersistentVolumeClaim, 0) dataClaims, err := storageDataVolumeClaims(c.nc.Spec.Storaged.DataVolumeClaims, componentType) if err != nil { return nil, fmt.Errorf("cannot parse storage request for %s data volumes, error: %v", componentType, err) } + claims = append(claims, dataClaims...) - claims := []corev1.PersistentVolumeClaim{ - { + if c.nc.Spec.Storaged.LogVolumeClaim != nil { + logSC, logRes := c.GetLogStorageClass(), c.GetLogStorageResources() + logReq, err := parseStorageRequest(logRes.Requests) + if err != nil { + return nil, fmt.Errorf("cannot parse storage request for %s log volume, error: %v", componentType, err) + } + + claims = append(claims, corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: logVolume(componentType), }, @@ -320,9 +332,8 @@ func (c *storagedComponent) GenerateVolumeClaim() ([]corev1.PersistentVolumeClai Resources: logReq, StorageClassName: logSC, }, - }, + }) } - claims = append(claims, dataClaims...) return claims, nil } diff --git a/pkg/controller/component/reclaimer/meta_reconciler.go b/pkg/controller/component/reclaimer/meta_reconciler.go index e2ec518d..a8784cc1 100644 --- a/pkg/controller/component/reclaimer/meta_reconciler.go +++ b/pkg/controller/component/reclaimer/meta_reconciler.go @@ -17,10 +17,10 @@ limitations under the License. package reclaimer import ( - "errors" "fmt" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/klog/v2" "github.com/vesoft-inc/nebula-operator/apis/apps/v1alpha1" @@ -54,8 +54,17 @@ func (m *meta) Reconcile(nc *v1alpha1.NebulaCluster) error { if !label.Label(pod.Labels).IsNebulaComponent() { continue } + + var hasDataPV bool + if pod.Labels[label.ComponentLabelKey] == label.GraphdLabelVal { + hasDataPV = false + } + pvcs, err := m.resolvePVCFromPod(&pod) if err != nil { + if errors.IsNotFound(err) && !hasDataPV { + continue + } return err } for i := range pvcs { @@ -100,7 +109,9 @@ func (m *meta) resolvePVCFromPod(pod *corev1.Pod) ([]*corev1.PersistentVolumeCla pvcs = append(pvcs, pvc) } if len(pvcs) == 0 { - return nil, errors.New("pvcs not found") + err := errors.NewNotFound(corev1.Resource("pvc"), "") + err.ErrStatus.Message = fmt.Sprintf("no PVC found for pod [%s/%s]", pod.Namespace, pod.Name) + return nil, err } return pvcs, nil }