Skip to content

Commit

Permalink
Merge pull request #3647 from chaunceyjiang/same-name-with-cluster
Browse files Browse the repository at this point in the history 
metrics-adapter: when different clusters have the same pod name, the annotations of different metrics have the same value
  • Loading branch information
@karmada-bot
karmada-bot committed Jun 9, 2023
2 parents e588a43 + 23f94a4 commit 76acb6d
Show file tree
Hide file tree
Showing 2 changed files with 47 additions and 72 deletions.
7 changes: 7 additions & 0 deletions pkg/apis/autoscaling/v1alpha1/well_known_constants.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package v1alpha1

const (
// QuerySourceAnnotationKey is the annotation used in karmada-metrics-adapter to
// record the query source cluster
QuerySourceAnnotationKey = "resource.karmada.io/query-from-cluster"
)
112 changes: 40 additions & 72 deletions pkg/metricsadapter/provider/resourcemetrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@ import (
"k8s.io/metrics/pkg/apis/metrics"
metricsv1beta1 "k8s.io/metrics/pkg/apis/metrics/v1beta1"

autoscalingv1alpha1 "github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1"
clusterlister "github.com/karmada-io/karmada/pkg/generated/listers/cluster/v1alpha1"
"github.com/karmada-io/karmada/pkg/util"
"github.com/karmada-io/karmada/pkg/util/fedinformer/genericmanager"
"github.com/karmada-io/karmada/pkg/util/helper"
)
Expand All @@ -28,9 +30,6 @@ const (
// namespaceSpecifiedAnnotation is the annotation used in karmada-metrics-adapter,
// to record the namespace specified by the user
namespaceSpecifiedAnnotation = "internal.karmada.io/namespace"
// querySourceAnnotationKey is the annotation used in karmada-metrics-adapter to
// record the query source cluster
querySourceAnnotationKey = "resource.karmada.io/query-from-cluster"
)

var (
Expand Down Expand Up @@ -150,10 +149,14 @@ func (r *ResourceMetricsProvider) queryPodMetricsByName(name, namespace string)
_, err := sci.Lister(PodsGVR).ByNamespace(namespace).Get(name)
return err
}
metricsQueryFunc := func(sci genericmanager.SingleClusterInformerManager, _ string) (interface{}, error) {
metrics, err := sci.GetClient().Resource(podMetricsGVR).
metricsQueryFunc := func(sci genericmanager.SingleClusterInformerManager, clusterName string) (interface{}, error) {
metricsValue, err := sci.GetClient().Resource(podMetricsGVR).
Namespace(namespace).Get(context.Background(), name, metav1.GetOptions{})
return metrics, err
if err != nil {
return nil, err
}
util.MergeAnnotation(metricsValue, autoscalingv1alpha1.QuerySourceAnnotationKey, clusterName)
return metricsValue, err
}

metricsQuery, err := r.getMetricsParallel(resourceQueryFunc, metricsQueryFunc)
Expand Down Expand Up @@ -192,12 +195,18 @@ func (r *ResourceMetricsProvider) queryPodMetricsBySelector(selector, namespace
}
return nil
}
metricsQueryFunc := func(sci genericmanager.SingleClusterInformerManager, _ string) (interface{}, error) {
metrics, err := sci.GetClient().Resource(podMetricsGVR).
metricsQueryFunc := func(sci genericmanager.SingleClusterInformerManager, clusterName string) (interface{}, error) {
metricsList, err := sci.GetClient().Resource(podMetricsGVR).
Namespace(namespace).List(context.Background(), metav1.ListOptions{
LabelSelector: selector,
})
return metrics, err
if err != nil {
return nil, err
}
for i := range metricsList.Items {
util.MergeAnnotation(&metricsList.Items[i], autoscalingv1alpha1.QuerySourceAnnotationKey, clusterName)
}
return metricsList, err
}

metricsQuery, err := r.getMetricsParallel(resourceQueryFunc, metricsQueryFunc)
Expand All @@ -224,9 +233,13 @@ func (r *ResourceMetricsProvider) queryNodeMetricsByName(name string) ([]metrics
_, err := sci.Lister(NodesGVR).Get(name)
return err
}
metricsQueryFunc := func(sci genericmanager.SingleClusterInformerManager, _ string) (interface{}, error) {
metrics, err := sci.GetClient().Resource(nodeMetricsGVR).Get(context.Background(), name, metav1.GetOptions{})
return metrics, err
metricsQueryFunc := func(sci genericmanager.SingleClusterInformerManager, clusterName string) (interface{}, error) {
metricsValue, err := sci.GetClient().Resource(nodeMetricsGVR).Get(context.Background(), name, metav1.GetOptions{})
if err != nil {
return nil, err
}
util.MergeAnnotation(metricsValue, autoscalingv1alpha1.QuerySourceAnnotationKey, clusterName)
return metricsValue, err
}

metricsQuery, err := r.getMetricsParallel(resourceQueryFunc, metricsQueryFunc)
Expand Down Expand Up @@ -265,11 +278,17 @@ func (r *ResourceMetricsProvider) queryNodeMetricsBySelector(selector string) ([
}
return nil
}
metricsQueryFunc := func(sci genericmanager.SingleClusterInformerManager, _ string) (interface{}, error) {
metrics, err := sci.GetClient().Resource(nodeMetricsGVR).List(context.Background(), metav1.ListOptions{
metricsQueryFunc := func(sci genericmanager.SingleClusterInformerManager, clusterName string) (interface{}, error) {
metricsList, err := sci.GetClient().Resource(nodeMetricsGVR).List(context.Background(), metav1.ListOptions{
LabelSelector: selector,
})
return metrics, err
if err != nil {
return nil, err
}
for i := range metricsList.Items {
util.MergeAnnotation(&metricsList.Items[i], autoscalingv1alpha1.QuerySourceAnnotationKey, clusterName)
}
return metricsList, err
}

metricsQuery, err := r.getMetricsParallel(resourceQueryFunc, metricsQueryFunc)
Expand All @@ -292,18 +311,10 @@ func (r *ResourceMetricsProvider) queryNodeMetricsBySelector(selector string) ([

// GetPodMetrics queries metrics by the internal constructed pod
func (r *ResourceMetricsProvider) GetPodMetrics(pods ...*metav1.PartialObjectMetadata) ([]metrics.PodMetrics, error) {
var ret []metrics.PodMetrics
var queryData []metrics.PodMetrics
if len(pods) == 0 {
return ret, nil
return queryData, nil
}

podsKeyMap := map[string]string{}
for _, pod := range pods {
podKey := generateNamespaceNameKey(pod.Namespace, pod.Name)
podsKeyMap[podKey] = pod.Annotations[querySourceAnnotationKey]
}

var queryData []metrics.PodMetrics
var err error
// In the previous step, we construct the annotations, so it couldn't be nil
if _, ok := pods[0].Annotations[labelSelectorAnnotationInternal]; ok {
Expand All @@ -313,39 +324,18 @@ func (r *ResourceMetricsProvider) GetPodMetrics(pods ...*metav1.PartialObjectMet
} else {
queryData, err = r.queryPodMetricsByName(pods[0].Name, pods[0].Namespace)
}

if err != nil {
return nil, err
}

for _, i := range queryData {
podKey := generateNamespaceNameKey(i.Namespace, i.Name)
if queryCluster, ok := podsKeyMap[podKey]; ok {
if i.Annotations == nil {
i.Annotations = make(map[string]string)
}
i.Annotations[querySourceAnnotationKey] = queryCluster
ret = append(ret, i)
}
}

return ret, nil
return queryData, nil
}

// GetNodeMetrics queries metrics by the internal constructed node
func (r *ResourceMetricsProvider) GetNodeMetrics(nodes ...*corev1.Node) ([]metrics.NodeMetrics, error) {
var ret []metrics.NodeMetrics
var queryData []metrics.NodeMetrics
if len(nodes) == 0 {
return ret, nil
return queryData, nil
}

nodesKeyMap := map[string]string{}
for _, node := range nodes {
nodeKey := generateNamespaceNameKey("", node.Name)
nodesKeyMap[nodeKey] = node.Annotations[querySourceAnnotationKey]
}

var queryData []metrics.NodeMetrics
var err error
// In the previous step, we construct the annotations, so it couldn't be nil
if _, ok := nodes[0].Annotations[labelSelectorAnnotationInternal]; ok {
Expand All @@ -354,23 +344,10 @@ func (r *ResourceMetricsProvider) GetNodeMetrics(nodes ...*corev1.Node) ([]metri
} else {
queryData, err = r.queryNodeMetricsByName(nodes[0].Name)
}

if err != nil {
return nil, err
}

for _, i := range queryData {
nodeKey := generateNamespaceNameKey(i.Namespace, i.Name)
if cluster, ok := nodesKeyMap[nodeKey]; ok {
if i.Annotations == nil {
i.Annotations = make(map[string]string)
}
i.Annotations[querySourceAnnotationKey] = cluster
ret = append(ret, i)
}
}

return ret, nil
return queryData, nil
}

// PodLister is an internal lister for pods
Expand Down Expand Up @@ -416,7 +393,6 @@ func (p *PodLister) List(selector labels.Selector) (ret []runtime.Object, err er
return nil, err
}
podPartial := p.convertToPodPartialData(podTyped, selector.String(), true)
podPartial.Annotations[querySourceAnnotationKey] = cluster.Name
ret = append(ret, podPartial)
}
}
Expand Down Expand Up @@ -481,7 +457,6 @@ func (p *PodLister) Get(name string) (runtime.Object, error) {
return nil, err
}
podPartial = p.convertToPodPartialData(podTyped, "", false)
podPartial.Annotations[querySourceAnnotationKey] = cluster.Name
}

if podPartial != nil {
Expand Down Expand Up @@ -550,7 +525,6 @@ func (n *NodeLister) List(selector labels.Selector) (ret []*corev1.Node, err err

//If user sets this annotation, we need to reset it.
nodeTyped.Annotations[labelSelectorAnnotationInternal] = selector.String()
nodeTyped.Annotations[querySourceAnnotationKey] = cluster.Name
ret = append(ret, nodeTyped)
}
}
Expand Down Expand Up @@ -599,7 +573,6 @@ func (n *NodeLister) Get(name string) (*corev1.Node, error) {

//If user sets this annotation, we need to remove it to avoid parsing wrong next.
delete(nodeTyped.Annotations, labelSelectorAnnotationInternal)
nodeTyped.Annotations[querySourceAnnotationKey] = cluster.Name
}

if nodeTyped != nil {
Expand Down Expand Up @@ -662,8 +635,3 @@ func metricsConvertV1beta1NodeToInternalNode(objs ...unstructured.Unstructured)

return nodeMetricsInternal, nil
}

// generateNamespaceNameKey generates namespace/name key
func generateNamespaceNameKey(namespace, name string) string {
return namespace + "/" + name
}

0 comments on commit 76acb6d

Please sign in to comment.