From 53b8e1e20430fd2bc25a153c750792520fdc857e Mon Sep 17 00:00:00 2001 From: LiZhenCheng9527 Date: Tue, 5 Nov 2024 10:44:52 +0800 Subject: [PATCH 1/7] Use bpf map to pass node IP and pod sub gateway. Skip kubelet probe traffic management from ebpf Signed-off-by: LiZhenCheng9527 --- bpf/include/bpf_log.h | 6 +- bpf/include/common.h | 10 +- bpf/kmesh/workload/sockops.c | 32 +++++- deploy/yaml/clusterrole.yaml | 2 +- pkg/bpf/bpf.go | 136 ++++++++++++++++++++++++ pkg/constants/constants.go | 6 ++ pkg/controller/telemetry/metric.go | 13 +-- pkg/controller/telemetry/metric_test.go | 2 +- pkg/status/status_server.go | 10 +- 9 files changed, 199 insertions(+), 18 deletions(-) diff --git a/bpf/include/bpf_log.h b/bpf/include/bpf_log.h index dca0ccd36..3e5efee4f 100644 --- a/bpf/include/bpf_log.h +++ b/bpf/include/bpf_log.h @@ -90,16 +90,16 @@ lower than 22.09, compile would report an error of bpf_snprintf dont exist */ static inline int map_lookup_log_level() { int zero = 0; - int *value = NULL; + struct kmesh_config *value = {0}; value = kmesh_map_lookup_elem(&kmesh_config_map, &zero); if (!value) return BPF_LOG_INFO; - return *value; + return value->bpf_log_level; } #define BPF_LOG(l, t, f, ...) \ do { \ - int level = map_lookup_log_level(); \ + int level = map_lookup_log_level(); \ int loglevel = BPF_MIN((int)level, ((int)BPF_LOG_DEBUG + (int)(BPF_LOGTYPE_##t))); \ if ((int)(BPF_LOG_##l) <= loglevel) { \ static const char fmt[] = "[" #t "] " #l ": " f ""; \ diff --git a/bpf/include/common.h b/bpf/include/common.h index 4b6ecf7ae..5b9df15b9 100644 --- a/bpf/include/common.h +++ b/bpf/include/common.h @@ -63,6 +63,12 @@ struct kmesh_context { bool via_waypoint; }; +struct kmesh_config { + __u32 bpf_log_level; + __u32 node_ip[4]; + __u32 pod_gateway[4]; +}; + static inline void *kmesh_map_lookup_elem(void *map, const void *key) { return bpf_map_lookup_elem(map, key); @@ -134,8 +140,8 @@ struct { struct { __uint(type, BPF_MAP_TYPE_ARRAY); __uint(max_entries, 1); - __uint(key_size, sizeof(__u32)); - __uint(value_size, sizeof(__u32)); + __type(key, int); + __type(value, struct kmesh_config); } kmesh_config_map SEC(".maps"); #if KERNEL_VERSION_HIGHER_5_13_0 diff --git a/bpf/kmesh/workload/sockops.c b/bpf/kmesh/workload/sockops.c index def454596..caf21ca89 100644 --- a/bpf/kmesh/workload/sockops.c +++ b/bpf/kmesh/workload/sockops.c @@ -46,6 +46,36 @@ static inline bool is_managed_by_kmesh(struct bpf_sock_ops *skops) return (*value == 0); } +static inline bool skip_specific_probe(struct bpf_sock_ops *skops) +{ + struct kmesh_config *data = {0}; + int key_of_kmesh_config = 0; + data = kmesh_map_lookup_elem(&kmesh_config_map, &key_of_kmesh_config); + if (!data) { + BPF_LOG(ERR, SOCKOPS, "get kmesh congfig failed"); + return false; + } + + BPF_LOG(ERR, SOCKOPS, "node ip is %u.%u", data->node_ip[0], data->node_ip[1]); + BPF_LOG(ERR, SOCKOPS, "node ip is %u.%u", data->node_ip[2], data->node_ip[3]); + BPF_LOG(ERR, SOCKOPS, "pod gateway is %u.%u", data->pod_gateway[0], data->pod_gateway[1]); + BPF_LOG(ERR, SOCKOPS, "pod gateway is %u.%u", data->pod_gateway[2], data->pod_gateway[3]); + BPF_LOG(ERR, SOCKOPS, "remote ip is %u", skops->remote_ip4); + + if (skops->family == AF_INET) { + if (data->node_ip[3] == skops->remote_ip4) { + return true; + } + if (data->pod_gateway[3] == skops->remote_ip4) { + return true; + } + } + + if (skops->family == AF_INET6) {} + + return false; +} + static inline void extract_skops_to_tuple(struct bpf_sock_ops *skops, struct bpf_sock_tuple *tuple_key) { if (skops->family == AF_INET) { @@ -233,7 +263,7 @@ int sockops_prog(struct bpf_sock_ops *skops) enable_encoding_metadata(skops); break; case BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB: - if (!is_managed_by_kmesh(skops)) + if (!is_managed_by_kmesh(skops) || skip_specific_probe(skops)) break; observe_on_connect_established(skops->sk, INBOUND); if (bpf_sock_ops_cb_flags_set(skops, BPF_SOCK_OPS_STATE_CB_FLAG) != 0) diff --git a/deploy/yaml/clusterrole.yaml b/deploy/yaml/clusterrole.yaml index f211cff00..474fce33a 100644 --- a/deploy/yaml/clusterrole.yaml +++ b/deploy/yaml/clusterrole.yaml @@ -6,7 +6,7 @@ metadata: app: kmesh rules: - apiGroups: [""] - resources: ["pods","services","namespaces"] + resources: ["pods","services","namespaces","nodes"] verbs: ["get", "update", "patch", "list", "watch"] - apiGroups: ["apps"] resources: ["daemonsets"] diff --git a/pkg/bpf/bpf.go b/pkg/bpf/bpf.go index c84f6ce99..05f021fa2 100644 --- a/pkg/bpf/bpf.go +++ b/pkg/bpf/bpf.go @@ -20,7 +20,10 @@ package bpf // #include "deserialization_to_bpf_map.h" import "C" import ( + "context" + "fmt" "hash/fnv" + "net" "os" "os/exec" "path/filepath" @@ -28,6 +31,8 @@ import ( "syscall" "github.com/cilium/ebpf" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "kmesh.net/kmesh/daemon/options" "kmesh.net/kmesh/pkg/bpf/ads" @@ -35,6 +40,7 @@ import ( "kmesh.net/kmesh/pkg/bpf/workload" "kmesh.net/kmesh/pkg/constants" "kmesh.net/kmesh/pkg/logger" + "kmesh.net/kmesh/pkg/utils" "kmesh.net/kmesh/pkg/version" ) @@ -98,6 +104,8 @@ func (l *BpfLoader) Start() error { } } + l.UpdateBpfProgOptions() + if restart.GetStartType() == restart.Restart { log.Infof("bpf load from last pinPath") } @@ -264,6 +272,134 @@ func recoverVersionMap(pinPath string) *ebpf.Map { return versionMap } +func (l *BpfLoader) UpdateBpfProgOptions() { + nodeName := os.Getenv("NODE_NAME") + if nodeName == "" { + log.Errorf("skip kubelet probe failed: %s", "node name empty") + return + } + + clientSet, err := utils.GetK8sclient() + if err != nil { + log.Errorf("get kubernetest client for getting node IP error: %v", err) + } + + node, err := clientSet.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) + if err != nil { + log.Errorf("failed to get node: %v", err) + return + } + + // pass node ip and pod gateway to skip processing of kubelet access traffic. + nodeIP := getNodeIPAddress(nodeName, node) + gateway := getNodePodSubGateway(nodeName, node) + + fmt.Printf("-------- nodeip: %v, pod gateway: %v\n", nodeIP, gateway) + + keyOfKmeshBpfConfig := uint32(0) + ValueOfKmeshBpfConfig := constants.KmeshBpfConfig{ + // Write this map only when the kmesh daemon starts, so set bpfloglevel to the default value. + BpfLogLevel: uint32(2), + NodeIP: nodeIP, + PodGateway: gateway, + } + + if l.kmeshConfig == nil { + log.Errorf("skip kubelet probe failed: %v", "kmeshConfigMap is nil") + return + } + if err := l.kmeshConfig.Update(&keyOfKmeshBpfConfig, &ValueOfKmeshBpfConfig, ebpf.UpdateAny); err != nil { + log.Errorf("update kmeshConfigMap failed: %v", err) + return + } +} + +func getNodeIPAddress(nodeName string, node *corev1.Node) [4]uint32 { + var nodeIPStr string + nodeAddresses := node.Status.Addresses + for _, address := range nodeAddresses { + if address.Type == corev1.NodeInternalIP { + nodeIPStr = address.Address + } + } + + nodeIP := net.ParseIP(nodeIPStr) + nodeIPToUint := IPToUint32(nodeIP) + + return nodeIPToUint +} + +func getNodePodSubGateway(nodeName string, node *corev1.Node) [4]uint32 { + podCIDR := node.Spec.PodCIDR + ip, _, err := net.ParseCIDR(podCIDR) + if err != nil { + log.Errorf("failed to resolve ip from podCIDR: %v", err) + return [4]uint32{0, 0, 0, 0} + } + + podGateway := IPToUint32(ip) + if isIPv6(ip) { + podGateway[0] = podGateway[0] + 1<<24 + } else { + podGateway[3] = podGateway[3] + 1<<24 + } + + return podGateway +} + +func IPToUint32(ip net.IP) [4]uint32 { + fmt.Printf("======== %v, %v ==========", ip, len(ip)) + ipToUint32 := [4]uint32{0, 0, 0, 0} + if isIPv6(ip) { + ipToUint32[0] = binaryToUint32(ip[:4]) + ipToUint32[1] = binaryToUint32(ip[4:8]) + ipToUint32[2] = binaryToUint32(ip[8:12]) + ipToUint32[3] = binaryToUint32(ip[12:16]) + } else { + if len(ip) == 16 { + // ipv4 to ipv6 + ipToUint32[3] = binaryToUint32(ip[12:16]) + } else { + ipToUint32[3] = binaryToUint32(ip) + } + } + + return ipToUint32 +} + +func isIPv6(ip net.IP) bool { + if len(ip) == 16 { + for i := 0; i < 10; i++ { + if ip[i] != 0 { + return true + } + } + + if ip[10] != 0xff { + return true + } + + if ip[11] != 0xff { + return true + } + } + return false +} + +// func calculateGateway(ipnet *net.IPNet) uint32 { +// if ipnet == nil { +// log.Errorf("failed to calculate pod sub gateway: %s", "*ipNet is empty") +// return uint32(0) +// } + +// networkInt := binaryToUint32(ipnet.IP.Mask(ipnet.Mask)) +// return networkInt + 1<<24 +// } + +func binaryToUint32(ip net.IP) uint32 { + return uint32(ip[3])<<24 + uint32(ip[2])<<16 + uint32(ip[1])<<8 + uint32(ip[0]) +} + func closeMap(m *ebpf.Map) { if m == nil { return diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index ac8568196..de718a640 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -61,3 +61,9 @@ const ( VersionPath = "/bpf_kmesh/map/" WorkloadVersionPath = "/bpf_kmesh_workload/map/" ) + +type KmeshBpfConfig struct { + BpfLogLevel uint32 + NodeIP [4]uint32 + PodGateway [4]uint32 +} diff --git a/pkg/controller/telemetry/metric.go b/pkg/controller/telemetry/metric.go index 6b53745ad..2d7ccd666 100644 --- a/pkg/controller/telemetry/metric.go +++ b/pkg/controller/telemetry/metric.go @@ -249,12 +249,7 @@ func (m *MetricController) Run(ctx context.Context, mapOfTcpInfo *ebpf.Map) { log.Errorf("get connection info failed: %v", err) continue } - workloadLabels, sourceWorkloadIsNil := m.buildWorkloadMetric(&data) - - // If the sourceworkload is empty, then neither metrics nor accesslog is printed. - if sourceWorkloadIsNil { - continue - } + workloadLabels := m.buildWorkloadMetric(&data) serviceLabels, accesslog := m.buildServiceMetric(&data) @@ -326,7 +321,7 @@ func buildV6Metric(buf *bytes.Buffer) (requestMetric, error) { return data, nil } -func (m *MetricController) buildWorkloadMetric(data *requestMetric) (workloadMetricLabels, bool) { +func (m *MetricController) buildWorkloadMetric(data *requestMetric) workloadMetricLabels { var dstAddr, srcAddr []byte for i := range data.dst { dstAddr = binary.LittleEndian.AppendUint32(dstAddr, data.dst[i]) @@ -337,7 +332,7 @@ func (m *MetricController) buildWorkloadMetric(data *requestMetric) (workloadMet srcWorkload, _ := m.getWorkloadByAddress(restoreIPv4(srcAddr)) if srcWorkload == nil { - return workloadMetricLabels{}, true + return workloadMetricLabels{} } trafficLabels := buildWorkloadMetric(dstWorkload, srcWorkload) @@ -346,7 +341,7 @@ func (m *MetricController) buildWorkloadMetric(data *requestMetric) (workloadMet trafficLabels.responseFlags = "-" trafficLabels.connectionSecurityPolicy = "mutual_tls" - return trafficLabels, false + return trafficLabels } func (m *MetricController) buildServiceMetric(data *requestMetric) (serviceMetricLabels, logInfo) { diff --git a/pkg/controller/telemetry/metric_test.go b/pkg/controller/telemetry/metric_test.go index a5ca06769..15ce18ba1 100644 --- a/pkg/controller/telemetry/metric_test.go +++ b/pkg/controller/telemetry/metric_test.go @@ -500,7 +500,7 @@ func TestBuildworkloadMetric(t *testing.T) { } m.workloadCache.AddOrUpdateWorkload(dstWorkload) m.workloadCache.AddOrUpdateWorkload(srcWorkload) - got, _ := m.buildWorkloadMetric(tt.args.data) + got := m.buildWorkloadMetric(tt.args.data) if !reflect.DeepEqual(got, tt.want) { t.Errorf("Metric.buildMetric() = %v, want %v", got, tt.want) } diff --git a/pkg/status/status_server.go b/pkg/status/status_server.go index 4b84cbbe5..3b11ff877 100644 --- a/pkg/status/status_server.go +++ b/pkg/status/status_server.go @@ -450,11 +450,19 @@ func (s *Server) setBpfLogLevel(w http.ResponseWriter, levelStr string) { return } key := uint32(0) - value := uint32(level) + // value := uint32(level) + value := constants.KmeshBpfConfig{} if s.kmeshConfigMap == nil { http.Error(w, fmt.Sprintf("update log level error: %v", "kmeshConfigMap is nil"), http.StatusBadRequest) return } + // Because kmesh config has pod gateway and node ip data. + // When change the log level, need to make sure that the pod gateway and node ip remain unchanged. + if err = s.kmeshConfigMap.Lookup(&key, &value); err != nil { + http.Error(w, fmt.Sprintf("get kmesh config error: %v", err), http.StatusBadRequest) + return + } + value.BpfLogLevel = uint32(level) if err = s.kmeshConfigMap.Update(&key, &value, ebpf.UpdateAny); err != nil { http.Error(w, fmt.Sprintf("update log level error: %v", err), http.StatusBadRequest) return From 03a1e121f4563b484cc0d61f5415e9e5a5041b0a Mon Sep 17 00:00:00 2001 From: LiZhenCheng9527 Date: Tue, 12 Nov 2024 16:56:15 +0800 Subject: [PATCH 2/7] support ipv6 Signed-off-by: LiZhenCheng9527 --- bpf/kmesh/workload/sockops.c | 21 ++++++++++++++------- pkg/bpf/bpf.go | 21 +-------------------- 2 files changed, 15 insertions(+), 27 deletions(-) diff --git a/bpf/kmesh/workload/sockops.c b/bpf/kmesh/workload/sockops.c index caf21ca89..d084d9f73 100644 --- a/bpf/kmesh/workload/sockops.c +++ b/bpf/kmesh/workload/sockops.c @@ -56,12 +56,6 @@ static inline bool skip_specific_probe(struct bpf_sock_ops *skops) return false; } - BPF_LOG(ERR, SOCKOPS, "node ip is %u.%u", data->node_ip[0], data->node_ip[1]); - BPF_LOG(ERR, SOCKOPS, "node ip is %u.%u", data->node_ip[2], data->node_ip[3]); - BPF_LOG(ERR, SOCKOPS, "pod gateway is %u.%u", data->pod_gateway[0], data->pod_gateway[1]); - BPF_LOG(ERR, SOCKOPS, "pod gateway is %u.%u", data->pod_gateway[2], data->pod_gateway[3]); - BPF_LOG(ERR, SOCKOPS, "remote ip is %u", skops->remote_ip4); - if (skops->family == AF_INET) { if (data->node_ip[3] == skops->remote_ip4) { return true; @@ -71,7 +65,20 @@ static inline bool skip_specific_probe(struct bpf_sock_ops *skops) } } - if (skops->family == AF_INET6) {} + if (skops->family == AF_INET6) { + if (data->node_ip[0] == skops->remote_ip6[0] && + data->node_ip[1] == skops->remote_ip6[1] && + data->node_ip[2] == skops->remote_ip6[2] && + data->node_ip[3] == skops->remote_ip6[3]) { + return true; + } + if (data->pod_gateway[0] == skops-> remote_ip6[0] && + data->pod_gateway[1] == skops-> remote_ip6[1] && + data->pod_gateway[2] == skops-> remote_ip6[2] && + data->pod_gateway[3] == skops-> remote_ip6[3]) { + return true; + } + } return false; } diff --git a/pkg/bpf/bpf.go b/pkg/bpf/bpf.go index 05f021fa2..11411a2f5 100644 --- a/pkg/bpf/bpf.go +++ b/pkg/bpf/bpf.go @@ -21,7 +21,6 @@ package bpf import "C" import ( "context" - "fmt" "hash/fnv" "net" "os" @@ -294,8 +293,6 @@ func (l *BpfLoader) UpdateBpfProgOptions() { nodeIP := getNodeIPAddress(nodeName, node) gateway := getNodePodSubGateway(nodeName, node) - fmt.Printf("-------- nodeip: %v, pod gateway: %v\n", nodeIP, gateway) - keyOfKmeshBpfConfig := uint32(0) ValueOfKmeshBpfConfig := constants.KmeshBpfConfig{ // Write this map only when the kmesh daemon starts, so set bpfloglevel to the default value. @@ -338,17 +335,11 @@ func getNodePodSubGateway(nodeName string, node *corev1.Node) [4]uint32 { } podGateway := IPToUint32(ip) - if isIPv6(ip) { - podGateway[0] = podGateway[0] + 1<<24 - } else { - podGateway[3] = podGateway[3] + 1<<24 - } - + podGateway[3] = podGateway[3] + 1<<24 return podGateway } func IPToUint32(ip net.IP) [4]uint32 { - fmt.Printf("======== %v, %v ==========", ip, len(ip)) ipToUint32 := [4]uint32{0, 0, 0, 0} if isIPv6(ip) { ipToUint32[0] = binaryToUint32(ip[:4]) @@ -386,16 +377,6 @@ func isIPv6(ip net.IP) bool { return false } -// func calculateGateway(ipnet *net.IPNet) uint32 { -// if ipnet == nil { -// log.Errorf("failed to calculate pod sub gateway: %s", "*ipNet is empty") -// return uint32(0) -// } - -// networkInt := binaryToUint32(ipnet.IP.Mask(ipnet.Mask)) -// return networkInt + 1<<24 -// } - func binaryToUint32(ip net.IP) uint32 { return uint32(ip[3])<<24 + uint32(ip[2])<<16 + uint32(ip[1])<<8 + uint32(ip[0]) } From e6d31f388e942bbd94922104511809847de68127 Mon Sep 17 00:00:00 2001 From: LiZhenCheng9527 Date: Tue, 12 Nov 2024 16:58:39 +0800 Subject: [PATCH 3/7] clean up Signed-off-by: LiZhenCheng9527 --- bpf/kmesh/workload/sockops.c | 2 +- pkg/bpf/bpf.go | 1 + pkg/status/status_server.go | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bpf/kmesh/workload/sockops.c b/bpf/kmesh/workload/sockops.c index d084d9f73..ba40557cb 100644 --- a/bpf/kmesh/workload/sockops.c +++ b/bpf/kmesh/workload/sockops.c @@ -52,7 +52,7 @@ static inline bool skip_specific_probe(struct bpf_sock_ops *skops) int key_of_kmesh_config = 0; data = kmesh_map_lookup_elem(&kmesh_config_map, &key_of_kmesh_config); if (!data) { - BPF_LOG(ERR, SOCKOPS, "get kmesh congfig failed"); + BPF_LOG(ERR, SOCKOPS, "get kmesh config failed"); return false; } diff --git a/pkg/bpf/bpf.go b/pkg/bpf/bpf.go index 11411a2f5..ed19ecc77 100644 --- a/pkg/bpf/bpf.go +++ b/pkg/bpf/bpf.go @@ -281,6 +281,7 @@ func (l *BpfLoader) UpdateBpfProgOptions() { clientSet, err := utils.GetK8sclient() if err != nil { log.Errorf("get kubernetest client for getting node IP error: %v", err) + return } node, err := clientSet.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) diff --git a/pkg/status/status_server.go b/pkg/status/status_server.go index 3b11ff877..ae8ac509a 100644 --- a/pkg/status/status_server.go +++ b/pkg/status/status_server.go @@ -450,7 +450,6 @@ func (s *Server) setBpfLogLevel(w http.ResponseWriter, levelStr string) { return } key := uint32(0) - // value := uint32(level) value := constants.KmeshBpfConfig{} if s.kmeshConfigMap == nil { http.Error(w, fmt.Sprintf("update log level error: %v", "kmeshConfigMap is nil"), http.StatusBadRequest) From e8e55822cb35b7035aeb10ee4f365d1d7edab641 Mon Sep 17 00:00:00 2001 From: LiZhenCheng9527 Date: Wed, 13 Nov 2024 16:53:38 +0800 Subject: [PATCH 4/7] make gen Signed-off-by: LiZhenCheng9527 --- bpf/include/bpf_log.h | 2 +- .../dualengine/kmeshcgroupsockworkload_bpfeb.go | 6 ++++++ .../dualengine/kmeshcgroupsockworkload_bpfel.go | 6 ++++++ .../kmeshcgroupsockworkloadcompat_bpfeb.go | 6 ++++++ .../kmeshcgroupsockworkloadcompat_bpfel.go | 6 ++++++ .../bpf2go/dualengine/kmeshsendmsg_bpfeb.go | 6 ++++++ .../bpf2go/dualengine/kmeshsendmsg_bpfel.go | 6 ++++++ .../dualengine/kmeshsendmsgcompat_bpfeb.go | 6 ++++++ .../dualengine/kmeshsendmsgcompat_bpfel.go | 6 ++++++ .../dualengine/kmeshsockopsworkload_bpfeb.go | 6 ++++++ .../dualengine/kmeshsockopsworkload_bpfel.go | 6 ++++++ .../kmeshsockopsworkloadcompat_bpfeb.go | 6 ++++++ .../kmeshsockopsworkloadcompat_bpfel.go | 6 ++++++ .../bpf2go/dualengine/kmeshxdpauth_bpfeb.go | 6 ++++++ .../bpf2go/dualengine/kmeshxdpauth_bpfel.go | 6 ++++++ .../dualengine/kmeshxdpauthcompat_bpfeb.go | 6 ++++++ .../dualengine/kmeshxdpauthcompat_bpfel.go | 6 ++++++ .../kernelnative/normal/kmeshcgroupsock_bpfeb.go | 6 ++++++ .../kernelnative/normal/kmeshcgroupsock_bpfel.go | 6 ++++++ .../normal/kmeshcgroupsockcompat_bpfeb.go | 6 ++++++ .../normal/kmeshcgroupsockcompat_bpfel.go | 6 ++++++ bpf/kmesh/workload/sockops.c | 16 ++++++---------- 22 files changed, 127 insertions(+), 11 deletions(-) diff --git a/bpf/include/bpf_log.h b/bpf/include/bpf_log.h index 3e5efee4f..bd0a42fb6 100644 --- a/bpf/include/bpf_log.h +++ b/bpf/include/bpf_log.h @@ -99,7 +99,7 @@ static inline int map_lookup_log_level() #define BPF_LOG(l, t, f, ...) \ do { \ - int level = map_lookup_log_level(); \ + int level = map_lookup_log_level(); \ int loglevel = BPF_MIN((int)level, ((int)BPF_LOG_DEBUG + (int)(BPF_LOGTYPE_##t))); \ if ((int)(BPF_LOG_##l) <= loglevel) { \ static const char fmt[] = "[" #t "] " #l ": " f ""; \ diff --git a/bpf/kmesh/bpf2go/dualengine/kmeshcgroupsockworkload_bpfeb.go b/bpf/kmesh/bpf2go/dualengine/kmeshcgroupsockworkload_bpfeb.go index 3c91b7003..4a1369d5b 100644 --- a/bpf/kmesh/bpf2go/dualengine/kmeshcgroupsockworkload_bpfeb.go +++ b/bpf/kmesh/bpf2go/dualengine/kmeshcgroupsockworkload_bpfeb.go @@ -24,6 +24,12 @@ type KmeshCgroupSockWorkloadBpfSockTuple struct { type KmeshCgroupSockWorkloadBuf struct{ Data [40]int8 } +type KmeshCgroupSockWorkloadKmeshConfig struct { + BpfLogLevel uint32 + NodeIp [4]uint32 + PodGateway [4]uint32 +} + type KmeshCgroupSockWorkloadLogEvent struct { Ret uint32 Msg [255]int8 diff --git a/bpf/kmesh/bpf2go/dualengine/kmeshcgroupsockworkload_bpfel.go b/bpf/kmesh/bpf2go/dualengine/kmeshcgroupsockworkload_bpfel.go index 1a9310d46..0fb6a1594 100644 --- a/bpf/kmesh/bpf2go/dualengine/kmeshcgroupsockworkload_bpfel.go +++ b/bpf/kmesh/bpf2go/dualengine/kmeshcgroupsockworkload_bpfel.go @@ -24,6 +24,12 @@ type KmeshCgroupSockWorkloadBpfSockTuple struct { type KmeshCgroupSockWorkloadBuf struct{ Data [40]int8 } +type KmeshCgroupSockWorkloadKmeshConfig struct { + BpfLogLevel uint32 + NodeIp [4]uint32 + PodGateway [4]uint32 +} + type KmeshCgroupSockWorkloadLogEvent struct { Ret uint32 Msg [255]int8 diff --git a/bpf/kmesh/bpf2go/dualengine/kmeshcgroupsockworkloadcompat_bpfeb.go b/bpf/kmesh/bpf2go/dualengine/kmeshcgroupsockworkloadcompat_bpfeb.go index 82373b92d..fd1ab2dc0 100644 --- a/bpf/kmesh/bpf2go/dualengine/kmeshcgroupsockworkloadcompat_bpfeb.go +++ b/bpf/kmesh/bpf2go/dualengine/kmeshcgroupsockworkloadcompat_bpfeb.go @@ -24,6 +24,12 @@ type KmeshCgroupSockWorkloadCompatBpfSockTuple struct { type KmeshCgroupSockWorkloadCompatBuf struct{ Data [40]int8 } +type KmeshCgroupSockWorkloadCompatKmeshConfig struct { + BpfLogLevel uint32 + NodeIp [4]uint32 + PodGateway [4]uint32 +} + type KmeshCgroupSockWorkloadCompatLogEvent struct { Ret uint32 Msg [255]int8 diff --git a/bpf/kmesh/bpf2go/dualengine/kmeshcgroupsockworkloadcompat_bpfel.go b/bpf/kmesh/bpf2go/dualengine/kmeshcgroupsockworkloadcompat_bpfel.go index c2f938178..3906668fa 100644 --- a/bpf/kmesh/bpf2go/dualengine/kmeshcgroupsockworkloadcompat_bpfel.go +++ b/bpf/kmesh/bpf2go/dualengine/kmeshcgroupsockworkloadcompat_bpfel.go @@ -24,6 +24,12 @@ type KmeshCgroupSockWorkloadCompatBpfSockTuple struct { type KmeshCgroupSockWorkloadCompatBuf struct{ Data [40]int8 } +type KmeshCgroupSockWorkloadCompatKmeshConfig struct { + BpfLogLevel uint32 + NodeIp [4]uint32 + PodGateway [4]uint32 +} + type KmeshCgroupSockWorkloadCompatLogEvent struct { Ret uint32 Msg [255]int8 diff --git a/bpf/kmesh/bpf2go/dualengine/kmeshsendmsg_bpfeb.go b/bpf/kmesh/bpf2go/dualengine/kmeshsendmsg_bpfeb.go index 121e72ce0..d08ceaf0d 100644 --- a/bpf/kmesh/bpf2go/dualengine/kmeshsendmsg_bpfeb.go +++ b/bpf/kmesh/bpf2go/dualengine/kmeshsendmsg_bpfeb.go @@ -24,6 +24,12 @@ type KmeshSendmsgBpfSockTuple struct { type KmeshSendmsgBuf struct{ Data [40]int8 } +type KmeshSendmsgKmeshConfig struct { + BpfLogLevel uint32 + NodeIp [4]uint32 + PodGateway [4]uint32 +} + type KmeshSendmsgLogEvent struct { Ret uint32 Msg [255]int8 diff --git a/bpf/kmesh/bpf2go/dualengine/kmeshsendmsg_bpfel.go b/bpf/kmesh/bpf2go/dualengine/kmeshsendmsg_bpfel.go index 538d62496..55298e174 100644 --- a/bpf/kmesh/bpf2go/dualengine/kmeshsendmsg_bpfel.go +++ b/bpf/kmesh/bpf2go/dualengine/kmeshsendmsg_bpfel.go @@ -24,6 +24,12 @@ type KmeshSendmsgBpfSockTuple struct { type KmeshSendmsgBuf struct{ Data [40]int8 } +type KmeshSendmsgKmeshConfig struct { + BpfLogLevel uint32 + NodeIp [4]uint32 + PodGateway [4]uint32 +} + type KmeshSendmsgLogEvent struct { Ret uint32 Msg [255]int8 diff --git a/bpf/kmesh/bpf2go/dualengine/kmeshsendmsgcompat_bpfeb.go b/bpf/kmesh/bpf2go/dualengine/kmeshsendmsgcompat_bpfeb.go index 44898b255..f57c72b13 100644 --- a/bpf/kmesh/bpf2go/dualengine/kmeshsendmsgcompat_bpfeb.go +++ b/bpf/kmesh/bpf2go/dualengine/kmeshsendmsgcompat_bpfeb.go @@ -24,6 +24,12 @@ type KmeshSendmsgCompatBpfSockTuple struct { type KmeshSendmsgCompatBuf struct{ Data [40]int8 } +type KmeshSendmsgCompatKmeshConfig struct { + BpfLogLevel uint32 + NodeIp [4]uint32 + PodGateway [4]uint32 +} + type KmeshSendmsgCompatLogEvent struct { Ret uint32 Msg [255]int8 diff --git a/bpf/kmesh/bpf2go/dualengine/kmeshsendmsgcompat_bpfel.go b/bpf/kmesh/bpf2go/dualengine/kmeshsendmsgcompat_bpfel.go index 7ecbfd257..28ab8db62 100644 --- a/bpf/kmesh/bpf2go/dualengine/kmeshsendmsgcompat_bpfel.go +++ b/bpf/kmesh/bpf2go/dualengine/kmeshsendmsgcompat_bpfel.go @@ -24,6 +24,12 @@ type KmeshSendmsgCompatBpfSockTuple struct { type KmeshSendmsgCompatBuf struct{ Data [40]int8 } +type KmeshSendmsgCompatKmeshConfig struct { + BpfLogLevel uint32 + NodeIp [4]uint32 + PodGateway [4]uint32 +} + type KmeshSendmsgCompatLogEvent struct { Ret uint32 Msg [255]int8 diff --git a/bpf/kmesh/bpf2go/dualengine/kmeshsockopsworkload_bpfeb.go b/bpf/kmesh/bpf2go/dualengine/kmeshsockopsworkload_bpfeb.go index 247e747a5..d7874c90b 100644 --- a/bpf/kmesh/bpf2go/dualengine/kmeshsockopsworkload_bpfeb.go +++ b/bpf/kmesh/bpf2go/dualengine/kmeshsockopsworkload_bpfeb.go @@ -24,6 +24,12 @@ type KmeshSockopsWorkloadBpfSockTuple struct { type KmeshSockopsWorkloadBuf struct{ Data [40]int8 } +type KmeshSockopsWorkloadKmeshConfig struct { + BpfLogLevel uint32 + NodeIp [4]uint32 + PodGateway [4]uint32 +} + type KmeshSockopsWorkloadLogEvent struct { Ret uint32 Msg [255]int8 diff --git a/bpf/kmesh/bpf2go/dualengine/kmeshsockopsworkload_bpfel.go b/bpf/kmesh/bpf2go/dualengine/kmeshsockopsworkload_bpfel.go index 2e36e830a..9edabce1a 100644 --- a/bpf/kmesh/bpf2go/dualengine/kmeshsockopsworkload_bpfel.go +++ b/bpf/kmesh/bpf2go/dualengine/kmeshsockopsworkload_bpfel.go @@ -24,6 +24,12 @@ type KmeshSockopsWorkloadBpfSockTuple struct { type KmeshSockopsWorkloadBuf struct{ Data [40]int8 } +type KmeshSockopsWorkloadKmeshConfig struct { + BpfLogLevel uint32 + NodeIp [4]uint32 + PodGateway [4]uint32 +} + type KmeshSockopsWorkloadLogEvent struct { Ret uint32 Msg [255]int8 diff --git a/bpf/kmesh/bpf2go/dualengine/kmeshsockopsworkloadcompat_bpfeb.go b/bpf/kmesh/bpf2go/dualengine/kmeshsockopsworkloadcompat_bpfeb.go index dfa6dc0ef..a56e81900 100644 --- a/bpf/kmesh/bpf2go/dualengine/kmeshsockopsworkloadcompat_bpfeb.go +++ b/bpf/kmesh/bpf2go/dualengine/kmeshsockopsworkloadcompat_bpfeb.go @@ -24,6 +24,12 @@ type KmeshSockopsWorkloadCompatBpfSockTuple struct { type KmeshSockopsWorkloadCompatBuf struct{ Data [40]int8 } +type KmeshSockopsWorkloadCompatKmeshConfig struct { + BpfLogLevel uint32 + NodeIp [4]uint32 + PodGateway [4]uint32 +} + type KmeshSockopsWorkloadCompatLogEvent struct { Ret uint32 Msg [255]int8 diff --git a/bpf/kmesh/bpf2go/dualengine/kmeshsockopsworkloadcompat_bpfel.go b/bpf/kmesh/bpf2go/dualengine/kmeshsockopsworkloadcompat_bpfel.go index faab12fc0..bb8e9ab91 100644 --- a/bpf/kmesh/bpf2go/dualengine/kmeshsockopsworkloadcompat_bpfel.go +++ b/bpf/kmesh/bpf2go/dualengine/kmeshsockopsworkloadcompat_bpfel.go @@ -24,6 +24,12 @@ type KmeshSockopsWorkloadCompatBpfSockTuple struct { type KmeshSockopsWorkloadCompatBuf struct{ Data [40]int8 } +type KmeshSockopsWorkloadCompatKmeshConfig struct { + BpfLogLevel uint32 + NodeIp [4]uint32 + PodGateway [4]uint32 +} + type KmeshSockopsWorkloadCompatLogEvent struct { Ret uint32 Msg [255]int8 diff --git a/bpf/kmesh/bpf2go/dualengine/kmeshxdpauth_bpfeb.go b/bpf/kmesh/bpf2go/dualengine/kmeshxdpauth_bpfeb.go index f49906e24..a04634a39 100644 --- a/bpf/kmesh/bpf2go/dualengine/kmeshxdpauth_bpfeb.go +++ b/bpf/kmesh/bpf2go/dualengine/kmeshxdpauth_bpfeb.go @@ -24,6 +24,12 @@ type KmeshXDPAuthBpfSockTuple struct { type KmeshXDPAuthBuf struct{ Data [40]int8 } +type KmeshXDPAuthKmeshConfig struct { + BpfLogLevel uint32 + NodeIp [4]uint32 + PodGateway [4]uint32 +} + type KmeshXDPAuthLogEvent struct { Ret uint32 Msg [255]int8 diff --git a/bpf/kmesh/bpf2go/dualengine/kmeshxdpauth_bpfel.go b/bpf/kmesh/bpf2go/dualengine/kmeshxdpauth_bpfel.go index 23b836527..a85815ac6 100644 --- a/bpf/kmesh/bpf2go/dualengine/kmeshxdpauth_bpfel.go +++ b/bpf/kmesh/bpf2go/dualengine/kmeshxdpauth_bpfel.go @@ -24,6 +24,12 @@ type KmeshXDPAuthBpfSockTuple struct { type KmeshXDPAuthBuf struct{ Data [40]int8 } +type KmeshXDPAuthKmeshConfig struct { + BpfLogLevel uint32 + NodeIp [4]uint32 + PodGateway [4]uint32 +} + type KmeshXDPAuthLogEvent struct { Ret uint32 Msg [255]int8 diff --git a/bpf/kmesh/bpf2go/dualengine/kmeshxdpauthcompat_bpfeb.go b/bpf/kmesh/bpf2go/dualengine/kmeshxdpauthcompat_bpfeb.go index b0bd880b8..66fc7e904 100644 --- a/bpf/kmesh/bpf2go/dualengine/kmeshxdpauthcompat_bpfeb.go +++ b/bpf/kmesh/bpf2go/dualengine/kmeshxdpauthcompat_bpfeb.go @@ -24,6 +24,12 @@ type KmeshXDPAuthCompatBpfSockTuple struct { type KmeshXDPAuthCompatBuf struct{ Data [40]int8 } +type KmeshXDPAuthCompatKmeshConfig struct { + BpfLogLevel uint32 + NodeIp [4]uint32 + PodGateway [4]uint32 +} + type KmeshXDPAuthCompatLogEvent struct { Ret uint32 Msg [255]int8 diff --git a/bpf/kmesh/bpf2go/dualengine/kmeshxdpauthcompat_bpfel.go b/bpf/kmesh/bpf2go/dualengine/kmeshxdpauthcompat_bpfel.go index 61a0f2242..d971ef8df 100644 --- a/bpf/kmesh/bpf2go/dualengine/kmeshxdpauthcompat_bpfel.go +++ b/bpf/kmesh/bpf2go/dualengine/kmeshxdpauthcompat_bpfel.go @@ -24,6 +24,12 @@ type KmeshXDPAuthCompatBpfSockTuple struct { type KmeshXDPAuthCompatBuf struct{ Data [40]int8 } +type KmeshXDPAuthCompatKmeshConfig struct { + BpfLogLevel uint32 + NodeIp [4]uint32 + PodGateway [4]uint32 +} + type KmeshXDPAuthCompatLogEvent struct { Ret uint32 Msg [255]int8 diff --git a/bpf/kmesh/bpf2go/kernelnative/normal/kmeshcgroupsock_bpfeb.go b/bpf/kmesh/bpf2go/kernelnative/normal/kmeshcgroupsock_bpfeb.go index ec6f1e440..cddc9b9f7 100644 --- a/bpf/kmesh/bpf2go/kernelnative/normal/kmeshcgroupsock_bpfeb.go +++ b/bpf/kmesh/bpf2go/kernelnative/normal/kmeshcgroupsock_bpfeb.go @@ -16,6 +16,12 @@ type KmeshCgroupSockBuf struct{ Data [40]int8 } type KmeshCgroupSockClusterSockData struct{ ClusterId uint32 } +type KmeshCgroupSockKmeshConfig struct { + BpfLogLevel uint32 + NodeIp [4]uint32 + PodGateway [4]uint32 +} + type KmeshCgroupSockLogEvent struct { Ret uint32 Msg [255]int8 diff --git a/bpf/kmesh/bpf2go/kernelnative/normal/kmeshcgroupsock_bpfel.go b/bpf/kmesh/bpf2go/kernelnative/normal/kmeshcgroupsock_bpfel.go index fea0fdd74..d2b6a262e 100644 --- a/bpf/kmesh/bpf2go/kernelnative/normal/kmeshcgroupsock_bpfel.go +++ b/bpf/kmesh/bpf2go/kernelnative/normal/kmeshcgroupsock_bpfel.go @@ -16,6 +16,12 @@ type KmeshCgroupSockBuf struct{ Data [40]int8 } type KmeshCgroupSockClusterSockData struct{ ClusterId uint32 } +type KmeshCgroupSockKmeshConfig struct { + BpfLogLevel uint32 + NodeIp [4]uint32 + PodGateway [4]uint32 +} + type KmeshCgroupSockLogEvent struct { Ret uint32 Msg [255]int8 diff --git a/bpf/kmesh/bpf2go/kernelnative/normal/kmeshcgroupsockcompat_bpfeb.go b/bpf/kmesh/bpf2go/kernelnative/normal/kmeshcgroupsockcompat_bpfeb.go index 830d651dc..d173b436d 100644 --- a/bpf/kmesh/bpf2go/kernelnative/normal/kmeshcgroupsockcompat_bpfeb.go +++ b/bpf/kmesh/bpf2go/kernelnative/normal/kmeshcgroupsockcompat_bpfeb.go @@ -16,6 +16,12 @@ type KmeshCgroupSockCompatBuf struct{ Data [40]int8 } type KmeshCgroupSockCompatClusterSockData struct{ ClusterId uint32 } +type KmeshCgroupSockCompatKmeshConfig struct { + BpfLogLevel uint32 + NodeIp [4]uint32 + PodGateway [4]uint32 +} + type KmeshCgroupSockCompatLogEvent struct { Ret uint32 Msg [255]int8 diff --git a/bpf/kmesh/bpf2go/kernelnative/normal/kmeshcgroupsockcompat_bpfel.go b/bpf/kmesh/bpf2go/kernelnative/normal/kmeshcgroupsockcompat_bpfel.go index 6286ccbdb..c44ad1f10 100644 --- a/bpf/kmesh/bpf2go/kernelnative/normal/kmeshcgroupsockcompat_bpfel.go +++ b/bpf/kmesh/bpf2go/kernelnative/normal/kmeshcgroupsockcompat_bpfel.go @@ -16,6 +16,12 @@ type KmeshCgroupSockCompatBuf struct{ Data [40]int8 } type KmeshCgroupSockCompatClusterSockData struct{ ClusterId uint32 } +type KmeshCgroupSockCompatKmeshConfig struct { + BpfLogLevel uint32 + NodeIp [4]uint32 + PodGateway [4]uint32 +} + type KmeshCgroupSockCompatLogEvent struct { Ret uint32 Msg [255]int8 diff --git a/bpf/kmesh/workload/sockops.c b/bpf/kmesh/workload/sockops.c index ba40557cb..ff8312839 100644 --- a/bpf/kmesh/workload/sockops.c +++ b/bpf/kmesh/workload/sockops.c @@ -47,7 +47,7 @@ static inline bool is_managed_by_kmesh(struct bpf_sock_ops *skops) } static inline bool skip_specific_probe(struct bpf_sock_ops *skops) -{ +{ struct kmesh_config *data = {0}; int key_of_kmesh_config = 0; data = kmesh_map_lookup_elem(&kmesh_config_map, &key_of_kmesh_config); @@ -56,7 +56,7 @@ static inline bool skip_specific_probe(struct bpf_sock_ops *skops) return false; } - if (skops->family == AF_INET) { + if (skops->family == AF_INET) { if (data->node_ip[3] == skops->remote_ip4) { return true; } @@ -66,16 +66,12 @@ static inline bool skip_specific_probe(struct bpf_sock_ops *skops) } if (skops->family == AF_INET6) { - if (data->node_ip[0] == skops->remote_ip6[0] && - data->node_ip[1] == skops->remote_ip6[1] && - data->node_ip[2] == skops->remote_ip6[2] && - data->node_ip[3] == skops->remote_ip6[3]) { + if (data->node_ip[0] == skops->remote_ip6[0] && data->node_ip[1] == skops->remote_ip6[1] + && data->node_ip[2] == skops->remote_ip6[2] && data->node_ip[3] == skops->remote_ip6[3]) { return true; } - if (data->pod_gateway[0] == skops-> remote_ip6[0] && - data->pod_gateway[1] == skops-> remote_ip6[1] && - data->pod_gateway[2] == skops-> remote_ip6[2] && - data->pod_gateway[3] == skops-> remote_ip6[3]) { + if (data->pod_gateway[0] == skops->remote_ip6[0] && data->pod_gateway[1] == skops->remote_ip6[1] + && data->pod_gateway[2] == skops->remote_ip6[2] && data->pod_gateway[3] == skops->remote_ip6[3]) { return true; } } From e99acab209cd409d01b54db4dfc61c75bf1c4967 Mon Sep 17 00:00:00 2001 From: LiZhenCheng9527 Date: Tue, 12 Nov 2024 20:20:27 +0800 Subject: [PATCH 5/7] fix ut Signed-off-by: LiZhenCheng9527 --- pkg/status/status_server.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/status/status_server.go b/pkg/status/status_server.go index ae8ac509a..04e855ce0 100644 --- a/pkg/status/status_server.go +++ b/pkg/status/status_server.go @@ -407,10 +407,11 @@ func (s *Server) readyProbe(w http.ResponseWriter, r *http.Request) { func (s *Server) getBpfLogLevel() (*LoggerInfo, error) { key := uint32(0) - value := uint32(0) + value := constants.KmeshBpfConfig{} if err := s.kmeshConfigMap.Lookup(&key, &value); err != nil { return nil, fmt.Errorf("get log level error: %v", err) } + logLevel := value.BpfLogLevel logLevelMap := map[int]string{ constants.BPF_LOG_ERR: "error", @@ -419,7 +420,7 @@ func (s *Server) getBpfLogLevel() (*LoggerInfo, error) { constants.BPF_LOG_DEBUG: "debug", } - loggerLevel, exists := logLevelMap[int(value)] + loggerLevel, exists := logLevelMap[int(logLevel)] if !exists { return nil, fmt.Errorf("unexpected invalid log level: %d", value) } From 7840b4b93cb811d887c5ac007fb790d18da355f7 Mon Sep 17 00:00:00 2001 From: LiZhenCheng9527 Date: Thu, 14 Nov 2024 15:52:05 +0800 Subject: [PATCH 6/7] Optimised handling of address calculations Signed-off-by: LiZhenCheng9527 --- pkg/bpf/bpf.go | 93 +++++++++++++------------------------ pkg/bpf/bpf_test.go | 33 +++++++++++++ pkg/constants/constants.go | 6 --- pkg/status/status_server.go | 5 +- 4 files changed, 69 insertions(+), 68 deletions(-) diff --git a/pkg/bpf/bpf.go b/pkg/bpf/bpf.go index ed19ecc77..1ff84973c 100644 --- a/pkg/bpf/bpf.go +++ b/pkg/bpf/bpf.go @@ -23,6 +23,7 @@ import ( "context" "hash/fnv" "net" + "net/netip" "os" "os/exec" "path/filepath" @@ -39,6 +40,7 @@ import ( "kmesh.net/kmesh/pkg/bpf/workload" "kmesh.net/kmesh/pkg/constants" "kmesh.net/kmesh/pkg/logger" + "kmesh.net/kmesh/pkg/nets" "kmesh.net/kmesh/pkg/utils" "kmesh.net/kmesh/pkg/version" ) @@ -57,6 +59,12 @@ type BpfLoader struct { versionMap *ebpf.Map } +type KmeshBpfConfig struct { + BpfLogLevel uint32 + NodeIP [16]byte + PodGateway [16]byte +} + func NewBpfLoader(config *options.BpfConfig) *BpfLoader { return &BpfLoader{ config: config, @@ -94,6 +102,8 @@ func (l *BpfLoader) Start() error { return err } l.kmeshConfig = l.workloadObj.GetKmeshConfigMap() + // TODO: set bpf prog option in kernel native node + l.setBpfProgOptions() } // TODO: move start mds out of bpf loader @@ -103,8 +113,6 @@ func (l *BpfLoader) Start() error { } } - l.UpdateBpfProgOptions() - if restart.GetStartType() == restart.Restart { log.Infof("bpf load from last pinPath") } @@ -271,10 +279,10 @@ func recoverVersionMap(pinPath string) *ebpf.Map { return versionMap } -func (l *BpfLoader) UpdateBpfProgOptions() { +func (l *BpfLoader) setBpfProgOptions() { nodeName := os.Getenv("NODE_NAME") if nodeName == "" { - log.Errorf("skip kubelet probe failed: %s", "node name empty") + log.Error("skip kubelet probe failed: node name empty") return } @@ -291,13 +299,13 @@ func (l *BpfLoader) UpdateBpfProgOptions() { } // pass node ip and pod gateway to skip processing of kubelet access traffic. - nodeIP := getNodeIPAddress(nodeName, node) - gateway := getNodePodSubGateway(nodeName, node) + nodeIP := getNodeIPAddress(node) + gateway := getNodePodSubGateway(node) keyOfKmeshBpfConfig := uint32(0) - ValueOfKmeshBpfConfig := constants.KmeshBpfConfig{ + ValueOfKmeshBpfConfig := KmeshBpfConfig{ // Write this map only when the kmesh daemon starts, so set bpfloglevel to the default value. - BpfLogLevel: uint32(2), + BpfLogLevel: constants.BPF_LOG_INFO, NodeIP: nodeIP, PodGateway: gateway, } @@ -312,7 +320,7 @@ func (l *BpfLoader) UpdateBpfProgOptions() { } } -func getNodeIPAddress(nodeName string, node *corev1.Node) [4]uint32 { +func getNodeIPAddress(node *corev1.Node) [16]byte { var nodeIPStr string nodeAddresses := node.Status.Addresses for _, address := range nodeAddresses { @@ -321,65 +329,30 @@ func getNodeIPAddress(nodeName string, node *corev1.Node) [4]uint32 { } } - nodeIP := net.ParseIP(nodeIPStr) - nodeIPToUint := IPToUint32(nodeIP) + nodeIP, err := netip.ParseAddr(nodeIPStr) + if err != nil { + log.Errorf("failed to parse node ip: %v", err) + return [16]byte{} + } - return nodeIPToUint + return nodeIP.As16() } -func getNodePodSubGateway(nodeName string, node *corev1.Node) [4]uint32 { +func getNodePodSubGateway(node *corev1.Node) [16]byte { podCIDR := node.Spec.PodCIDR - ip, _, err := net.ParseCIDR(podCIDR) + _, subNet, err := net.ParseCIDR(podCIDR) if err != nil { log.Errorf("failed to resolve ip from podCIDR: %v", err) - return [4]uint32{0, 0, 0, 0} - } - - podGateway := IPToUint32(ip) - podGateway[3] = podGateway[3] + 1<<24 - return podGateway -} - -func IPToUint32(ip net.IP) [4]uint32 { - ipToUint32 := [4]uint32{0, 0, 0, 0} - if isIPv6(ip) { - ipToUint32[0] = binaryToUint32(ip[:4]) - ipToUint32[1] = binaryToUint32(ip[4:8]) - ipToUint32[2] = binaryToUint32(ip[8:12]) - ipToUint32[3] = binaryToUint32(ip[12:16]) - } else { - if len(ip) == 16 { - // ipv4 to ipv6 - ipToUint32[3] = binaryToUint32(ip[12:16]) - } else { - ipToUint32[3] = binaryToUint32(ip) - } + return [16]byte{0} } - - return ipToUint32 -} - -func isIPv6(ip net.IP) bool { - if len(ip) == 16 { - for i := 0; i < 10; i++ { - if ip[i] != 0 { - return true - } - } - - if ip[10] != 0xff { - return true - } - - if ip[11] != 0xff { - return true - } + podGateway := [16]byte{0} + nets.CopyIpByteFromSlice(&podGateway, subNet.IP.To16()) + if err != nil { + log.Errorf("failed to parse pod gateway: %v", err) + return [16]byte{} } - return false -} - -func binaryToUint32(ip net.IP) uint32 { - return uint32(ip[3])<<24 + uint32(ip[2])<<16 + uint32(ip[1])<<8 + uint32(ip[0]) + podGateway[15] = podGateway[15] + 1 + return podGateway } func closeMap(m *ebpf.Map) { diff --git a/pkg/bpf/bpf_test.go b/pkg/bpf/bpf_test.go index f3f97230b..9a68e73b6 100644 --- a/pkg/bpf/bpf_test.go +++ b/pkg/bpf/bpf_test.go @@ -22,8 +22,11 @@ import ( "syscall" "testing" + "reflect" + "github.com/cilium/ebpf/rlimit" "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" "kmesh.net/kmesh/daemon/options" "kmesh.net/kmesh/pkg/bpf/restart" @@ -106,3 +109,33 @@ func runTestRestart(t *testing.T) { restart.SetExitType(restart.Normal) bpfLoader.Stop() } + +func TestGetNodePodSubGateway(t *testing.T) { + type args struct { + node *corev1.Node + } + tests := []struct { + name string + args args + want [16]byte + }{ + { + name: "test Generated nodeIP", + args: args{ + node: &corev1.Node{ + Spec: corev1.NodeSpec{ + PodCIDR: "10.244.0.0/24", + }, + }, + }, + want: [16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 10, 244, 0, 1}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getNodePodSubGateway(tt.args.node); !reflect.DeepEqual(got, tt.want) { + assert.Equal(t, tt.want, got) + } + }) + } +} diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index de718a640..ac8568196 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -61,9 +61,3 @@ const ( VersionPath = "/bpf_kmesh/map/" WorkloadVersionPath = "/bpf_kmesh_workload/map/" ) - -type KmeshBpfConfig struct { - BpfLogLevel uint32 - NodeIP [4]uint32 - PodGateway [4]uint32 -} diff --git a/pkg/status/status_server.go b/pkg/status/status_server.go index 04e855ce0..d81eb9977 100644 --- a/pkg/status/status_server.go +++ b/pkg/status/status_server.go @@ -33,6 +33,7 @@ import ( adminv2 "kmesh.net/kmesh/api/v2/admin" "kmesh.net/kmesh/api/v2/workloadapi/security" "kmesh.net/kmesh/daemon/options" + "kmesh.net/kmesh/pkg/bpf" bpfads "kmesh.net/kmesh/pkg/bpf/ads" maps_v2 "kmesh.net/kmesh/pkg/cache/v2/maps" "kmesh.net/kmesh/pkg/constants" @@ -407,7 +408,7 @@ func (s *Server) readyProbe(w http.ResponseWriter, r *http.Request) { func (s *Server) getBpfLogLevel() (*LoggerInfo, error) { key := uint32(0) - value := constants.KmeshBpfConfig{} + value := bpf.KmeshBpfConfig{} if err := s.kmeshConfigMap.Lookup(&key, &value); err != nil { return nil, fmt.Errorf("get log level error: %v", err) } @@ -451,7 +452,7 @@ func (s *Server) setBpfLogLevel(w http.ResponseWriter, levelStr string) { return } key := uint32(0) - value := constants.KmeshBpfConfig{} + value := bpf.KmeshBpfConfig{} if s.kmeshConfigMap == nil { http.Error(w, fmt.Sprintf("update log level error: %v", "kmeshConfigMap is nil"), http.StatusBadRequest) return From f14f7d2c5ad1356d878456e770aaa24642a719a1 Mon Sep 17 00:00:00 2001 From: LiZhenCheng9527 Date: Thu, 14 Nov 2024 20:47:00 +0800 Subject: [PATCH 7/7] fix by comments Signed-off-by: LiZhenCheng9527 --- pkg/bpf/bpf.go | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/pkg/bpf/bpf.go b/pkg/bpf/bpf.go index 1ff84973c..562598b99 100644 --- a/pkg/bpf/bpf.go +++ b/pkg/bpf/bpf.go @@ -39,9 +39,9 @@ import ( "kmesh.net/kmesh/pkg/bpf/restart" "kmesh.net/kmesh/pkg/bpf/workload" "kmesh.net/kmesh/pkg/constants" + "kmesh.net/kmesh/pkg/kube" "kmesh.net/kmesh/pkg/logger" "kmesh.net/kmesh/pkg/nets" - "kmesh.net/kmesh/pkg/utils" "kmesh.net/kmesh/pkg/version" ) @@ -286,7 +286,7 @@ func (l *BpfLoader) setBpfProgOptions() { return } - clientSet, err := utils.GetK8sclient() + clientSet, err := kube.CreateKubeClient("") if err != nil { log.Errorf("get kubernetest client for getting node IP error: %v", err) return @@ -310,13 +310,11 @@ func (l *BpfLoader) setBpfProgOptions() { PodGateway: gateway, } - if l.kmeshConfig == nil { - log.Errorf("skip kubelet probe failed: %v", "kmeshConfigMap is nil") - return - } - if err := l.kmeshConfig.Update(&keyOfKmeshBpfConfig, &ValueOfKmeshBpfConfig, ebpf.UpdateAny); err != nil { - log.Errorf("update kmeshConfigMap failed: %v", err) - return + if l.kmeshConfig != nil { + if err := l.kmeshConfig.Update(&keyOfKmeshBpfConfig, &ValueOfKmeshBpfConfig, ebpf.UpdateAny); err != nil { + log.Errorf("update kmeshConfig map failed: %v", err) + return + } } } @@ -347,10 +345,6 @@ func getNodePodSubGateway(node *corev1.Node) [16]byte { } podGateway := [16]byte{0} nets.CopyIpByteFromSlice(&podGateway, subNet.IP.To16()) - if err != nil { - log.Errorf("failed to parse pod gateway: %v", err) - return [16]byte{} - } podGateway[15] = podGateway[15] + 1 return podGateway }