接入skywalking不生效

[hoocoo]

1.helm安装 设置--set global.onlyPushRouteCluster=false --set higress-core.skywalking.enabled=true --set higress-core.skywalking.service.address=skywalking.higress-system.svc.cluster.local --set higress-core.skywalking.service.port=11809

2.全局配置为

  higress: |-
    tracing:
      enable: true
      sampling: 100
      timeout: 500
      skywalking:
       service: skywalking.higress-system.svc.cluster.local
       port: 11809

3.由于skywalking是非k8s服务，所以新建了一个service 配置如下

apiVersion: v1
kind: Service
metadata:
  name: skywalking
  namespace: higress-system
spec:
  type: ExternalName
  externalName: skywalking.dev.cn
  ports:
    - protocol: TCP
      port: 11809

4.在geteway pod里面查看listener 和 cluster部分配置如下

  "listeners": [
      {
       "address": {
        "socket_address": {
         "address": "0.0.0.0",
         "port_value": 15090
        }
       },
       "filter_chains": [
        {
         "filters": [
          {
           "name": "envoy.filters.network.http_connection_manager",
           "typed_config": {
            "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
            "stat_prefix": "stats",
            "route_config": {
             "virtual_hosts": [
              {
               "name": "backend",
               "domains": [
                "*"
               ],
               "routes": [
                {
                 "match": {
                  "prefix": "/stats/prometheus"
                 },
                 "route": {
                  "cluster": "prometheus_stats"
                 }
                }
               ]
              }
             ]
            },
            "http_filters": [
             {
              "name": "envoy.filters.http.router",
              "typed_config": {
               "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"
              }
             }
            ]
           }
          }
         ]
        }
       ]
      },
      {
       "address": {
        "socket_address": {
         "address": "0.0.0.0",
         "port_value": 15021
        }
       },
       "filter_chains": [
        {
         "filters": [
          {
           "name": "envoy.filters.network.http_connection_manager",
           "typed_config": {
            "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
            "stat_prefix": "agent",
            "route_config": {
             "virtual_hosts": [
              {
               "name": "backend",
               "domains": [
                "*"
               ],
               "routes": [
                {
                 "match": {
                  "prefix": "/healthz/ready"
                 },
                 "route": {
                  "cluster": "agent"
                 }
                }
               ]
              }
             ]
            },
            "http_filters": [
             {
              "name": "envoy.filters.http.router",
              "typed_config": {
               "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"
              }
             }
            ]
           }
          }
         ]
        }
       ]
      }
     ],
     "clusters": [
      {
       "name": "prometheus_stats",
       "type": "STATIC",
       "connect_timeout": "0.250s",
       "load_assignment": {
        "cluster_name": "prometheus_stats",
        "endpoints": [
         {
          "lb_endpoints": [
           {
            "endpoint": {
             "address": {
              "socket_address": {
               "address": "127.0.0.1",
               "port_value": 15000
              }
             }
            }
           }
          ]
         }
        ]
       }
      },
      {
       "name": "agent",
       "type": "STATIC",
       "connect_timeout": "0.250s",
       "load_assignment": {
        "cluster_name": "agent",
        "endpoints": [
         {
          "lb_endpoints": [
           {
            "endpoint": {
             "address": {
              "socket_address": {
               "address": "127.0.0.1",
               "port_value": 15020
              }
             }
            }
           }
          ]
         }
        ]
       }
      },
      {
       "name": "sds-grpc",
       "type": "STATIC",
       "connect_timeout": "1s",
       "load_assignment": {
        "cluster_name": "sds-grpc",
        "endpoints": [
         {
          "lb_endpoints": [
           {
            "endpoint": {
             "address": {
              "pipe": {
               "path": "./etc/istio/proxy/SDS"
              }
             }
            }
           }
          ]
         }
        ]
       },
       "typed_extension_protocol_options": {
        "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": {
         "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions",
         "explicit_http_config": {
          "http2_protocol_options": {}
         }
        }
       }
      },
      {
       "name": "xds-grpc",
       "type": "STATIC",
       "connect_timeout": "1s",
       "max_requests_per_connection": 1,
       "circuit_breakers": {
        "thresholds": [
         {
          "max_connections": 100000,
          "max_pending_requests": 100000,
          "max_requests": 100000
         },
         {
          "priority": "HIGH",
          "max_connections": 100000,
          "max_pending_requests": 100000,
          "max_requests": 100000
         }
        ]
       },
       "upstream_connection_options": {
        "tcp_keepalive": {
         "keepalive_time": 300
        }
       },
       "load_assignment": {
        "cluster_name": "xds-grpc",
        "endpoints": [
         {
          "lb_endpoints": [
           {
            "endpoint": {
             "address": {
              "pipe": {
               "path": "./etc/istio/proxy/XDS"
              }
             }
            }
           }
          ]
         }
        ]
       },
       "typed_extension_protocol_options": {
        "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": {
         "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions",
         "explicit_http_config": {
          "http2_protocol_options": {}
         }
        }
       }
      },
      {
       "name": "zipkin",
       "type": "STRICT_DNS",
       "connect_timeout": "1s",
       "dns_refresh_rate": "30s",
       "dns_lookup_family": "V4_ONLY",
       "load_assignment": {
        "cluster_name": "zipkin",
        "endpoints": [
         {
          "lb_endpoints": [
           {
            "endpoint": {
             "address": {
              "socket_address": {
               "address": "zipkin.higress-system",
               "port_value": 9411
              }
             }
            }
           }
          ]
         }
        ]
       },
       "respect_dns_ttl": true
      },
      {
       "name": "service_skywalking",
       "type": "LOGICAL_DNS",
       "connect_timeout": "5s",
       "http2_protocol_options": {},
       "dns_lookup_family": "V4_ONLY",
       "load_assignment": {
        "cluster_name": "service_skywalking",
        "endpoints": [
         {
          "lb_endpoints": [
           {
            "endpoint": {
             "address": {
              "socket_address": {
               "address": "skywalking.higress-system.svc.cluster.local",
               "port_value": 11809
              }
             }
            }
           }
          ]
         }
        ]
       }
      }
     ]
    },
    "dynamic_resources": {
     "lds_config": {
      "ads": {},
      "initial_fetch_timeout": "0s",
      "resource_api_version": "V3"
     },
     "cds_config": {
      "ads": {},
      "initial_fetch_timeout": "0s",
      "resource_api_version": "V3"
     },
     "ads_config": {
      "api_type": "GRPC",
      "grpc_services": [
       {
        "envoy_grpc": {
         "cluster_name": "xds-grpc"
        }
       }
      ],
      "set_node_on_first_message_only": true,
      "transport_api_version": "V3"
     }
    },
    "stats_sinks": [
     {
      "name": "envoy.metrics_service",
      "typed_config": {
       "@type": "type.googleapis.com/envoy.config.metrics.v3.MetricsServiceConfig",
       "grpc_service": {
        "envoy_grpc": {
         "cluster_name": "service_skywalking"
        }
       },
       "transport_api_version": "V3"
      }
     }
    ],
    "tracing": {
     "http": {
      "name": "envoy.tracers.zipkin",
      "typed_config": {
       "@type": "type.googleapis.com/envoy.config.trace.v3.ZipkinConfig",
       "collector_cluster": "zipkin",
       "collector_endpoint": "/api/v2/spans",
       "trace_id_128bit": true,
       "shared_span_context": false,
       "collector_endpoint_version": "HTTP_JSON"
      }
     }
    },
    "admin": {

[johnlanni]

@hoocoo 要看 dynamic_listener 和 dynamic_active_cluster 部分

[hoocoo]

 "dynamic_resources": {
     "lds_config": {
      "ads": {},
      "initial_fetch_timeout": "0s",
      "resource_api_version": "V3"
     },
     "cds_config": {
      "ads": {},
      "initial_fetch_timeout": "0s",
      "resource_api_version": "V3"
     },
     "ads_config": {
      "api_type": "GRPC",
      "grpc_services": [
       {
        "envoy_grpc": {
         "cluster_name": "xds-grpc"
        }
       }
      ],
      "set_node_on_first_message_only": true,
      "transport_api_version": "V3"
     }
    },
    "stats_sinks": [
     {
      "name": "envoy.metrics_service",
      "typed_config": {
       "@type": "type.googleapis.com/envoy.config.metrics.v3.MetricsServiceConfig",
       "grpc_service": {
        "envoy_grpc": {
         "cluster_name": "service_skywalking"
        }
       },
       "transport_api_version": "V3"
      }
     }
    ],
    "tracing": {
     "http": {
      "name": "envoy.tracers.zipkin",
      "typed_config": {
       "@type": "type.googleapis.com/envoy.config.trace.v3.ZipkinConfig",
       "collector_cluster": "zipkin",
       "collector_endpoint": "/api/v2/spans",
       "trace_id_128bit": true,
       "shared_span_context": false,
       "collector_endpoint_version": "HTTP_JSON"
      }
     }
    },



 "dynamic_active_clusters": [
    {
     "version_info": "2024-01-24T02:53:14Z/19",
     "cluster": {
      "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
      "name": "outbound|11809||skywalking.higress-system.svc.cluster.local",
      "type": "STRICT_DNS",
      "connect_timeout": "10s",
      "circuit_breakers": {       
	  "thresholds": [
        {
         "max_connections": 4294967295,
         "max_pending_requests": 4294967295,
         "max_requests": 4294967295,
         "max_retries": 4294967295,
         "track_remaining": true
        }
       ]
      },
      "dns_refresh_rate": "200s",
      "dns_lookup_family": "V4_ONLY",
      "metadata": {
       "filter_metadata": {
        "istio": {
         "services": [
          {
           "namespace": "higress-system",
           "name": "skywalking",
           "host": "skywalking.higress-system.svc.cluster.local"
          }
         ],
         "default_original_port": 11809
        }
       }
      },
      "common_lb_config": {
       "locality_weighted_lb_config": {}
      },
      "load_assignment": {
       "cluster_name": "outbound|11809||skywalking.higress-system.svc.cluster.local",
       "endpoints": [
        {
         "locality": {},
         "lb_endpoints": [
          {
           "endpoint": {
            "address": {
             "socket_address": {
              "address": "skywalking.dev.cn",
              "port_value": 11809
             }
            }
           },
           "metadata": {
            "filter_metadata": {
             "istio": {
              "workload": ";;;;"
             }
            }
           },           
		   "load_balancing_weight": 1
          }
         ],
         "load_balancing_weight": 1
        }
       ]
      },
      "respect_dns_ttl": true
     },
     "last_updated": "2024-01-24T02:53:44.224Z"
    }

[hoocoo]

好像dynamic_active_cluster 中的name跟listener中的"cluster_name": "service_skywalking"对不上？

[johnlanni]

  {
   "name": "service_skywalking",
   "type": "LOGICAL_DNS",
   "connect_timeout": "5s",
   "http2_protocol_options": {},
   "dns_lookup_family": "V4_ONLY",
   "load_assignment": {
    "cluster_name": "service_skywalking",
    "endpoints": [
     {
      "lb_endpoints": [
       {
        "endpoint": {
         "address": {
          "socket_address": {
           "address": "skywalking.higress-system.svc.cluster.local",
           "port_value": 11809
          }
         }
        }
       }
      ]
     }

能对上的，我记错了，这个配置在static cluster里。看上去配置没啥问题，有什么报错信息吗

[hoocoo]

没报错，下面这段是启动打印的日志

2024-01-24T12:00:31.310477Z     info    Envoy command: [-c etc/istio/proxy/envoy-rev0.json --restart-epoch 0 --drain-time-s 45 --drain-strategy immediate --parent-shutdown-time-s 60 --local-address-ip-version v4 --file-flush-interval-msec 1000 --disable-hot-restart --log-format [Envoy (Epoch 0)] [%Y-%m-%d %T.%e][%t][%l][%n] %v -l warning --component-log-level misc:error --config-yaml {
  "stats_sinks": [
    {
      "name": "envoy.metrics_service",
      "typed_config": {
        "@type": "type.googleapis.com/envoy.config.metrics.v3.MetricsServiceConfig",
        "transport_api_version": "V3",
        "grpc_service": {
          "envoy_grpc": {
            "cluster_name": "service_skywalking"
          }
        }
      }
    }
  ],
  "static_resources": {
    "clusters": [
      {
        "name": "service_skywalking",
        "type": "LOGICAL_DNS",
        "connect_timeout": "5s",
        "http2_protocol_options": {
        },
        "dns_lookup_family": "V4_ONLY",
        "lb_policy": "ROUND_ROBIN",
        "load_assignment": {
          "cluster_name": "service_skywalking",
          "endpoints": [
            {
              "lb_endpoints": [
                {
                  "endpoint": {
                    "address": {
                      "socket_address": {
                        "address": "skywalking.higress-system.svc.cluster.local",
                        "port_value": "11809"
                      }
                    }
                  }
                }
              ]
            }
          ]
        }
      }
    ]
  }
} --concurrency 2]

[qixiaobo]

SameProblem