Mesh-Gateway k8s: Error initializing configuration

[nikfot]

I am installing mesh gateways in kubernetes with the latest consul chart.
All components are up and running correctly, however the mesh-gateways pods have trouble initializing configuration. The consul sidecar is up and running the error message is:

[1][critical][main] [source/server/server.cc:113] error initializing configuration '/tmp/envoy-168db8514e3f05e3-bootstrap.json': The v2 xDS major version is deprecated and disabled by default. Support for v2 will be removed from Envoy at the start of Q1 2021. You may make use of v2 in Q4 2020 by following the advice in https://www.envoyproxy.io/docs/envoy/latest/faq/api/transition. (Unknown field in: {
  "admin": {
    "access_log_path": "/dev/null",
    "address": {
      "socket_address": {
        "address": "127.0.0.1",
        "port_value": 19000
      }
    }
  },
  "node": {
    "cluster": "mesh-gateway",
    "id": "mesh-gateway",
    "metadata": {
      "namespace": "default",
      "envoy_version": "1.16.4"
    }
  },
  "static_resources": {
    "clusters": [
      {
        "name": "local_agent",
        "connect_timeout": "1s",
        "type": "STATIC",
        "tls_context": {
          "common_tls_context": {
            "validation_context": {
              "trusted_ca": {
                "inline_string": "-----BEGIN CERTIFICATE-----\n_{{CERT}}_\n-----END CERTIFICATE-----\n"
              }
            }
          }
        },
        "http2_protocol_options": {},
        "hosts": [
          {
            "socket_address": {
              "address": "10.xxx.xxx.xxx",
              "port_value": 8502
            }
          }
        ]
      }
    ]
  },
  "stats_config": {
    "stats_tags": [
      {
        "regex": "^cluster\\.((?:([^.]+)~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
        "tag_name": "consul.destination.custom_hash"
      },
      {
        "regex": "^cluster\\.((?:[^.]+~)?(?:([^.]+)\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
        "tag_name": "consul.destination.service_subset"
      },
      {
        "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?([^.]+)\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
        "tag_name": "consul.destination.service"
      },
      {
        "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
        "tag_name": "consul.destination.namespace"
      },
      {
        "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.([^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)",
        "tag_name": "consul.destination.datacenter"
      },
      {
        "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.([^.]+)\\.[^.]+\\.consul\\.)",
        "tag_name": "consul.destination.routing_type"
      },
      {
        "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.([^.]+)\\.consul\\.)",
        "tag_name": "consul.destination.trust_domain"
      },
      {
        "regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)",
        "tag_name": "consul.destination.target"
      },
      {
        "regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+)\\.consul\\.)",
        "tag_name": "consul.destination.full_target"
      },
      {
        "regex": "^(?:tcp|http)\\.upstream\\.(([^.]+)(?:\\.[^.]+)?\\.[^.]+\\.)",
        "tag_name": "consul.upstream.service"
      },
      {
        "regex": "^(?:tcp|http)\\.upstream\\.([^.]+(?:\\.[^.]+)?\\.([^.]+)\\.)",
        "tag_name": "consul.upstream.datacenter"
      },
      {
        "regex": "^(?:tcp|http)\\.upstream\\.([^.]+(?:\\.([^.]+))?\\.[^.]+\\.)",
        "tag_name": "consul.upstream.namespace"
      },
      {
        "regex": "^cluster\\.((?:([^.]+)~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
        "tag_name": "consul.custom_hash"
      },
      {
        "regex": "^cluster\\.((?:[^.]+~)?(?:([^.]+)\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
        "tag_name": "consul.service_subset"
      },
      {
        "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?([^.]+)\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
        "tag_name": "consul.service"
      },
      {
        "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
        "tag_name": "consul.namespace"
      },
      {
        "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.([^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)",
        "tag_name": "consul.datacenter"
      },
      {
        "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.([^.]+)\\.[^.]+\\.consul\\.)",
        "tag_name": "consul.routing_type"
      },
      {
        "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.([^.]+)\\.consul\\.)",
        "tag_name": "consul.trust_domain"
      },
      {
        "regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)",
        "tag_name": "consul.target"
      },
      {
        "regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+)\\.consul\\.)",
        "tag_name": "consul.full_target"
      },
      {
        "tag_name": "local_cluster",
        "fixed_value": "mesh-gateway"
      },
      {
        "tag_name": "consul.source.service",
        "fixed_value": "mesh-gateway"
      },
      {
        "tag_name": "consul.source.namespace",
        "fixed_value": "default"
      },
      {
        "tag_name": "consul.source.datacenter",
        "fixed_value": "perf"
      }
    ],
    "use_all_default_tags": true
  },
  "dynamic_resources": {
    "lds_config": {
      "ads": {}
    },
    "cds_config": {
      "ads": {}
    },
    "ads_config": {
      "api_type": "GRPC",
      "grpc_services": {
        "initial_metadata": [
          {
            "key": "x-consul-token",
            "value": ""
          }
        ],
        "envoy_grpc": {
          "cluster_name": "local_agent"
        }
      }
    }
  },
  "layered_runtime": {
    "layers": [
      {
        "name": "static_layer",
        "static_layer": {
          "envoy.deprecated_features:envoy.api.v2.Cluster.tls_context": true,
          "envoy.deprecated_features:envoy.config.trace.v2.ZipkinConfig.HTTP_JSON_V1": true,
          "envoy.deprecated_features:envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager.Tracing.operation_name": true
        }
      }
    ]
  }
}
)

I also see tha envoy version is 1_16_4 although i use the image 1.18.4
Any ideas?

[thisisnotashwin]

Hey @nikfot
Can you share the values file you are using for this deployment?

[nikfot]

Sure:

global:
  datacenter: PERF
  imageEnvoy: "envoyproxy/envoy-alpine:v1.18.3"
  federation:
     enabled: true
     createFederationSecret: true
  image: "consul:1.9.6"
  gossipEncryption:
    secretName: "consul-gossip-encryption-key"
    secretKey: "key"
  acls:
    manageSystemACLs: false
    createReplicationToken: false
  tls:
    enabled: true
    verify: true
    httpsOnly: true
    enableAutoEncrypt: true
    serverAdditionalDNSSANs: ["consul.test"]
    serverAdditionalIPSANs: ["10.xxx.xxx.xxx"]
ui:
  enabled: true
  service:
    type: 'ClusterIP'
  ingress:
    hosts:
      - consul.test
    annotations: |
     "some-annotations":"test"

meshGateway:
    enabled: true
    service:
      type: ClusterIP

connectInject:
  enabled: true

controller:
  enabled: true

client:
  enabled: true
  grpc: true
  updateStrategy: |
      type: OnDelete

dns:
  enabled: true

syncCatalog:
  enabled: true
  resources:
    requests:
      memory: "256Mi"
      cpu: "50m"
    limits:
      memory: "256Mi"
      cpu: "50m"

server:
  updatePartition: 3
  replicas: 3
  bootstrapExpect: 3
  disruptionBudget:
    enabled: true
    maxUnavailable: 0
  storageClass: somestorageclass
  resources:
    requests:
      memory: "256Mi"
      cpu: "100m"
    limits:
      memory: "256Mi"
      cpu: "100m"

service:
  name: perf-consul
  type: ClusterIP
  externalPort: 443
  internalPort: 8501

[nikfot]

++ @thisisnotashwin I tried changing the envoy image to imageEnvoy: "envoyproxy/envoy-alpine:v1.16.0" and it works fine.
So it is apparently a bug in envoy, or chart configuration.

[david-yu]

Hi @nikfot We have linked an envoy compatibility matrix from our upgrade guide on K8s. You would need 1.16.x since your chart is still using Consul 1.9.x. Hope that helps!

[nikfot]

@david-yu that's great! Thanks!

[david-yu]

Great, it looks like this is no longer blocking you so I'll go ahead and closed the issue!