request help: https access apisix ssl protocol error

[flyingfish7]

Issue description

https access apisix ssl protocol error,

config-default.yaml

apisix:
  node_listen: 9280              # APISIX listening port
  enable_admin: true
  enable_admin_cors: true         # Admin API support CORS response headers.
  enable_debug: false
  enable_dev_mode: false          # Sets nginx worker_processes to 1 if set to true
  enable_reuseport: true          # Enable nginx SO_REUSEPORT switch if set to true.
  enable_ipv6: true
  config_center: etcd             # etcd: use etcd to store the config value
                                  # yaml: fetch the config value from local yaml file `/your_path/conf/apisix.yaml`

  #proxy_protocol:                 # Proxy Protocol configuration
  #  listen_http_port: 9181        # The port with proxy protocol for http, it differs from node_listen and port_admin.
                                   # This port can only receive http request with proxy protocol, but node_listen & port_admin
                                   # can only receive http request. If you enable proxy protocol, you must use this port to
                                   # receive http request with proxy protocol
  #  listen_https_port: 9182       # The port with proxy protocol for https
  #  enable_tcp_pp: true           # Enable the proxy protocol for tcp proxy, it works for stream_proxy.tcp option
  #  enable_tcp_pp_to_upstream: true # Enables the proxy protocol to the upstream server

  proxy_cache:                     # Proxy Caching configuration
    cache_ttl: 10s                 # The default caching time if the upstream does not specify the cache time
    zones:                         # The parameters of a cache
    - name: disk_cache_one         # The name of the cache, administrator can be specify
                                   # which cache to use by name in the admin api
      memory_size: 50m             # The size of shared memory, it's used to store the cache index
      disk_size: 1G                # The size of disk, it's used to store the cache data
      disk_path: "/tmp/disk_cache_one" # The path to store the cache data
      cache_levels: "1:2"           # The hierarchy levels of a cache
  #  - name: disk_cache_two
  #    memory_size: 50m
  #    disk_size: 1G
  #    disk_path: "/tmp/disk_cache_two"
  #    cache_levels: "1:2"

  allow_admin:                  # http://nginx.org/en/docs/http/ngx_http_access_module.html#allow
    - 127.0.0.0/24              # If we don't set any IP list, then any IP access is allowed by default.
  #   - "::/64"
  port_admin: 9380              # use a separate port
  # https_admin: true             # enable HTTPS when use a separate port for Admin API.
                                # Admin API will use conf/apisix_admin_api.crt and conf/apisix_admin_api.key as certificate.
  admin_api_mtls:               # Depends on `port_admin` and `https_admin`.
    admin_ssl_cert: ""             # Path of your self-signed server side cert.
    admin_ssl_cert_key: ""         # Path of your self-signed server side key.
    admin_ssl_ca_cert: ""          # Path of your self-signed ca cert.The CA is used to sign all admin api callers' certificates.

  # Default token when use API to call for Admin API.
  # *NOTE*: Highly recommended to modify this value to protect APISIX's Admin API.
  # Disabling this configuration item means that the Admin API does not
  # require any authentication.
  admin_key:
    -
      name: "admin"
      key: edd1c9f034335f136f87ad84b625c8f1
      role: admin                 # admin: manage all configuration data
                                  # viewer: only can view configuration data
    #-
    #  name: "viewer"
    #  key: 4054f7cf07e344346cd3f287985e76a2
    #  role: viewer

  delete_uri_tail_slash: false   # delete the '/' at the end of the URI
  router:
    http: 'radixtree_uri'         # radixtree_uri: match route by uri(base on radixtree)
                                  # radixtree_host_uri: match route by host + uri(base on radixtree)
    ssl: 'radixtree_sni'          # radixtree_sni: match route by SNI(base on radixtree)
  # stream_proxy:                 # TCP/UDP proxy
  #   tcp:                        # TCP proxy port list
  #     - 9100
  #     - 9101
  #   udp:                        # UDP proxy port list
  #     - 9200
  #     - 9211
  # dns_resolver:                   # If not set, read from `/etc/resolv.conf`
  #  - 1.1.1.1
  #  - 8.8.8.8
  dns_resolver_valid: 30          # valid time for dns result 30 seconds
  resolver_timeout: 5             # resolver timeout
  ssl:
    enable: true
    enable_http2: true
    listen_port: 10443
    ssl_protocols: "TLSv1 TLSv1.1 TLSv1.2 TLSv1.3"
    ssl_ciphers: "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES
128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256
-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA256:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA"
    #ssl_protocols: "TLSv1.2 TLSv1.3"
    #ssl_ciphers: "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AE
S128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384"
    #key_encrypt_salt: "edd1c9f0985e76a2"    #  If not set, will save origin ssl key into etcd.
                                            #  If set this, must be a string of length 16. And it will encrypt ssl key with AES-128-CBC
                                            #  !!! So do not change it after saving your ssl, it can't decrypt the ssl keys have be saved if you change !!
                                            #
    #ssl_protocols: "TLSv1 TLSv1.1 TLSv1.2 TLSv1.3"
    #ssl_ciphers: "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AE
S128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES25
6-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA256:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA"
#  discovery: eureka               # service discovery center
nginx_config:                     # config for render the template to genarate nginx.conf
  error_log: "logs/error.log"
  error_log_level: "warn"         # warn,error
  worker_processes: auto
  worker_rlimit_nofile: 20480     # the number of files a worker process can open, should be larger than worker_connections
  worker_shutdown_timeout: 240s     # timeout for a graceful shutdown of worker processes
  event:
    worker_connections: 10620
  #envs:                            # allow to get a list of environment variables
  #  - TEST_ENV
  http:
    access_log: "logs/access.log"
    access_log_format: "$remote_addr - $remote_user [$time_local] $http_host \"$request\" $status $body_bytes_sent $request_time \"$http_referer\" \"$http_user_agent\" $upstream_addr $upstream_status $up
stream_response_time"
    keepalive_timeout: 60s         # timeout during which a keep-alive client connection will stay open on the server side.
    client_header_timeout: 60s     # timeout for reading client request header, then 408 (Request Time-out) error is returned to the client
    client_body_timeout: 60s       # timeout for reading client request body, then 408 (Request Time-out) error is returned to the client
    client_max_body_size: 0        # The maximum allowed size of the client request body.
                                   # If exceeded, the 413 (Request Entity Too Large) error is returned to the client.
                                   # Note that unlike Nginx, we don't limit the body size by default.

    send_timeout: 10s              # timeout for transmitting a response to the client.then the connection is closed
    underscores_in_headers: "on"   # default enables the use of underscores in client request header fields
    real_ip_header: "X-Real-IP"    # http://nginx.org/en/docs/http/ngx_http_realip_module.html#real_ip_header
    real_ip_from:                  # http://nginx.org/en/docs/http/ngx_http_realip_module.html#set_real_ip_from
      - 127.0.0.1
      - 'unix:'
    #lua_shared_dicts:              # add custom shared cache to nginx.conf
    #  ipc_shared_dict: 100m        # custom shared cache, format: `cache-key: cache-size`

etcd:
  host:                           # it's possible to define multiple etcd hosts addresses of the same etcd cluster.
    - "http://127.0.0.1:3379"     # multiple etcd address
  prefix: "/apisix"               # apisix configurations prefix
  timeout: 30                     # 30 seconds
  # user: root                     # root username for etcd
  # password: 5tHkHhYkjr6cQY        # root password for etcd
#eureka:
#  host:                           # it's possible to define multiple eureka hosts addresses of the same eureka cluster.
#    - "http://127.0.0.1:8761"
#  prefix: "/eureka/"
#  fetch_interval: 30              # default 30s
#  weight: 100                     # default weight for node
#  timeout:
#    connect: 2000                 # default 2000ms
#    send: 2000                    # default 2000ms
#    read: 5000                    # default 5000ms

plugins:                          # plugin list
  - example-plugin
  - limit-req
  - limit-count
  - limit-conn
  - key-auth
  - basic-auth
  - prometheus
  - node-status
  - jwt-auth
  - zipkin
  - ip-restriction
  - referer-restriction
  - grpc-transcode
  - serverless-pre-function
  - serverless-post-function
  - openid-connect
  - proxy-rewrite
  - redirect
  - response-rewrite
  - fault-injection
  - udp-logger
  - wolf-rbac
  - tcp-logger
  - kafka-logger
  - cors
  - consumer-restriction
  - syslog
  - batch-requests
  - http-logger
  - skywalking
  - echo
  - authz-keycloak
  - uri-blocker
  - request-validation
  - proxy-cache
  - proxy-mirror
  - request-id
  - hmac-auth

stream_plugins:
  - mqtt-proxy

plugin_attr:
  log-rotate:
    interval: 3600    # rotate interval (unit: second)
    max_kept: 168     # max number of log files will be kept

error.log

2020/11/02 11:35:48 [error] 23546#23546: 1507 [lua] radixtree_sni.lua:213: match_and_set(): failed to find any SSL certificate by SNI: test-api.yestae.com, context: ssl_certificate_by_lua, client: 172.17.186.164, server: 0.0.0.0:10443
2020/11/02 11:35:48 [crit] 23546#23546: *1505 SSL_do_handshake() failed (SSL: error:1417A179:SSL routines:tls_post_process_client_hello:cert cb error) while SSL handshaking, client: 172.17.186.164, server: 0.0.0.0:10443

Environment

• apisix version (cmd: apisix version): 2.0
• OS: CentOS Linux release 7.6.1810 (Core)

[spacewander]

2020/11/02 11:35:48 [error] 23546#23546: 1507 [lua] radixtree_sni.lua:213: match_and_set(): failed to find any SSL certificate by SNI: test-api.yestae.com, context: ssl_certificate_by_lua, client: 172.17.186.164, server: 0.0.0.0:10443

The domain "test-api.yestae.com" sent by client's SNI is unknown to your APISIX.

[flyingfish7]

2020/11/02 11:35:48 [error] 23546#23546: 1507 [lua] radixtree_sni.lua:213: match_and_set(): failed to find any SSL certificate by SNI: test-api.yestae.com, context: ssl_certificate_by_lua, client: 172.17.186.164, server: 0.0.0.0:10443

The domain "test-api.yestae.com" sent by client's SNI is unknown to your APISIX.

In my etcd ssl configuration sni has "*.yestae.com" configuration ，

[root@test-open-api apisix-dashboard]# etcdctl_new  --endpoints=http://127.0.0.1:3379  get /apisix/ssl  --prefix
/apisix/ssl/
init_dir
/apisix/ssl/326743630154826275
{"id":"326743630154826275","create_time":1604283980,"update_time":1604286383,"cert":"...","key":"...","snis":["*.yestae.com","autodiscover.yestae.com","mail.yestae.com","owa.yestae.com","www.yestae.com","yestae.com"],"status":0,"validity_start":1542879426,"validity_end":1613457880}
[root@test-open-api apisix-dashboard]# 

[spacewander]

Interesting. Can't reproduce in my side. I set the SSL configuration via apisix's admin API:

# etcdctl get /apisix/ssl  --prefix
/apisix/ssl/
init_dir
/apisix/ssl/1
{"cert":"...","id":"1","status":1,"key":"...","snis":["*.yestae.com","autodiscover.yestae.com","mail.yestae.com","owa.yestae.com","www.yestae.com","yestae.com"]}

But I got: radixtree_sni.lua:208: match_and_set(): sni: test-api.yestae.com.

Maybe you need to add more log in radixtree_sni.lua.

[flyingfish7]

apisix/ssl/router/radixtree_sni.lua

210     local sni_rev = sni:reverse()
211     core.log.debug("sni_rev: ",sni_rev)
212     core.log.debug("api_ctx.matched_sni: ",api_ctx.matched_sni)
213     core.log.debug("api_ctx.matched_ssl: ",api_ctx.matched_ssl)
214     local ok = radixtree_router:dispatch(sni_rev, nil, api_ctx)
215     if not ok then

208

2020/11/02 16:34:42 [debug] 29687#29687: *130 [lua] init.lua:164: http_ssl_phase(): api_ctx:nil
2020/11/02 16:34:42 [debug] 29687#29687: *130 [lua] init.lua:168: http_ssl_phase(): api_ctx:{}
2020/11/02 16:34:42 [info] 29687#29687: 130 [lua] radixtree_sni.lua:146: create_router(): route items: {}, context: ssl_certificate_by_lua, client: 172.17.186.164, server: 0.0.0.0:10443
2020/11/02 16:34:42 [debug] 29687#29687: *130 [lua] radixtree_sni.lua:208: match_and_set(): sni: test-api.yestae.com
2020/11/02 16:34:42 [debug] 29687#29687: *130 [lua] radixtree_sni.lua:211: match_and_set(): sni_rev: moc.eatsey.ipa-tset
2020/11/02 16:34:42 [debug] 29687#29687: *130 [lua] radixtree_sni.lua:212: match_and_set(): api_ctx.matched_sni: nil
2020/11/02 16:34:42 [debug] 29687#29687: *130 [lua] radixtree_sni.lua:213: match_and_set(): api_ctx.matched_ssl: nil
2020/11/02 16:34:42 [error] 29687#29687: 130 [lua] radixtree_sni.lua:216: match_and_set(): failed to find any SSL certificate by SNI: test-api.yestae.com, context: ssl_certificate_by_lua, client: 172.17.186.164, server: 0.0.0.0:10443
2020/11/02 16:34:42 [crit] 29687#29687: *127 SSL_do_handshake() failed (SSL: error:1417A179:SSL routines:tls_post_process_client_hello:cert cb error) while SSL handshaking, client: 172.17.186.164, server: 0.0.0.0:10443

[spacewander]

Look strange to me.
Could you provide a minimal example to reproduce the issue?
Will change SSL configure make this problem disappear?

[flyingfish7]

Look strange to me.
Could you provide a minimal example to reproduce the issue?
Will change SSL configure make this problem disappear?

Thank you very much, I have found the cause of the problem: Because my apisix-dashboard uses 2.0-rc2, my ssl configuration is configured from the dashboard, there may be incompatible configuration formats, I solved it through apisix-admin api Fix the problem, thanks again for your help