help request: grpc-web plugin doesn't work

[nomikz]

Description

I started an apisix locally.

./docker-compose.yaml

version: "3"

services:
  apisix-dashboard:
    image: apache/apisix-dashboard:3.0.1-alpine
    restart: always
    volumes:
      - ./dashboard_conf/conf.yaml:/usr/local/apisix-dashboard/conf/conf.yaml
    ports:
      - "9000:9000"
    networks:
      apisix:

  apisix:
    image: apache/apisix:${APISIX_IMAGE_TAG:-3.3.0-debian}
    restart: always
    volumes:
      - ./apisix_conf/config.yaml:/usr/local/apisix/conf/config.yaml:ro
    depends_on:
      - etcd
    ##network_mode: host
    ports:
      - "9180:9180/tcp"
      - "9080:9080/tcp" # apisix listening port
      - "9092:9092/tcp" # controle plane
    networks:
      apisix:

  etcd:
    image: bitnami/etcd:3.4.15
    restart: always
    volumes:
      - etcd_data:/bitnami/etcd
    environment:
      ETCD_ENABLE_V2: "true"
      ALLOW_NONE_AUTHENTICATION: "yes"
      ETCD_ADVERTISE_CLIENT_URLS: "http://etcd:2379"
      ETCD_LISTEN_CLIENT_URLS: "http://0.0.0.0:2379"
    ports:
      - "2379:2379/tcp"
    networks:
      apisix:

networks:
  apisix:
    driver: bridge

volumes:
  etcd_data:
    driver: local

./apisix_conf/config.yaml:

apisix:
  node_listen: 9080
  enable_ipv6: false

  enable_control: true
  control:
    ip: "0.0.0.0"
    port: 9092

deployment:
  admin:
    allow_admin:
      - 0.0.0.0/0

    admin_key:
      - name: "admin"
        key: edd1c9f034335f136f87ad84b625c8f1
        role: admin

      - name: "viewer"
        key: 4054f7cf07e344346cd3f287985e76a2
        role: viewer

  etcd:
    host:
      - "http://etcd:2379"
    prefix: "/apisix"
    timeout: 30                

./apisix_conf/config.yaml

conf:
  listen:
    host: 0.0.0.0     # `manager api` listening ip or host name
    port: 9000          # `manager api` listening port
  allow_list:           # If we don't set any IP list, then any IP access is allowed by default.
    - 0.0.0.0/0
  etcd:
    endpoints:          # supports defining multiple etcd host addresses for an etcd cluster
      - "http://etcd:2379"
                          # yamllint disable rule:comments-indentation
                          # etcd basic auth info
    # username: "root"    # ignore etcd username if not enable etcd auth
    # password: "123456"  # ignore etcd password if not enable etcd auth
    mtls:
      key_file: ""          # Path of your self-signed client side key
      cert_file: ""         # Path of your self-signed client side cert
      ca_file: ""           # Path of your self-signed ca cert, the CA is used to sign callers' certificates
    # prefix: /apisix     # apisix config's prefix in etcd, /apisix by default
  log:
    error_log:
      level: warn       # supports levels, lower to higher: debug, info, warn, error, panic, fatal
      file_path:
        logs/error.log  # supports relative path, absolute path, standard output
                        # such as: logs/error.log, /tmp/logs/error.log, /dev/stdout, /dev/stderr
    access_log:
      file_path:
        logs/access.log  # supports relative path, absolute path, standard output
                         # such as: logs/access.log, /tmp/logs/access.log, /dev/stdout, /dev/stderr
                         # log example: 2020-12-09T16:38:09.039+0800	INFO	filter/logging.go:46	/apisix/admin/routes/r1	{"status": 401, "host": "127.0.0.1:9000", "query": "asdfsafd=adf&a=a", "requestId": "3d50ecb8-758c-46d1-af5b-cd9d1c820156", "latency": 0, "remoteIP": "127.0.0.1", "method": "PUT", "errs": []}
  security:
      # access_control_allow_origin: "http://httpbin.org"
      # access_control_allow_credentials: true          # support using custom cors configration
      # access_control_allow_headers: "Authorization"
      # access_control-allow_methods: "*"
      # x_frame_options: "deny"
      content_security_policy: "default-src 'self'; script-src 'self' 'unsafe-eval' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; frame-src *"  # You can set frame-src to provide content for your grafana panel.

authentication:
  secret:
    secret              
                        
                        
  expire_time: 3600     
  users:                
    - username: admin   
      password: admin
    - username: user
      password: user

plugins:                
  - api-breaker
  - authz-keycloak
  - basic-auth
  - batch-requests
  - consumer-restriction
  - cors
  # - dubbo-proxy
  - echo
  # - error-log-logger
  # - example-plugin
  - fault-injection
  - grpc-transcode
  - grpc-web
  - hmac-auth
  - http-logger
  - ip-restriction
  - jwt-auth
  - kafka-logger
  - key-auth
  - limit-conn
  - limit-count
  - limit-req
  # - log-rotate
  # - node-status
  - openid-connect
  - prometheus
  - proxy-cache
  - proxy-mirror
  - proxy-rewrite
  - redirect
  - referer-restriction
  - request-id
  - request-validation
  - response-rewrite
  - serverless-post-function
  - serverless-pre-function
  # - skywalking
  - sls-logger
  - syslog
  - tcp-logger
  - udp-logger
  - uri-blocker
  - wolf-rbac
  - zipkin
  - server-info
  - traffic-split

command to start: docker compose up -d

f79687170fc1   apache/apisix:3.3.0-debian             "/docker-entrypoint.…"   About an hour ago   Up About an hour   0.0.0.0:9080->9080/tcp, 0.0.0.0:9092->9092/tcp, 0.0.0.0:9180->9180/tcp, 9443/tcp   apisix-apisix-1
48bdf30abcbf   apache/apisix-dashboard:3.0.1-alpine   "/usr/local/apisix-d…"   About an hour ago   Up About an hour   0.0.0.0:9000->9000/tcp                                                             apisix-apisix-dashboard-1
448e806a7243   bitnami/etcd:3.4.15                    "/opt/bitnami/script…"   About an hour ago   Up About an hour   0.0.0.0:2379->2379/tcp, 2380/tcp                                                   apisix-etcd-1

I created a route with an upstream:

curl http://127.0.0.1:9180/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
{
    "uri":"/*",
    "plugins":{
        "grpc-web":{}
    },
    "upstream":{
        "scheme":"grpc",
        "type":"roundrobin",
        "nodes":{
            "127.0.0.1:50051":1
        }
    }
}'

I have grpc server written in go which works fine if I test it directly with Postman grpc (localhost:50051). But when I try to reach it through apisix proxy postman client fails and returns an error (localhost:9080):

Received RST_STREAM with code 2 triggered by internal client error: Protocol error

Logs from apisix container:

docker container logs apisix-apisix-1
/usr/local/openresty//luajit/bin/luajit ./apisix/cli/apisix.lua init

WARNING: using fixed Admin API token has security risk.
Please modify "admin_key" in conf/config.yaml .


/usr/local/openresty//luajit/bin/luajit ./apisix/cli/apisix.lua init_etcd
request etcd endpoint 'http://etcd:2379/version' error, connection refused
all etcd nodes are unavailable
Warning! Request etcd endpoint 'http://etcd:2379/version' error, connection refused, retry time=1
Warning! Request etcd endpoint 'http://etcd:2379/version' error, connection refused, retry time=2
/usr/local/openresty//luajit/bin/luajit ./apisix/cli/apisix.lua init

WARNING: using fixed Admin API token has security risk.
Please modify "admin_key" in conf/config.yaml .


/usr/local/openresty//luajit/bin/luajit ./apisix/cli/apisix.lua init_etcd
request etcd endpoint 'http://etcd:2379/version' error, connection refused
all etcd nodes are unavailable
Warning! Request etcd endpoint 'http://etcd:2379/version' error, connection refused, retry time=1
Warning! Request etcd endpoint 'http://etcd:2379/version' error, connection refused, retry time=2
/usr/local/openresty//luajit/bin/luajit ./apisix/cli/apisix.lua init

WARNING: using fixed Admin API token has security risk.
Please modify "admin_key" in conf/config.yaml .


/usr/local/openresty//luajit/bin/luajit ./apisix/cli/apisix.lua init_etcd
2023/06/27 08:36:11 [warn] 52#52: *2 [lua] plugin.lua:202: load(): new plugins: {"grpc-transcode":true,"skywalking-logger":true,"limit-count":true,"key-auth":true,"wolf-rbac":true,"forward-auth":true,"elasticsearch-logger":true,"udp-logger":true,"aws-lambda":true,"example-plugin":true,"csrf":true,"request-id":true,"prometheus":true,"tencent-cloud-cls":true,"ext-plugin-pre-req":true,"echo":true,"server-info":true,"kafka-logger":true,"grpc-web":true,"basic-auth":true,"ua-restriction":true,"authz-casdoor":true,"real-ip":true,"datadog":true,"proxy-mirror":true,"kafka-proxy":true,"splunk-hec-logging":true,"client-control":true,"google-cloud-logging":true,"inspect":true,"cas-auth":true,"syslog":true,"serverless-pre-function":true,"file-logger":true,"zipkin":true,"ext-plugin-post-resp":true,"gzip":true,"ext-plugin-post-req":true,"fault-injection":true,"serverless-post-function":true,"openfunction":true,"azure-functions":true,"openwhisk":true,"ip-restriction":true,"authz-keycloak":true,"consumer-restriction":true,"ldap-auth":true,"jwt-auth":true,"request-validation":true,"opa":true,"loggly":true,"proxy-cache":true,"limit-conn":true,"traffic-split":true,"tcp-logger":true,"cors":true,"body-transformer":true,"sls-logger":true,"uri-blocker":true,"mocking":true,"rocketmq-logger":true,"hmac-auth":true,"http-logger":true,"clickhouse-logger":true,"workflow":true,"ai":true,"openid-connect":true,"proxy-rewrite":true,"api-breaker":true,"authz-casbin":true,"referer-restriction":true,"degraphql":true,"redirect":true,"response-rewrite":true,"public-api":true,"limit-req":true,"proxy-control":true}, context: init_worker_by_lua*
2023/06/27 08:36:11 [warn] 50#50: *5 [lua] plugin.lua:202: load(): new plugins: {"grpc-transcode":true,"skywalking-logger":true,"limit-count":true,"key-auth":true,"wolf-rbac":true,"forward-auth":true,"elasticsearch-logger":true,"udp-logger":true,"aws-lambda":true,"example-plugin":true,"csrf":true,"request-id":true,"prometheus":true,"tencent-cloud-cls":true,"ext-plugin-pre-req":true,"echo":true,"server-info":true,"kafka-logger":true,"grpc-web":true,"basic-auth":true,"ua-restriction":true,"authz-casdoor":true,"real-ip":true,"datadog":true,"proxy-mirror":true,"kafka-proxy":true,"splunk-hec-logging":true,"client-control":true,"google-cloud-logging":true,"inspect":true,"cas-auth":true,"syslog":true,"serverless-pre-function":true,"file-logger":true,"zipkin":true,"ext-plugin-post-resp":true,"gzip":true,"ext-plugin-post-req":true,"fault-injection":true,"serverless-post-function":true,"openfunction":true,"azure-functions":true,"openwhisk":true,"ip-restriction":true,"authz-keycloak":true,"consumer-restriction":true,"ldap-auth":true,"jwt-auth":true,"request-validation":true,"opa":true,"loggly":true,"proxy-cache":true,"limit-conn":true,"traffic-split":true,"tcp-logger":true,"cors":true,"body-transformer":true,"sls-logger":true,"uri-blocker":true,"mocking":true,"rocketmq-logger":true,"hmac-auth":true,"http-logger":true,"clickhouse-logger":true,"workflow":true,"ai":true,"openid-connect":true,"proxy-rewrite":true,"api-breaker":true,"authz-casbin":true,"referer-restriction":true,"degraphql":true,"redirect":true,"response-rewrite":true,"public-api":true,"limit-req":true,"proxy-control":true}, context: init_worker_by_lua*
2023/06/27 08:36:11 [warn] 51#51: *1 [lua] plugin.lua:202: load(): new plugins: {"grpc-transcode":true,"skywalking-logger":true,"limit-count":true,"key-auth":true,"wolf-rbac":true,"forward-auth":true,"elasticsearch-logger":true,"udp-logger":true,"aws-lambda":true,"example-plugin":true,"csrf":true,"request-id":true,"prometheus":true,"tencent-cloud-cls":true,"ext-plugin-pre-req":true,"echo":true,"server-info":true,"kafka-logger":true,"grpc-web":true,"basic-auth":true,"ua-restriction":true,"authz-casdoor":true,"real-ip":true,"datadog":true,"proxy-mirror":true,"kafka-proxy":true,"splunk-hec-logging":true,"client-control":true,"google-cloud-logging":true,"inspect":true,"cas-auth":true,"syslog":true,"serverless-pre-function":true,"file-logger":true,"zipkin":true,"ext-plugin-post-resp":true,"gzip":true,"ext-plugin-post-req":true,"fault-injection":true,"serverless-post-function":true,"openfunction":true,"azure-functions":true,"openwhisk":true,"ip-restriction":true,"authz-keycloak":true,"consumer-restriction":true,"ldap-auth":true,"jwt-auth":true,"request-validation":true,"opa":true,"loggly":true,"proxy-cache":true,"limit-conn":true,"traffic-split":true,"tcp-logger":true,"cors":true,"body-transformer":true,"sls-logger":true,"uri-blocker":true,"mocking":true,"rocketmq-logger":true,"hmac-auth":true,"http-logger":true,"clickhouse-logger":true,"workflow":true,"ai":true,"openid-connect":true,"proxy-rewrite":true,"api-breaker":true,"authz-casbin":true,"referer-restriction":true,"degraphql":true,"redirect":true,"response-rewrite":true,"public-api":true,"limit-req":true,"proxy-control":true}, context: init_worker_by_lua*
2023/06/27 08:36:11 [warn] 49#49: *3 [lua] plugin.lua:202: load(): new plugins: {"grpc-transcode":true,"skywalking-logger":true,"limit-count":true,"key-auth":true,"wolf-rbac":true,"forward-auth":true,"elasticsearch-logger":true,"udp-logger":true,"aws-lambda":true,"example-plugin":true,"csrf":true,"request-id":true,"prometheus":true,"tencent-cloud-cls":true,"ext-plugin-pre-req":true,"echo":true,"server-info":true,"kafka-logger":true,"grpc-web":true,"basic-auth":true,"ua-restriction":true,"authz-casdoor":true,"real-ip":true,"datadog":true,"proxy-mirror":true,"kafka-proxy":true,"splunk-hec-logging":true,"client-control":true,"google-cloud-logging":true,"inspect":true,"cas-auth":true,"syslog":true,"serverless-pre-function":true,"file-logger":true,"zipkin":true,"ext-plugin-post-resp":true,"gzip":true,"ext-plugin-post-req":true,"fault-injection":true,"serverless-post-function":true,"openfunction":true,"azure-functions":true,"openwhisk":true,"ip-restriction":true,"authz-keycloak":true,"consumer-restriction":true,"ldap-auth":true,"jwt-auth":true,"request-validation":true,"opa":true,"loggly":true,"proxy-cache":true,"limit-conn":true,"traffic-split":true,"tcp-logger":true,"cors":true,"body-transformer":true,"sls-logger":true,"uri-blocker":true,"mocking":true,"rocketmq-logger":true,"hmac-auth":true,"http-logger":true,"clickhouse-logger":true,"workflow":true,"ai":true,"openid-connect":true,"proxy-rewrite":true,"api-breaker":true,"authz-casbin":true,"referer-restriction":true,"degraphql":true,"redirect":true,"response-rewrite":true,"public-api":true,"limit-req":true,"proxy-control":true}, context: init_worker_by_lua*
2023/06/27 08:36:11 [warn] 53#53: *4 [lua] plugin.lua:202: load(): new plugins: {"grpc-transcode":true,"skywalking-logger":true,"limit-count":true,"key-auth":true,"wolf-rbac":true,"forward-auth":true,"elasticsearch-logger":true,"udp-logger":true,"aws-lambda":true,"example-plugin":true,"csrf":true,"request-id":true,"prometheus":true,"tencent-cloud-cls":true,"ext-plugin-pre-req":true,"echo":true,"server-info":true,"kafka-logger":true,"grpc-web":true,"basic-auth":true,"ua-restriction":true,"authz-casdoor":true,"real-ip":true,"datadog":true,"proxy-mirror":true,"kafka-proxy":true,"splunk-hec-logging":true,"client-control":true,"google-cloud-logging":true,"inspect":true,"cas-auth":true,"syslog":true,"serverless-pre-function":true,"file-logger":true,"zipkin":true,"ext-plugin-post-resp":true,"gzip":true,"ext-plugin-post-req":true,"fault-injection":true,"serverless-post-function":true,"openfunction":true,"azure-functions":true,"openwhisk":true,"ip-restriction":true,"authz-keycloak":true,"consumer-restriction":true,"ldap-auth":true,"jwt-auth":true,"request-validation":true,"opa":true,"loggly":true,"proxy-cache":true,"limit-conn":true,"traffic-split":true,"tcp-logger":true,"cors":true,"body-transformer":true,"sls-logger":true,"uri-blocker":true,"mocking":true,"rocketmq-logger":true,"hmac-auth":true,"http-logger":true,"clickhouse-logger":true,"workflow":true,"ai":true,"openid-connect":true,"proxy-rewrite":true,"api-breaker":true,"authz-casbin":true,"referer-restriction":true,"degraphql":true,"redirect":true,"response-rewrite":true,"public-api":true,"limit-req":true,"proxy-control":true}, context: init_worker_by_lua*
2023/06/27 08:36:11 [warn] 56#56: *6 [lua] plugin.lua:202: load(): new plugins: {"grpc-transcode":true,"skywalking-logger":true,"limit-count":true,"key-auth":true,"wolf-rbac":true,"forward-auth":true,"elasticsearch-logger":true,"udp-logger":true,"aws-lambda":true,"example-plugin":true,"csrf":true,"request-id":true,"prometheus":true,"tencent-cloud-cls":true,"ext-plugin-pre-req":true,"echo":true,"server-info":true,"kafka-logger":true,"grpc-web":true,"basic-auth":true,"ua-restriction":true,"authz-casdoor":true,"real-ip":true,"datadog":true,"proxy-mirror":true,"kafka-proxy":true,"splunk-hec-logging":true,"client-control":true,"google-cloud-logging":true,"inspect":true,"cas-auth":true,"syslog":true,"serverless-pre-function":true,"file-logger":true,"zipkin":true,"ext-plugin-post-resp":true,"gzip":true,"ext-plugin-post-req":true,"fault-injection":true,"serverless-post-function":true,"openfunction":true,"azure-functions":true,"openwhisk":true,"ip-restriction":true,"authz-keycloak":true,"consumer-restriction":true,"ldap-auth":true,"jwt-auth":true,"request-validation":true,"opa":true,"loggly":true,"proxy-cache":true,"limit-conn":true,"traffic-split":true,"tcp-logger":true,"cors":true,"body-transformer":true,"sls-logger":true,"uri-blocker":true,"mocking":true,"rocketmq-logger":true,"hmac-auth":true,"http-logger":true,"clickhouse-logger":true,"workflow":true,"ai":true,"openid-connect":true,"proxy-rewrite":true,"api-breaker":true,"authz-casbin":true,"referer-restriction":true,"degraphql":true,"redirect":true,"response-rewrite":true,"public-api":true,"limit-req":true,"proxy-control":true}, context: init_worker_by_lua*
2023/06/27 08:36:11 [warn] 52#52: *2 [lua] plugin.lua:252: load_stream(): new plugins: {"syslog":true,"mqtt-proxy":true,"ip-restriction":true,"limit-conn":true}, context: init_worker_by_lua*
2023/06/27 08:36:11 [warn] 51#51: *1 [lua] plugin.lua:252: load_stream(): new plugins: {"syslog":true,"mqtt-proxy":true,"ip-restriction":true,"limit-conn":true}, context: init_worker_by_lua*
2023/06/27 08:36:11 [warn] 53#53: *4 [lua] plugin.lua:252: load_stream(): new plugins: {"syslog":true,"mqtt-proxy":true,"ip-restriction":true,"limit-conn":true}, context: init_worker_by_lua*
2023/06/27 08:36:11 [warn] 50#50: *5 [lua] plugin.lua:252: load_stream(): new plugins: {"syslog":true,"mqtt-proxy":true,"ip-restriction":true,"limit-conn":true}, context: init_worker_by_lua*
2023/06/27 08:36:11 [warn] 49#49: *3 [lua] plugin.lua:252: load_stream(): new plugins: {"syslog":true,"mqtt-proxy":true,"ip-restriction":true,"limit-conn":true}, context: init_worker_by_lua*
2023/06/27 08:36:11 [warn] 56#56: *6 [lua] plugin.lua:252: load_stream(): new plugins: {"syslog":true,"mqtt-proxy":true,"ip-restriction":true,"limit-conn":true}, context: init_worker_by_lua*
172.28.0.1 - - [27/Jun/2023:09:23:40 +0000] 127.0.0.1:9180 "PUT /apisix/admin/routes/1 HTTP/1.1" 201 284 0.039 "-" "curl/7.87.0" - - - "http://127.0.0.1:9180"
172.28.0.1 - - [27/Jun/2023:09:24:11 +0000] - "PRI * HTTP/2.0" 400 229 0.002 "-" "-" - - - "://"
172.28.0.1 - - [27/Jun/2023:09:24:11 +0000] - "PRI * HTTP/2.0" 400 229 0.002 "-" "-" - - - "://"
172.28.0.1 - - [27/Jun/2023:09:24:20 +0000] - "PRI * HTTP/2.0" 400 229 0.001 "-" "-" - - - "://"
172.28.0.1 - - [27/Jun/2023:09:24:20 +0000] - "PRI * HTTP/2.0" 400 229 0.001 "-" "-" - - - "://"
172.28.0.1 - - [27/Jun/2023:09:39:17 +0000] - "PRI * HTTP/2.0" 400 229 0.002 "-" "-" - - - "://"
172.28.0.1 - - [27/Jun/2023:09:39:17 +0000] - "PRI * HTTP/2.0" 400 229 0.002 "-" "-" - - - "://"

Environment

Docker image:
apache/apisix:3.3.0-debian, apache/apisix-dashboard:3.0.1-alpine, bitnami/etcd:3.4.15

• Operating system (run uname -a):
  Darwin PUKZALML0035 22.4.0 Darwin Kernel Version 22.4.0: Mon Mar 6 20:59:28 PST 2023; root:xnu-8796.101.5~3/RELEASE_ARM64_T6000 arm64

[Revolyssup]

@nomikz Can you show the format of the request (body and other things) from POSTMAN client?

[Revolyssup]

@nomikz Is your issue solved? If not, then will you provide the info asked above?

[github-actions]

Due to lack of the reporter's response this issue has been labeled with "no response". It will be close in 3 days if no further activity occurs. If this issue is still relevant, please simply write any comment. Even if closed, you can still revive the issue at any time or discuss it on the dev@apisix.apache.org list. Thank you for your contributions.

[github-actions]

This issue has been closed due to lack of activity. If you think that is incorrect, or the issue requires additional review, you can revive the issue at any time.