[type: test] [E2e Test] - add e2e web socket sync

.github/workflows/e2e-k8s.yml

@@ -209,6 +209,8 @@ jobs:
             script: e2e-sofa-sync
           - case: shenyu-e2e-case-grpc
             script: e2e-grpc-sync
+          - case: shenyu-e2e-case-websocket
+            script: e2e-websocket-sync
     steps:
       - uses: actions/checkout@v2
         with:

shenyu-e2e/pom.xml

@@ -57,6 +57,7 @@
         <maven-checkstyle-plugin.version>3.1.0</maven-checkstyle-plugin.version>
         <guava.version>31.1-jre</guava.version>
         <commons-collection.verion>4.4</commons-collection.verion>
+        <websocket.version>1.5.1</websocket.version>
     </properties>
 
     <modules>
@@ -67,6 +68,12 @@
     </modules>
 
     <dependencies>
+        <dependency>
+            <groupId>org.java-websocket</groupId>
+            <artifactId>Java-WebSocket</artifactId>
+            <version>${websocket.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>

shenyu-e2e/shenyu-e2e-case/pom.xml

@@ -37,6 +37,7 @@
         <module>shenyu-e2e-case-motan</module>
         <module>shenyu-e2e-case-grpc</module>
         <module>shenyu-e2e-case-brpc</module>
+        <module>shenyu-e2e-case-websocket</module>
     </modules>
 
     <dependencies>

shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-websocket/k8s/script/e2e-websocket-sync.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+docker shenyu-examples-websocket:latest | sudo k3s ctr images import -
+
+# init kubernetes for mysql
+SHENYU_TESTCASE_DIR=$(dirname "$(dirname "$(dirname "$(dirname "$0")")")")
+bash "${SHENYU_TESTCASE_DIR}"/k8s/script/storage/storage_init_mysql.sh
+
+# init register center
+CUR_PATH=$(readlink -f "$(dirname "$0")")
+PRGDIR=$(dirname "$CUR_PATH")
+echo "$PRGDIR"
+kubectl apply -f "${PRGDIR}"/shenyu-cm.yml
+
+# init shenyu sync
+SYNC_ARRAY=("websocket" "http" "zookeeper" "etcd")
+#SYNC_ARRAY=("websocket" "nacos")
+MIDDLEWARE_SYNC_ARRAY=("zookeeper" "etcd" "nacos")
+for sync in ${SYNC_ARRAY[@]}; do
+  echo -e "------------------\n"
+  kubectl apply -f "$SHENYU_TESTCASE_DIR"/k8s/shenyu-mysql.yml
+  sleep 30s
+  echo "[Start ${sync} synchronous] create shenyu-admin-${sync}.yml shenyu-bootstrap-${sync}.yml shenyu-examples-websocket.yml"
+  # shellcheck disable=SC2199
+  # shellcheck disable=SC2076
+  if [[ "${MIDDLEWARE_SYNC_ARRAY[@]}" =~ "${sync}" ]]; then
+    kubectl apply -f "${SHENYU_TESTCASE_DIR}"/k8s/shenyu-"${sync}".yml
+    sleep 10s
+  fi
+  kubectl apply -f "${SHENYU_TESTCASE_DIR}"/k8s/sync/shenyu-admin-"${sync}".yml
+  sh "${CUR_PATH}"/healthcheck.sh http://localhost:31095/actuator/health
+  sh "${CUR_PATH}"/healthcheck.sh http://localhost:30761/actuator/health
+  kubectl apply -f "${SHENYU_TESTCASE_DIR}"/k8s/sync/shenyu-bootstrap-"${sync}".yml
+  sh "${CUR_PATH}"/healthcheck.sh http://localhost:31195/actuator/health
+  kubectl apply -f "${PRGDIR}"/shenyu-examples-springcloud.yml
+  sh "${CUR_PATH}"/healthcheck.sh http://localhost:30884/actuator/health
+  sleep 10s
+  kubectl get pod -o wide
+
+  ## run e2e-test
+  ./mvnw -B -f ./shenyu-e2e/pom.xml -pl shenyu-e2e-case/shenyu-e2e-case-websocket -am test
+  # shellcheck disable=SC2181
+  if (($?)); then
+    echo "${sync}-sync-e2e-test failed"
+    echo "shenyu-admin log:"
+    echo "------------------"
+    kubectl logs "$(kubectl get pod -o wide | grep shenyu-admin | awk '{print $1}')"
+    echo "shenyu-bootstrap log:"
+    echo "------------------"
+    kubectl logs "$(kubectl get pod -o wide | grep shenyu-bootstrap | awk '{print $1}')"
+    exit 1
+  fi
+  kubectl delete -f "${SHENYU_TESTCASE_DIR}"/k8s/shenyu-mysql.yml
+  kubectl delete -f "${SHENYU_TESTCASE_DIR}"/k8s/sync/shenyu-admin-"${sync}".yml
+  kubectl delete -f "${SHENYU_TESTCASE_DIR}"/k8s/sync/shenyu-bootstrap-"${sync}".yml
+  kubectl delete -f "${PRGDIR}"/shenyu-examples-websocket.yml
+  # shellcheck disable=SC2199
+  # shellcheck disable=SC2076
+  if [[ "${MIDDLEWARE_SYNC_ARRAY[@]}" =~ "${sync}" ]]; then
+    kubectl delete -f "${SHENYU_TESTCASE_DIR}"/k8s/shenyu-"${sync}".yml
+  fi
+  echo "[Remove ${sync} synchronous] delete shenyu-admin-${sync}.yml shenyu-bootstrap-${sync}.yml shenyu-examples-websocket.yml"
+done

shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-websocket/k8s/script/healthcheck.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+for loop in $(seq 1 30); do
+  status=$(curl -s -o /dev/null -w %{http_code} -X GET "${1}" -H "accept: */*")
+  echo -e "${loop} curl ${1} response $status"
+  if [ "$status" -eq 200 ]; then
+    break
+  fi
+  sleep 2s
+done
+
+status=$(curl -s -o /dev/null -w "%{http_code}" -X GET "${1}" -H "accept: */*")
+
+if [ "$status" -eq 200 ]; then
+  echo -e "\n-------------------"
+  echo -e "Success to ${1} send request: $status"
+  echo -e "\n-------------------"
+  exit 0
+fi
+echo -e "\n-------------------"
+echo -e "Failed to send request from shenyu-admin : $status"
+echo -e "\n-------------------"
+exit 1

shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-websocket/k8s/shenyu-examples-websocket.yml

@@ -0,0 +1,73 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: shenyu-examples-websocket
+  labels:
+    app: shenyu-examples-websocket
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: shenyu-examples-websocket
+  strategy: {}
+  template:
+    metadata:
+      labels:
+        app: shenyu-examples-websocket
+    spec:
+      containers:
+        - image: shenyu-examples-websocket:latest
+          name: shenyu-examples-websocket
+          livenessProbe:
+            exec:
+              command:
+                - wget -q -O - http://localhost:8001/actuator/health | grep UP || exit 1
+            initialDelaySeconds: 30
+            failureThreshold: 3
+            timeoutSeconds: 2
+          env:
+            - name: shenyu.register.serverLists
+              value: http://shenyu-admin:9095
+            - name: websocket.registry.address
+              value: zookeeper://shenyu-zookeeper:2181
+          ports:
+            - containerPort: 8848
+            - containerPort: 31188
+          imagePullPolicy: IfNotPresent
+      restartPolicy: Always
+status: {}
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: shenyu-examples-websocket
+  labels:
+    app: shenyu-examples-websocket
+spec:
+  selector:
+    app: shenyu-examples-websocket
+  type: NodePort
+  ports:
+    - name: "8848"
+      port: 8848
+      targetPort: 8848
+      nodePort: 31191
+status:
+  loadBalancer: {}

shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-websocket/pom.xml

@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <groupId>org.apache.shenyu</groupId>
+        <artifactId>shenyu-e2e-case</artifactId>
+        <version>0.0.1-SNAPSHOT</version>
+    </parent>
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>shenyu-e2e-case-websocket</artifactId>
+
+    <properties>
+        <websocket.version>1.5.1</websocket.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.java-websocket</groupId>
+            <artifactId>Java-WebSocket</artifactId>
+            <version>1.5.3</version>
+        </dependency>
+    </dependencies>
+
+</project>

shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-websocket/src/test/java/org/apache/shenyue/e2e/testcase/websocket/DataSynTest.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shenyue.e2e.testcase.websocket;
+
+import org.apache.shenyu.e2e.client.WaitDataSync;
+import org.apache.shenyu.e2e.client.admin.AdminClient;
+import org.apache.shenyu.e2e.client.gateway.GatewayClient;
+import org.apache.shenyu.e2e.engine.annotation.ShenYuTest;
+import org.apache.shenyu.e2e.enums.ServiceTypeEnum;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Testing the correctness of etcd data synchronization method.
+ */
+@ShenYuTest(environments = {
+        @ShenYuTest.Environment(
+                serviceName = "shenyu-e2e-admin",
+                service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e",
+                        baseUrl = "http://localhost:9095",
+                        type = ServiceTypeEnum.SHENYU_ADMIN,
+                        parameters = {
+                                @ShenYuTest.Parameter(key = "username", value = "admin"),
+                                @ShenYuTest.Parameter(key = "password", value = "123456")
+                        }
+                )
+        ),
+        @ShenYuTest.Environment(
+                serviceName = "shenyu-e2e-gateway",
+                service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e",
+                        baseUrl = "http://localhost:9195",
+                        type = ServiceTypeEnum.SHENYU_GATEWAY
+                )
+        )
+})
+public class DataSynTest {
+
+    @Test
+    void testDataSyn(final AdminClient adminClient, final GatewayClient gatewayClient) throws Exception {
+        adminClient.login();
+        WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllSelectors, gatewayClient::getSelectorCache, adminClient);
+        WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllMetaData, gatewayClient::getMetaDataCache, adminClient);
+        WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllRules, gatewayClient::getRuleCache, adminClient);
+    }
+}

shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-websocket/src/test/java/org/apache/shenyue/e2e/testcase/websocket/WebSocketPluginCases.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shenyue.e2e.testcase.websocket;
+
+import com.google.common.collect.Lists;
+import org.apache.shenyu.e2e.engine.scenario.ShenYuScenarioProvider;
+import org.apache.shenyu.e2e.engine.scenario.specification.ScenarioSpec;
+import org.apache.shenyu.e2e.engine.scenario.specification.ShenYuBeforeEachSpec;
+import org.apache.shenyu.e2e.engine.scenario.specification.ShenYuCaseSpec;
+import org.apache.shenyu.e2e.engine.scenario.specification.ShenYuScenarioSpec;
+
+import java.util.List;
+
+import static org.apache.shenyu.e2e.engine.scenario.function.WebSocketCheckers.exists;
+
+public class WebSocketPluginCases implements ShenYuScenarioProvider {
+
+    private static final String WEBSOCKET_URI = "ws://localhost:9195/ws-annotation/myWs?token=Jack";
+
+    @Override
+    public List<ScenarioSpec> get() {
+        return Lists.newArrayList(
+                testWebSocket()
+        );
+    }
+
+    /**
+     * test websocket
+     *
+     * @return ShenYuScenarioSpec
+     */
+    public ShenYuScenarioSpec testWebSocket() {
+        return ShenYuScenarioSpec.builder()
+                .name("single-websocket uri starts_with]")
+                .beforeEachSpec(ShenYuBeforeEachSpec.builder()
+                        .checker(exists("/ws-annotation/myWs", "Hello ShenYu", "server send message：Hello ShenYu"))
+                        .build())
+                .caseSpec(ShenYuCaseSpec.builder()
+                        .addExists("/ws-annotation/myWs", "Hello ShenYu", "server send message：Hello ShenYu")
+                        .addNotExists("/ws-native/myWs", "Hello ShenYu")
+                        .build())
+                .build();
+    }
+}