tests: rework e2e and unit tests

Signed-off-by: Aaron Pham <29749331+aarnphm@users.noreply.github.com>
aarnphm · Sep 20, 2022 · 1886023 · 1886023
1 parent 2e38f35
commit 1886023
Show file tree

Hide file tree

Showing 59 changed files with 3,344 additions and 683 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -13,6 +13,11 @@ env:
   LINES: 120
   COLUMNS: 120
 
+# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#defaultsrun
+defaults:
+  run:
+    shell: bash --noprofile --norc -exo pipefail {0}
+
 jobs:
   diff:
     runs-on: ubuntu-latest
@@ -35,7 +40,10 @@ jobs:
               - scripts/ci/config.yml
               - scripts/ci/run_tests.sh
               - requirements/tests-requirements.txt
+            protos: &protos
+              - "bentoml/grpc/**/*.proto"
             bentoml:
+              - *protos
               - *related
               - "bentoml/**"
               - "tests/**"
@@ -47,9 +55,6 @@ jobs:
 
   codestyle_check:
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash
     needs:
       - diff
 
@@ -73,9 +78,13 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: "17"
-      - name: install pyright
+      - name: Install pyright
         run: |
           npm install -g npm@^7 pyright
+      - name: Setup bufbuild/buf
+        uses: bufbuild/buf-setup-action@v1.8.0
+        with:
+          github_token: ${{ github.token }}
 
       - name: Cache pip dependencies
         uses: actions/cache@v3
@@ -95,12 +104,11 @@ jobs:
         run: make ci-lint
       - name: Type check
         run: make ci-pyright
+      - name: Proto check
+        if: ${{ (github.event_name == 'pull_request' && needs.diff.outputs.protos == 'true') || github.event_name == 'push' }}
+        run: buf lint --config "bentoml/grpc/buf.yaml" --error-format msvs --path "bentoml/grpc"
 
   documentation_spelling_check:
-    defaults:
-      run:
-        shell: bash
-
     runs-on: ubuntu-latest
     needs:
       - diff
@@ -139,7 +147,6 @@ jobs:
 
       - name: Run spellcheck script
         run: make spellcheck-docs
-        shell: bash
 
   unit_tests:
     needs:
@@ -150,9 +157,6 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
         python-version: ["3.7", "3.8", "3.9", "3.10"]
-    defaults:
-      run:
-        shell: bash
 
     if: ${{ (github.event_name == 'pull_request' && needs.diff.outputs.bentoml == 'true') || github.event_name == 'push' }}
     name: python${{ matrix.python-version }}_unit_tests (${{ matrix.os }})
@@ -181,19 +185,19 @@ jobs:
           path: ${{ steps.cache-dir.outputs.dir }}
           key: ${{ runner.os }}-tests-${{ hashFiles('requirements/tests-requirements.txt') }}
 
+      # Simulate ./scripts/generate_grpc_stubs.sh
+      - name: Generate gRPC stubs
+        run: |
+          pip install protobuf==3.19.4 "grpcio-tools==1.41"
+          find bentoml/grpc/v1alpha1 -type f -name "*.proto" -exec python -m grpc_tools.protoc -I. --grpc_python_out=. --python_out=. "{}" \;
+
       - name: Install dependencies
         run: |
-          pip install .
+          pip install ".[grpc]"
           pip install -r requirements/tests-requirements.txt
 
       - name: Run unit tests
-        if: ${{ matrix.os != 'windows-latest' }}
-        run: make tests-unit
-
-      - name: Run unit tests (Windows)
-        if: ${{ matrix.os == 'windows-latest' }}
-        run: make tests-unit
-        shell: bash
+        run: ./scripts/ci/run_tests.sh unit --verbose
 
       - name: Upload test coverage to Codecov
         uses: codecov/codecov-action@v3
@@ -214,12 +218,13 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
         python-version: ["3.7", "3.8", "3.9", "3.10"]
-    defaults:
-      run:
-        shell: bash
+        server_type: ["http", "grpc"]
+        exclude:
+          - os: windows-latest
+            server_type: "grpc"
 
     if: ${{ (github.event_name == 'pull_request' && needs.diff.outputs.bentoml == 'true') || github.event_name == 'push' }}
-    name: python${{ matrix.python-version }}_e2e_tests (${{ matrix.os }})
+    name: python${{ matrix.python-version }}_${{ matrix.server_type }}_e2e_tests (${{ matrix.os }})
     runs-on: ${{ matrix.os }}
     timeout-minutes: 20
 
@@ -257,24 +262,29 @@ jobs:
           path: ${{ steps.cache-dir.outputs.dir }}
           key: ${{ runner.os }}-tests-${{ hashFiles('requirements/tests-requirements.txt') }}
 
-      - name: Install dependencies
+      - name: Install dependencies for ${{ matrix.server_type }}-based tests.
         run: |
-          pip install -e ".[grpc]"
           pip install -r requirements/tests-requirements.txt
-          pip install -r tests/e2e/bento_server_general_features/requirements.txt
-
-      - name: Export Action Envvar
-        run: export GITHUB_ACTION=true
-
-      - name: Run tests and generate coverage report
-        run: ./scripts/ci/run_tests.sh general_features
+          if [ "${{ matrix.server_type }}" == 'grpc' ]; then
+            pip install -e ".[grpc]"
+          else
+            pip install -e .
+          fi
+          if [ -f "tests/e2e/bento_server_${{ matrix.server_type }}/requirements.txt" ]; then
+            pip install -r tests/e2e/bento_server_${{ matrix.server_type }}/requirements.txt
+          fi
+
+      - name: Run ${{ matrix.server_type }} tests and generate coverage report
+        run: ./scripts/ci/run_tests.sh ${{ matrix.server_type }}_server --verbose
 
       - name: Upload test coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
-          flags: e2e-tests
+          flags: e2e-tests-${{ matrix.server_type }}
+          name: codecov-${{ matrix.os }}-python${{ matrix.python-version }}-e2e
+          fail_ci_if_error: true
           directory: ./
-          files: ./tests/e2e/bento_server_general_features/general_features.xml
+          files: ./tests/e2e/bento_server_${{ matrix.server_type }}/${{ matrix.server_type }}_server.xml
           verbose: true
 
 concurrency:

diff --git a/bentoml/_internal/io_descriptors/multipart.py b/bentoml/_internal/io_descriptors/multipart.py
@@ -249,16 +249,11 @@ async def from_proto(self, field: pb.Multipart) -> dict[str, t.Any]:
         reqs = await asyncio.gather(
             *tuple(
                 io_.from_proto(getattr(input_pb, io_._proto_fields[0]))
-                for io_, input_pb in self.io_fields_mapping(message).items()
+                for io_, input_pb in zip(self._inputs.values(), message.values())
             )
         )
         return dict(zip(message, reqs))
 
-    def io_fields_mapping(
-        self, message: t.MutableMapping[str, pb.Part]
-    ) -> dict[IODescriptor[t.Any], pb.Part]:
-        return {io_: part for io_, part in zip(self._inputs.values(), message.values())}
-
     async def to_proto(self, obj: dict[str, t.Any]) -> pb.Multipart:
         self.validate_input_mapping(obj)
         resps = await asyncio.gather(
@@ -268,13 +263,13 @@ async def to_proto(self, obj: dict[str, t.Any]) -> pb.Multipart:
             )
         )
         return pb.Multipart(
-            fields={
-                key: pb.Part(
-                    **{
-                        io_._proto_fields[0]: resp
+            fields=dict(
+                zip(
+                    obj,
+                    [
+                        pb.Part(**{io_._proto_fields[0]: resp})
                         for io_, resp in zip(self._inputs.values(), resps)
-                    }
+                    ],
                 )
-                for key in obj
-            }
+            )
         )
diff --git a/bentoml/grpc/types.py b/bentoml/grpc/types.py
@@ -24,7 +24,6 @@
     RequestDeserializerFn = t.Callable[[Request | None], object] | None
     ResponseSerializerFn = t.Callable[[bytes], Response | None] | None
 
-    HandlerMethod = t.Callable[[Request, BentoServicerContext], P]
     AsyncHandlerMethod = t.Callable[[Request, BentoServicerContext], t.Awaitable[P]]
 
     class RpcMethodHandler(
@@ -34,10 +33,10 @@ class RpcMethodHandler(
             response_streaming=bool,
             request_deserializer=RequestDeserializerFn,
             response_serializer=ResponseSerializerFn,
-            unary_unary=t.Optional[HandlerMethod[Response]],
-            unary_stream=t.Optional[HandlerMethod[Response]],
-            stream_unary=t.Optional[HandlerMethod[Response]],
-            stream_stream=t.Optional[HandlerMethod[Response]],
+            unary_unary=t.Optional[AsyncHandlerMethod[Response]],
+            unary_stream=t.Optional[AsyncHandlerMethod[Response]],
+            stream_unary=t.Optional[AsyncHandlerMethod[Response]],
+            stream_stream=t.Optional[AsyncHandlerMethod[Response]],
         ),
         grpc.RpcMethodHandler,
     ):
@@ -47,10 +46,10 @@ class RpcMethodHandler(
         response_streaming: bool
         request_deserializer: RequestDeserializerFn
         response_serializer: ResponseSerializerFn
-        unary_unary: t.Optional[HandlerMethod[Response]]
-        unary_stream: t.Optional[HandlerMethod[Response]]
-        stream_unary: t.Optional[HandlerMethod[Response]]
-        stream_stream: t.Optional[HandlerMethod[Response]]
+        unary_unary: t.Optional[AsyncHandlerMethod[Response]]
+        unary_stream: t.Optional[AsyncHandlerMethod[Response]]
+        stream_unary: t.Optional[AsyncHandlerMethod[Response]]
+        stream_stream: t.Optional[AsyncHandlerMethod[Response]]
 
     class HandlerCallDetails(
         t.NamedTuple(

diff --git a/bentoml/grpc/utils/__init__.py b/bentoml/grpc/utils/__init__.py
@@ -18,6 +18,8 @@
 
     from bentoml.exceptions import BentoMLException
     from bentoml.grpc.types import RpcMethodHandler
+    from bentoml.grpc.types import AsyncHandlerMethod
+    from bentoml.grpc.types import BentoServicerContext
     from bentoml.grpc.v1alpha1 import service_pb2 as pb
 
     # We need this here so that __all__ is detected due to lazy import
@@ -179,7 +181,13 @@ def parse_method_name(method_name: str) -> tuple[MethodName, bool]:
 
 
 def wrap_rpc_handler(
-    wrapper: t.Callable[..., t.Any],
+    wrapper: t.Callable[
+        [AsyncHandlerMethod[pb.Response]],
+        t.Callable[
+            [pb.Request, BentoServicerContext],
+            t.Coroutine[t.Any, t.Any, pb.Response | t.Awaitable[pb.Response]],
+        ],
+    ],
     handler: RpcMethodHandler | None,
 ) -> RpcMethodHandler | None:
     if not handler: