Aidan/concrete servicer stubs

.gitignore

@@ -8,6 +8,8 @@ __pycache__/
 *.pyc
 /test/generated/**/*.py
 !/test/generated/**/__init__.py
+/test/generated-concrete/**/*.py
+!/test/generated-concrete/**/__init__.py
 .pytest_cache
 /build/
 /dist/

CHANGELOG.md

@@ -6,6 +6,7 @@
   - With some more work this could be added back in a testing refactor
   - Protobuf <6.32 still had the edition enums and field options, so it *should* still work. But is untested
 - Add support for editions (up to 2024)
+- Add `generate_concrete_servicer_stubs` option to generate concrete instead of abstract servicer stubs
 
 ## 3.7.0
 

README.md

@@ -337,7 +337,14 @@ and insert it into the deprecation warning. This option will instead use a stand
 
 ```
 protoc --python_out=output/location --mypy_out=use_default_deprecation_warning:output/location
+```
+
+### `generate_concrete_servicer_stubs`
 
+By default mypy-protobuf will output servicer stubs with abstract methods. To output concrete stubs, set this option
+
+```
+protoc --python_out=output/location --mypy_grpc_out=generate_concrete_servicer_stubs:output/location
 ```
 
 ### Output suppression

mypy_protobuf/main.py

@@ -151,13 +151,15 @@ def __init__(
         readable_stubs: bool,
         relax_strict_optional_primitives: bool,
         use_default_deprecation_warnings: bool,
+        generate_concrete_servicer_stubs: bool,
         grpc: bool,
     ) -> None:
         self.fd = fd
         self.descriptors = descriptors
         self.readable_stubs = readable_stubs
         self.relax_strict_optional_primitives = relax_strict_optional_primitives
         self.use_default_depreaction_warnings = use_default_deprecation_warnings
+        self.generate_concrete_servicer_stubs = generate_concrete_servicer_stubs
         self.grpc = grpc
         self.lines: List[str] = []
         self.indent = ""
@@ -689,6 +691,7 @@ def write_services(
                 self._import("google.protobuf.service", "Service"),
                 self._import("abc", "ABCMeta"),
             )
+            # The servicer interface
             with self._indent():
                 if self._write_comments(scl):
                     wl("")
@@ -850,7 +853,8 @@ def write_grpc_methods(self, service: d.ServiceDescriptorProto, scl_prefix: Sour
         for i, method in methods:
             scl = scl_prefix + [d.ServiceDescriptorProto.METHOD_FIELD_NUMBER, i]
 
-            wl("@{}", self._import("abc", "abstractmethod"))
+            if self.generate_concrete_servicer_stubs is False:
+                wl("@{}", self._import("abc", "abstractmethod"))
             wl("def {}(", method.name)
             with self._indent():
                 wl("self,")
@@ -950,11 +954,17 @@ def write_grpc_services(
                     scl + [d.ServiceDescriptorProto.OPTIONS_FIELD_NUMBER] + [d.ServiceOptions.DEPRECATED_FIELD_NUMBER],
                     "This servicer has been marked as deprecated using proto service options.",
                 )
-            wl(
-                "class {}Servicer(metaclass={}):",
-                service.name,
-                self._import("abc", "ABCMeta"),
-            )
+            if self.generate_concrete_servicer_stubs is False:
+                wl(
+                    "class {}Servicer(metaclass={}):",
+                    service.name,
+                    self._import("abc", "ABCMeta"),
+                )
+            else:
+                wl(
+                    "class {}Servicer:",
+                    service.name,
+                )
             with self._indent():
                 if self._write_comments(scl):
                     wl("")
@@ -1126,6 +1136,7 @@ def generate_mypy_stubs(
     readable_stubs: bool,
     relax_strict_optional_primitives: bool,
     use_default_deprecation_warnings: bool,
+    generate_concrete_servicer_stubs: bool,
 ) -> None:
     for name, fd in descriptors.to_generate.items():
         pkg_writer = PkgWriter(
@@ -1134,6 +1145,7 @@ def generate_mypy_stubs(
             readable_stubs,
             relax_strict_optional_primitives,
             use_default_deprecation_warnings,
+            generate_concrete_servicer_stubs,
             grpc=False,
         )
 
@@ -1158,6 +1170,7 @@ def generate_mypy_grpc_stubs(
     readable_stubs: bool,
     relax_strict_optional_primitives: bool,
     use_default_deprecation_warnings: bool,
+    generate_concrete_servicer_stubs: bool,
 ) -> None:
     for name, fd in descriptors.to_generate.items():
         pkg_writer = PkgWriter(
@@ -1166,6 +1179,7 @@ def generate_mypy_grpc_stubs(
             readable_stubs,
             relax_strict_optional_primitives,
             use_default_deprecation_warnings,
+            generate_concrete_servicer_stubs,
             grpc=True,
         )
         pkg_writer.write_grpc_async_hacks()
@@ -1222,6 +1236,7 @@ def main() -> None:
             "readable_stubs" in request.parameter,
             "relax_strict_optional_primitives" in request.parameter,
             "use_default_deprecation_warnings" in request.parameter,
+            "generate_concrete_servicer_stubs" in request.parameter,
         )
 
 
@@ -1235,6 +1250,7 @@ def grpc() -> None:
             "readable_stubs" in request.parameter,
             "relax_strict_optional_primitives" in request.parameter,
             "use_default_deprecation_warnings" in request.parameter,
+            "generate_concrete_servicer_stubs" in request.parameter,
         )
 
 

pyproject.toml

@@ -32,11 +32,13 @@ include = [
 ]
 exclude = [
     "**/*_pb2.py",
-    "**/*_pb2_grpc.py"
+    "**/*_pb2_grpc.py",
+    "test/test_concrete.py"
 ]
 
 executionEnvironments = [
     # Due to how upb is typed, we need to disable incompatible variable override checks
     { root = "test/generated", extraPaths = ["./"], reportIncompatibleVariableOverride = "none" },
+    { root = "test/generated-concrete", extraPaths = ["./"], reportIncompatibleVariableOverride = "none" },
     { root = "mypy_protobuf/extensions_pb2.pyi", reportIncompatibleVariableOverride = "none" },
 ]

run_test.sh

@@ -124,18 +124,26 @@
     find proto -name "*.proto" -print0 | xargs -0 "$PROTOC" "${PROTOC_ARGS[@]}" --mypy_out=readable_stubs:test/generated
     find proto -name "*.proto" -print0 | xargs -0 "$PROTOC" "${PROTOC_ARGS[@]}" --mypy_out=relax_strict_optional_primitives:test/generated
     find proto -name "*.proto" -print0 | xargs -0 "$PROTOC" "${PROTOC_ARGS[@]}" --mypy_out=use_default_deprecation_warnings:test/generated
+    find proto -name "*.proto" -print0 | xargs -0 "$PROTOC" "${PROTOC_ARGS[@]}" --mypy_out=generate_concrete_servicer_stubs:test/generated
     # Overwrite w/ run with mypy-protobuf without flags
     find proto -name "*.proto" -print0 | xargs -0 "$PROTOC" "${PROTOC_ARGS[@]}" --mypy_out=test/generated
 
     # Generate grpc protos
     find proto/testproto/grpc -name "*.proto" -print0 | xargs -0 "$PROTOC" "${PROTOC_ARGS[@]}" --mypy_grpc_out=test/generated
 
+    # Generate with concrete service stubs for testing
+    find proto -name "*.proto" -print0 | xargs -0 "$PROTOC" "${PROTOC_ARGS[@]}" --mypy_out=generate_concrete_servicer_stubs:test/generated-concrete
+    find proto/testproto/grpc -name "*.proto" -print0 | xargs -0 "$PROTOC" "${PROTOC_ARGS[@]}" --mypy_grpc_out=generate_concrete_servicer_stubs:test/generated-concrete
+
+
     if [[ -n $VALIDATE ]] && ! diff <(echo "$SHA_BEFORE") <(find test/generated -name "*.pyi" -print0 | xargs -0 sha1sum); then
         echo -e "${RED}Some .pyi files did not match. Please commit those files${NC}"
         exit 1
     fi
 )
 
+ERRORS=()
+
 for PY_VER in $PY_VER_UNIT_TESTS; do
     UNIT_TESTS_VENV=venv_$PY_VER
     PY_VER_MYPY_TARGET=$(echo "$PY_VER" | cut -d. -f1-2)
@@ -149,6 +157,10 @@
     # Run mypy on unit tests / generated output
     (
         source "$MYPY_VENV"/bin/activate
+        # Run concrete mypy
+        CONCRETE_MODULES=( -m test.test_concrete )
+        MYPYPATH=$MYPYPATH:test/generated-concrete mypy ${CUSTOM_TYPESHED_DIR_ARG:+"$CUSTOM_TYPESHED_DIR_ARG"} --python-executable="$UNIT_TESTS_VENV"/bin/python --python-version="$PY_VER_MYPY_TARGET" "${CONCRETE_MODULES[@]}"
+
         export MYPYPATH=$MYPYPATH:test/generated
 
         # Run mypy
@@ -184,13 +196,13 @@
 
         call_mypy "$PY_VER" "${NEGATIVE_MODULES[@]}"
         if ! diff "$MYPY_OUTPUT/mypy_output" "test_negative/output.expected.$PY_VER_MYPY_TARGET" || ! diff "$MYPY_OUTPUT/mypy_output.omit_linenos" "test_negative/output.expected.$PY_VER_MYPY_TARGET.omit_linenos"; then
-            echo -e "${RED}test_negative/output.expected.$PY_VER_MYPY_TARGET didnt match. Copying over for you. Now rerun${NC}"
-
             # Copy over all the mypy results for the developer.
             call_mypy "$PY_VER" "${NEGATIVE_MODULES[@]}"
             cp "$MYPY_OUTPUT/mypy_output" "test_negative/output.expected.$PY_VER_MYPY_TARGET"
             cp "$MYPY_OUTPUT/mypy_output.omit_linenos" "test_negative/output.expected.$PY_VER_MYPY_TARGET.omit_linenos"
-            exit 1
+
+            # Record error instead of echoing and exiting
+            ERRORS+=("test_negative/output.expected.$PY_VER_MYPY_TARGET didnt match. Copying over for you.")
         fi
     )
 
@@ -200,3 +212,13 @@
         PYTHONPATH=test/generated py.test --ignore=test/generated -v
     )
 done
+
+# Report all errors at the end
+if [ ${#ERRORS[@]} -gt 0 ]; then
+    echo -e "\n${RED}===============================================${NC}"
+    for error in "${ERRORS[@]}"; do
+        echo -e "${RED}$error${NC}"
+    done
+    echo -e "${RED}Now rerun${NC}"
+    exit 1
+fi

test/generated-concrete/google/protobuf/duration_pb2.pyi

@@ -0,0 +1,136 @@
+"""
+@generated by mypy-protobuf.  Do not edit manually!
+isort:skip_file
+Protocol Buffers - Google's data interchange format
+Copyright 2008 Google Inc.  All rights reserved.
+https://developers.google.com/protocol-buffers/
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+    * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""
+
+import builtins
+import google.protobuf.descriptor
+import google.protobuf.internal.well_known_types
+import google.protobuf.message
+import sys
+import typing
+
+if sys.version_info >= (3, 10):
+    import typing as typing_extensions
+else:
+    import typing_extensions
+
+DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
+
+@typing.final
+class Duration(google.protobuf.message.Message, google.protobuf.internal.well_known_types.Duration):
+    """A Duration represents a signed, fixed-length span of time represented
+    as a count of seconds and fractions of seconds at nanosecond
+    resolution. It is independent of any calendar and concepts like "day"
+    or "month". It is related to Timestamp in that the difference between
+    two Timestamp values is a Duration and it can be added or subtracted
+    from a Timestamp. Range is approximately +-10,000 years.
+
+    # Examples
+
+    Example 1: Compute Duration from two Timestamps in pseudo code.
+
+        Timestamp start = ...;
+        Timestamp end = ...;
+        Duration duration = ...;
+
+        duration.seconds = end.seconds - start.seconds;
+        duration.nanos = end.nanos - start.nanos;
+
+        if (duration.seconds < 0 && duration.nanos > 0) {
+          duration.seconds += 1;
+          duration.nanos -= 1000000000;
+        } else if (duration.seconds > 0 && duration.nanos < 0) {
+          duration.seconds -= 1;
+          duration.nanos += 1000000000;
+        }
+
+    Example 2: Compute Timestamp from Timestamp + Duration in pseudo code.
+
+        Timestamp start = ...;
+        Duration duration = ...;
+        Timestamp end = ...;
+
+        end.seconds = start.seconds + duration.seconds;
+        end.nanos = start.nanos + duration.nanos;
+
+        if (end.nanos < 0) {
+          end.seconds -= 1;
+          end.nanos += 1000000000;
+        } else if (end.nanos >= 1000000000) {
+          end.seconds += 1;
+          end.nanos -= 1000000000;
+        }
+
+    Example 3: Compute Duration from datetime.timedelta in Python.
+
+        td = datetime.timedelta(days=3, minutes=10)
+        duration = Duration()
+        duration.FromTimedelta(td)
+
+    # JSON Mapping
+
+    In JSON format, the Duration type is encoded as a string rather than an
+    object, where the string ends in the suffix "s" (indicating seconds) and
+    is preceded by the number of seconds, with nanoseconds expressed as
+    fractional seconds. For example, 3 seconds with 0 nanoseconds should be
+    encoded in JSON format as "3s", while 3 seconds and 1 nanosecond should
+    be expressed in JSON format as "3.000000001s", and 3 seconds and 1
+    microsecond should be expressed in JSON format as "3.000001s".
+    """
+
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    SECONDS_FIELD_NUMBER: builtins.int
+    NANOS_FIELD_NUMBER: builtins.int
+    seconds: builtins.int
+    """Signed seconds of the span of time. Must be from -315,576,000,000
+    to +315,576,000,000 inclusive. Note: these bounds are computed from:
+    60 sec/min * 60 min/hr * 24 hr/day * 365.25 days/year * 10000 years
+    """
+    nanos: builtins.int
+    """Signed fractions of a second at nanosecond resolution of the span
+    of time. Durations less than one second are represented with a 0
+    `seconds` field and a positive or negative `nanos` field. For durations
+    of one second or more, a non-zero value for the `nanos` field must be
+    of the same sign as the `seconds` field. Must be from -999,999,999
+    to +999,999,999 inclusive.
+    """
+    def __init__(
+        self,
+        *,
+        seconds: builtins.int = ...,
+        nanos: builtins.int = ...,
+    ) -> None: ...
+    def ClearField(self, field_name: typing.Literal["nanos", b"nanos", "seconds", b"seconds"]) -> None: ...
+
+Global___Duration: typing_extensions.TypeAlias = Duration