fix: Fix Async function support in Python

Author: dstufft

packages/jsii-pacmak/lib/targets/python.ts

@@ -857,6 +857,11 @@ class Method extends BaseMethod {
     protected readonly jsiiMethod: string = "invoke";
 }
 
+class AsyncMethod extends BaseMethod {
+    protected readonly implicitParameter: string = "self";
+    protected readonly jsiiMethod: string = "ainvoke";
+}
+
 class StaticProperty extends BaseProperty {
     protected readonly decorator: string = "classproperty";
     protected readonly implicitParameter: string = "cls";
@@ -1473,15 +1478,27 @@ class PythonGenerator extends Generator {
     protected onMethod(cls: spec.ClassType, method: spec.Method) {
         const { parameters = [] } = method;
 
-        this.getPythonType(cls.fqn).addMember(
-            new Method(
-                toPythonMethodName(method.name!, method.protected),
-                method.name,
-                parameters,
-                method.returns,
-                { abstract: method.abstract, liftedProp: this.getliftedProp(method) },
-            )
-        );
+        if (this.isAsyncMethod(method)) {
+            this.getPythonType(cls.fqn).addMember(
+                new AsyncMethod(
+                    toPythonMethodName(method.name!, method.protected),
+                    method.name,
+                    parameters,
+                    method.returns,
+                    { abstract: method.abstract, liftedProp: this.getliftedProp(method) },
+                )
+            );
+        } else {
+            this.getPythonType(cls.fqn).addMember(
+                new Method(
+                    toPythonMethodName(method.name!, method.protected),
+                    method.name,
+                    parameters,
+                    method.returns,
+                    { abstract: method.abstract, liftedProp: this.getliftedProp(method) },
+                )
+            );
+        }
     }
 
     protected onProperty(cls: spec.ClassType, prop: spec.Property) {
@@ -1651,4 +1668,8 @@ class PythonGenerator extends Generator {
 
         return abstractBases;
     }
+
+    private isAsyncMethod(method: spec.Method): boolean {
+        return method.returns !== undefined && method.returns.promise !== undefined && method.returns.promise;
+    }
 }

packages/jsii-python-runtime/src/jsii/__init__.py

@@ -30,6 +30,7 @@
 sget = kernel.sget
 sset = kernel.sset
 invoke = kernel.invoke
+ainvoke = kernel.ainvoke
 sinvoke = kernel.sinvoke
 stats = kernel.stats
 
@@ -56,6 +57,7 @@
     "sget",
     "sset",
     "invoke",
+    "ainvoke",
     "sinvoke",
     "stats",
 ]

packages/jsii-python-runtime/src/jsii/_kernel/__init__.py

@@ -15,8 +15,12 @@
 from jsii._kernel.types import (
     EnumRef,
     LoadRequest,
+    BeginRequest,
+    CallbacksRequest,
     CreateRequest,
+    CompleteRequest,
     DeleteRequest,
+    EndRequest,
     GetRequest,
     InvokeRequest,
     SetRequest,
@@ -36,6 +40,12 @@ class Object:
     __jsii_type__ = "Object"
 
 
+def _handle_callback(kernel, callback):
+    obj = _reference_map.resolve_id(callback.invoke.objref.ref)
+    method = getattr(obj, callback.cookie)
+    return method(*callback.invoke.args)
+
+
 def _get_overides(klass: JSClass, obj: Any) -> List[Override]:
     overrides = []
 
@@ -218,6 +228,42 @@ def sinvoke(
             )
         ).result
 
+    @_dereferenced
+    def ainvoke(
+        self, obj: Referenceable, method: str, args: Optional[List[Any]] = None
+    ) -> Any:
+        if args is None:
+            args = []
+
+        promise = self.provider.begin(
+            BeginRequest(
+                objref=obj.__jsii_ref__,
+                method=method,
+                args=_make_reference_for_native(self, args),
+            )
+        )
+
+        callbacks = self.provider.callbacks(CallbacksRequest()).callbacks
+        while callbacks:
+            for callback in callbacks:
+                try:
+                    result = _handle_callback(self, callback)
+                except Exception as exc:
+                    # TODO: Maybe we want to print the whole traceback here?
+                    complete = self.provider.complete(
+                        CompleteRequest(cbid=callback.cbid, err=str(exc))
+                    )
+                else:
+                    complete = self.provider.complete(
+                        CompleteRequest(cbid=callback.cbid, result=result)
+                    )
+
+                assert complete.cbid == callback.cbid
+
+            callbacks = self.provider.callbacks(CallbacksRequest()).callbacks
+
+        return self.provider.end(EndRequest(promiseid=promise.promiseid)).result
+
     def stats(self):
         resp = self.provider.stats(StatsRequest())
 

packages/jsii-python-runtime/src/jsii/_kernel/providers/base.py

@@ -18,6 +18,14 @@
     StaticGetRequest,
     StaticInvokeRequest,
     StaticSetRequest,
+    BeginRequest,
+    BeginResponse,
+    EndRequest,
+    EndResponse,
+    CallbacksRequest,
+    CallbacksResponse,
+    CompleteRequest,
+    CompleteResponse,
     StatsRequest,
     StatsResponse,
 )
@@ -66,6 +74,22 @@ def sinvoke(self, request: StaticInvokeRequest) -> InvokeResponse:
     def delete(self, request: DeleteRequest) -> DeleteResponse:
         ...
 
+    @abc.abstractmethod
+    def begin(self, request: BeginRequest) -> BeginResponse:
+        ...
+
+    @abc.abstractmethod
+    def end(self, request: EndRequest) -> EndResponse:
+        ...
+
+    @abc.abstractmethod
+    def callbacks(self, request: CallbacksRequest) -> CallbacksResponse:
+        ...
+
+    @abc.abstractmethod
+    def complete(self, request: CompleteRequest) -> CompleteResponse:
+        ...
+
     @abc.abstractmethod
     def stats(self, request: Optional[StatsRequest] = None) -> StatsResponse:
         ...

packages/jsii-python-runtime/src/jsii/_kernel/providers/process.py

@@ -42,6 +42,14 @@
     StaticGetRequest,
     StaticInvokeRequest,
     StaticSetRequest,
+    BeginRequest,
+    BeginResponse,
+    EndRequest,
+    EndResponse,
+    CallbacksRequest,
+    CallbacksResponse,
+    CompleteRequest,
+    CompleteResponse,
     StatsRequest,
     StatsResponse,
 )
@@ -183,6 +191,21 @@ def __init__(self):
             StaticInvokeRequest,
             _with_api_key("sinvoke", self._serializer.unstructure_attrs_asdict),
         )
+        self._serializer.register_unstructure_hook(
+            BeginRequest,
+            _with_api_key("begin", self._serializer.unstructure_attrs_asdict),
+        )
+        self._serializer.register_unstructure_hook(
+            EndRequest, _with_api_key("end", self._serializer.unstructure_attrs_asdict)
+        )
+        self._serializer.register_unstructure_hook(
+            CallbacksRequest,
+            _with_api_key("callbacks", self._serializer.unstructure_attrs_asdict),
+        )
+        self._serializer.register_unstructure_hook(
+            CompleteRequest,
+            _with_api_key("complete", self._serializer.unstructure_attrs_asdict),
+        )
         self._serializer.register_unstructure_hook(
             StatsRequest,
             _with_api_key("stats", self._serializer.unstructure_attrs_asdict),
@@ -337,6 +360,18 @@ def sinvoke(self, request: StaticInvokeRequest) -> InvokeResponse:
     def delete(self, request: DeleteRequest) -> DeleteResponse:
         return self._process.send(request, DeleteResponse)
 
+    def begin(self, request: BeginRequest) -> BeginResponse:
+        return self._process.send(request, BeginResponse)
+
+    def end(self, request: EndRequest) -> EndResponse:
+        return self._process.send(request, EndResponse)
+
+    def callbacks(self, request: CallbacksRequest) -> CallbacksResponse:
+        return self._process.send(request, CallbacksResponse)
+
+    def complete(self, request: CompleteRequest) -> CompleteResponse:
+        return self._process.send(request, CompleteResponse)
+
     def stats(self, request: Optional[StatsRequest] = None) -> StatsResponse:
         if request is None:
             request = StatsRequest()

packages/jsii-python-runtime/src/jsii/_kernel/types.py

@@ -164,10 +164,10 @@ class EndResponse:
 class Callback:
 
     cbid: str
-    cookie: Optional[str]
-    invoke: Optional[InvokeRequest]
-    get: Optional[GetRequest]
-    set: Optional[SetRequest]
+    cookie: Optional[str] = None
+    invoke: Optional[InvokeRequest] = None
+    get: Optional[GetRequest] = None
+    set: Optional[SetRequest] = None
 
 
 @attr.s(auto_attribs=True, frozen=True, slots=True)

packages/jsii-python-runtime/src/jsii/_reference_map.py

@@ -98,9 +98,13 @@ def resolve(self, kernel, ref):
 
         return inst
 
+    def resolve_id(self, id):
+        return self._refs[id]
+
 
 _refs = _ReferenceMap(_types)
 
 
 register_reference = _refs.register
 resolve_reference = _refs.resolve
+resolve_id = _refs.resolve_id

packages/jsii-python-runtime/tests/test_compliance.py

@@ -49,8 +49,7 @@
 #       Tests as closely as possible to make keeping them in sync easier.
 
 # These map distinct reasons for failures, so we an easily find them.
-xfail_async = pytest.mark.xfail(reason="Implement async methods", strict=True)
-xfail_callbacks = pytest.mark.xfail(reason="Implement callback support", strict=True)
+xfail_callbacks = pytest.mark.skip(reason="Implement callback support")
 
 
 class DerivedFromAllTypes(AllTypes):
@@ -464,39 +463,33 @@ def test_creationOfNativeObjectsFromJavaScriptObjects():
     assert unmarshalled_native_obj.__class__ == MulTen
 
 
-@xfail_async
 def test_asyncOverrides_callAsyncMethod():
     obj = AsyncVirtualMethods()
     assert obj.call_me() == 128
     assert obj.override_me(44) == 528
 
 
-@xfail_async
 def test_asyncOverrides_overrideAsyncMethod():
     obj = OverrideAsyncMethods()
     obj.call_me() == 4452
 
 
-@xfail_async
 def test_asyncOverrides_overrideAsyncMethodByParentClass():
     obj = OverrideAsyncMethodsByBaseClass()
     obj.call_me() == 4452
 
 
-@xfail_async
 def test_asyncOverrides_overrideCallsSuper():
     obj = OverrideCallsSuper()
     assert obj.override_me(12) == 1441
     assert obj.call_me() == 1209
 
 
-@xfail_async
 def test_asyncOverrides_twoOverrides():
     obj = TwoOverrides()
     assert obj.call_me() == 684
 
 
-@xfail_async
 def test_asyncOverrides_overrideThrows():
     class ThrowingAsyncVirtualMethods(AsyncVirtualMethods):
         def override_me(self, mult):
@@ -783,15 +776,14 @@ def test_reservedKeywordsAreSlugifiedInMethodNames():
     obj.return_()
 
 
-@xfail_async
 def test_nodeStandardLibrary():
     obj = NodeStandardLibrary()
 
     assert obj.fs_read_file() == "Hello, resource!"
     assert obj.fs_read_file_sync() == "Hello, resource! SYNC!"
-    assert len(obj.get_os_platform()) > 0
+    assert len(obj.os_platform) > 0
     assert (
-        obj.crypto_sha_256()
+        obj.crypto_sha256()
         == "6a2da20943931e9834fc12cfe5bb47bbd9ae43489a30726962b576f4e3993e50"
     )