Fix python package and publish typing information (#347)

* Add __init__.py files to generated stubs

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>

* Publish stubs in the package

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>

* Include __init__.py in verification workflow

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>

* Bump versions of remote plugins

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Co-authored-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>

flyteidl/.github/workflows/verification.yml

@@ -113,6 +113,8 @@ jobs:
         run: rm -rf ./gen
       - name: "Run buf generate"
         run: buf generate
+      - name: "Include __init__.py files"
+        run: find gen/pb_python -type d -exec touch {}/__init__.py \;
       - uses: actions/upload-artifact@master
         with:
           name: generated-buf

flyteidl/buf.gen.yaml

@@ -2,9 +2,9 @@ version: v1
 managed:
   enabled: true
 plugins:
-  - remote: buf.build/grpc/plugins/python:v1.49.1-1
+  - remote: buf.build/grpc/plugins/python:v1.51.1-1
     out: gen/pb_python
-  - remote: buf.build/protocolbuffers/plugins/python:v21.7.0-1
+  - remote: buf.build/protocolbuffers/plugins/python:v21.9.0-1
     out: gen/pb_python
   - remote: buf.build/protocolbuffers/plugins/pyi:v21.9.0-1
     out: gen/pb_python

flyteidl/generate_protos.sh

@@ -29,9 +29,16 @@ do
 done
 
 # Buf migration
+# TODO: fix stub generation via docker image. The trick of forcing the current user as a user in
+# the container (used extensively in this file) does not work.
 # docker run --volume "$(pwd):/workspace" --workdir /workspace bufbuild/buf generate
 buf generate
 
+# Unfortunately the python protoc plugin does not add __init__.py files to the generated code
+# (as described in https://github.com/protocolbuffers/protobuf/issues/881). One of the
+# suggestions is to manually create such files, which is what we do here:
+find gen/pb_python -type d -exec touch {}/__init__.py \;
+
 # Docs generated
 
 # Remove any currently generated core docs file

flyteidl/setup.py

@@ -11,6 +11,10 @@
     maintainer_email='admin@flyte.org',
     packages=find_packages('gen/pb_python'),
     package_dir={'': 'gen/pb_python'},
+    # https://github.com/pypa/setuptools/issues/3136 describes an extension to
+    # setuptools that would involve a simpler way to specify this, but while
+    # that does not happen we have to package the pyi files manually like so:
+    package_data={'flyteidl': ["*.pyi", "**/*.pyi"]},
     dependency_links=[],
     install_requires=[
         'googleapis-common-protos',