Use buf to generate python stubs (#346)

* Buf python migration

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>

* Generate pyi files

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>

* Add venv to .gitgnore

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>

* Use buf to generate python stubs

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>

* Use buf docker image to generate stubs

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>

* Add stubs produced by call to `buf generate` using buf's docker image

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>

* Add pyi files

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>

* Use buf locally

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>

* Verify that generated protos by using buf

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>

* Copy generated code to a separate artifact

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>

* Move back to go_generate.yml@master

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Co-authored-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>

.github/workflows/verification.yml

@@ -51,19 +51,20 @@ jobs:
       - name: Proto-Datacatalog-Go
         run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/datacatalog -l go --go_source_relative --validate_out
 
+      # Python is going to be generated using buf
       # Python
-      - name: Proto-Service-Python
-        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/service -l python
-      - name: Proto-Admin-Python
-        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/admin -l python
-      - name: Proto-Core-Python
-        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/core -l python
-      - name: Proto-Event-Python
-        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/event -l python
-      - name: Proto-Plugins-Python
-        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/plugins -l python
-      - name: Proto-Datacatalog-Python
-        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/datacatalog -l python
+      # - name: Proto-Service-Python
+      #   run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/service -l python
+      # - name: Proto-Admin-Python
+      #   run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/admin -l python
+      # - name: Proto-Core-Python
+      #   run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/core -l python
+      # - name: Proto-Event-Python
+      #   run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/event -l python
+      # - name: Proto-Plugins-Python
+      #   run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/plugins -l python
+      # - name: Proto-Datacatalog-Python
+      #   run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/datacatalog -l python
 
       # Cpp
       - name: Proto-Service-Cpp
@@ -98,10 +99,31 @@ jobs:
           name: generated
           path: ./gen
 
+  generate-protos-buf:
+    name: Generate python protos using buf
+    runs-on: ubuntu-latest
+    needs: [ generate-protos ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - uses: bufbuild/buf-setup-action@v1
+      - name: "Clean Generated"
+        run: rm -rf ./gen
+      - name: "Run buf generate"
+        run: buf generate
+      - uses: actions/upload-artifact@master
+        with:
+          name: generated-buf
+          path: ./gen
+
   generate-admin-swagger-code:
     runs-on: ubuntu-latest
     name: Generate Admin Swagger Code
-    needs: [ generate-protos ]
+    needs:
+      - generate-protos
+      - generate-protos-buf
     container:
       image: lyft/protocgenerator:8167e11d3b3439373c2f033080a4b550078884a2
       options: --cpus 1
@@ -116,6 +138,10 @@ jobs:
         with:
           name: generated
           path: ./gen
+      - uses: actions/download-artifact@master
+        with:
+          name: generated-buf
+          path: ./gen
       - run: ls gen/pb-go/flyteidl/service/
       - run: cp -R gen/pb-go/flyteidl/service/ ../
       - run: ls ../service/

.gitignore

@@ -16,3 +16,5 @@ docs/build/
 tmp/
 .python-version
 __pycache__/
+
+venv/

buf.gen.yaml

@@ -0,0 +1,10 @@
+version: v1
+managed:
+  enabled: true
+plugins:
+  - remote: buf.build/grpc/plugins/python:v1.49.1-1
+    out: gen/pb_python
+  - remote: buf.build/protocolbuffers/plugins/python:v21.7.0-1
+    out: gen/pb_python
+  - remote: buf.build/protocolbuffers/plugins/pyi:v21.9.0-1
+    out: gen/pb_python