refactor: adopt latest protobufs (#34)

Because

- the latest protobuf requires the adoption for pipeline-backend

This commit

- update with the latest protobuf which simplify output and update patch
- close #32

.github/.codecov.yml

@@ -0,0 +1,6 @@
+coverage:
+  status:
+    project:
+      default:
+        target: auto
+        threshold: 50%

cmd/interceptors.go

@@ -3,12 +3,13 @@ package main
 import (
 	"context"
 
-	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
-	grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
+
+	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
+	grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery"
 )
 
 // RecoveryInterceptor - panic handler

cmd/main.go

@@ -10,26 +10,29 @@ import (
 	"strings"
 	"syscall"
 
+	"golang.org/x/net/http2"
+	"golang.org/x/net/http2/h2c"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/protobuf/encoding/protojson"
+
 	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	grpc_zap "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap"
 	grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery"
+
 	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	"github.com/instill-ai/pipeline-backend/configs"
-	cache "github.com/instill-ai/pipeline-backend/internal/cache"
-	database "github.com/instill-ai/pipeline-backend/internal/db"
 	"github.com/instill-ai/pipeline-backend/internal/logger"
 	"github.com/instill-ai/pipeline-backend/internal/temporal"
 	"github.com/instill-ai/pipeline-backend/pkg/repository"
 	"github.com/instill-ai/pipeline-backend/pkg/service"
 	"github.com/instill-ai/pipeline-backend/rpc"
-	modelPB "github.com/instill-ai/protogen-go/model"
-	pipelinePB "github.com/instill-ai/protogen-go/pipeline"
-	"golang.org/x/net/http2"
-	"golang.org/x/net/http2/h2c"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials"
-	"google.golang.org/grpc/credentials/insecure"
-	"google.golang.org/protobuf/encoding/protojson"
+
+	cache "github.com/instill-ai/pipeline-backend/internal/cache"
+	database "github.com/instill-ai/pipeline-backend/internal/db"
+	modelPB "github.com/instill-ai/protogen-go/model/v1alpha"
+	pipelinePB "github.com/instill-ai/protogen-go/pipeline/v1alpha"
 )
 
 func grpcHandlerFunc(grpcServer *grpc.Server, gwHandler http.Handler) http.Handler {
@@ -47,6 +50,7 @@ func grpcHandlerFunc(grpcServer *grpc.Server, gwHandler http.Handler) http.Handl
 func main() {
 
 	logger, _ := logger.GetZapLogger()
+	defer logger.Sync() //nolint
 	grpc_zap.ReplaceGrpcLoggerV2(logger)
 
 	if err := configs.Init(); err != nil {
@@ -74,7 +78,7 @@ func main() {
 		grpc_zap.WithDecider(func(fullMethodName string, err error) bool {
 			// will not log gRPC calls if it was a call to liveness or readiness and no error was raised
 			if err == nil {
-				if match, _ := regexp.MatchString("instill.pipeline.Pipeline/.*ness$", fullMethodName); match {
+				if match, _ := regexp.MatchString("instill.pipeline.v1alpha.PipelineService/.*ness$", fullMethodName); match {
 					return false
 				}
 			}
@@ -114,21 +118,20 @@ func main() {
 		logger.Fatal(err.Error())
 	}
 
-	modelServiceClient := modelPB.NewModelClient(clientConn)
+	modelServiceClient := modelPB.NewModelServiceClient(clientConn)
 
 	pipelineRepository := repository.NewPipelineRepository(db)
 	pipelineService := service.NewPipelineService(pipelineRepository, modelServiceClient)
 	pipelineHandler := rpc.NewPipelineServiceHandlers(pipelineService)
 
 	grpcS := grpc.NewServer(grpcServerOpts...)
-	pipelinePB.RegisterPipelineServer(grpcS, pipelineHandler)
+	pipelinePB.RegisterPipelineServiceServer(grpcS, pipelineHandler)
 
 	gwS := runtime.NewServeMux(
 		runtime.WithForwardResponseOption(httpResponseModifier),
 		runtime.WithIncomingHeaderMatcher(customMatcher),
 		runtime.WithMarshalerOption(runtime.MIMEWildcard, &runtime.JSONPb{
 			MarshalOptions: protojson.MarshalOptions{
-				UseProtoNames:   true,
 				EmitUnpopulated: true,
 			},
 			UnmarshalOptions: protojson.UnmarshalOptions{
@@ -137,7 +140,7 @@ func main() {
 		}),
 	)
 
-	// Register custom route for  GET /hello/{name}
+	// Register custom route for POST multipart form data
 	if err := gwS.HandlePath("POST", "/pipelines/{name}/upload/outputs", appendCustomHeaderMiddleware(rpc.HandleUploadOutput)); err != nil {
 		panic(err)
 	}
@@ -149,7 +152,7 @@ func main() {
 		dialOpts = []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
 	}
 
-	if err := pipelinePB.RegisterPipelineHandlerFromEndpoint(ctx, gwS, fmt.Sprintf(":%v", configs.Config.Server.Port), dialOpts); err != nil {
+	if err := pipelinePB.RegisterPipelineServiceHandlerFromEndpoint(ctx, gwS, fmt.Sprintf(":%v", configs.Config.Server.Port), dialOpts); err != nil {
 		logger.Fatal(err.Error())
 	}
 
@@ -190,10 +193,9 @@ func main() {
 
 	logger.Info("Shutting down server...")
 
-	grpcS.GracefulStop()
-	database.Close(db)
-	cache.Close()
+	// grpcS.GracefulStop()
 	temporal.Close()
+	cache.Close()
+	database.Close(db)
 
-	_ = logger.Sync()
 }

configs/configs.go

@@ -7,12 +7,13 @@ import (
 	"time"
 
 	"github.com/go-redis/redis/v8"
-	"github.com/instill-ai/pipeline-backend/internal/logger"
 	"github.com/knadh/koanf"
 	"github.com/knadh/koanf/parsers/yaml"
 	"github.com/knadh/koanf/providers/env"
 	"github.com/knadh/koanf/providers/file"
 	"go.temporal.io/sdk/client"
+
+	"github.com/instill-ai/pipeline-backend/internal/logger"
 )
 
 // Config - Global variable to export

go.mod

@@ -9,15 +9,15 @@ require (
 	github.com/golang/mock v1.6.0
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.3
-	github.com/instill-ai/protogen-go v0.1.1-alpha
-	github.com/instill-ai/vdp v0.1.2-alpha
+	github.com/instill-ai/protogen-go v0.1.4-alpha
+	github.com/instill-ai/vdp v0.1.3-alpha.0.20220320033117-c3e0db375b45
 	github.com/instill-ai/x v0.1.0-alpha
 	github.com/knadh/koanf v1.4.0
 	github.com/stretchr/testify v1.7.0
-	go.temporal.io/sdk v1.13.0
+	go.temporal.io/sdk v1.13.1
 	go.uber.org/zap v1.21.0
-	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd
-	google.golang.org/grpc v1.44.0
+	golang.org/x/net v0.0.0-20220225172249-27dd8689420f
+	google.golang.org/grpc v1.45.0
 	google.golang.org/protobuf v1.27.1
 	gorm.io/driver/postgres v1.2.3
 	gorm.io/gorm v1.22.5
@@ -57,10 +57,10 @@ require (
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
-	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
+	golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
-	google.golang.org/genproto v0.0.0-20220118154757-00ab72f36ad5 // indirect
+	google.golang.org/genproto v0.0.0-20220317150908-0efb43f6373e // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )

go.sum

@@ -595,11 +595,11 @@ github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/instill-ai/protogen-go v0.0.0-20220219173653-3c0c45e11858/go.mod h1:q2Pq4P0AY/59RGibT4nSDnOsA4wD4XhLueFRoGYNBjk=
-github.com/instill-ai/protogen-go v0.1.1-alpha h1:rZ/mGWOi8sbSfub840WBDB/AmaClgkRjGT/8sDvCveo=
-github.com/instill-ai/protogen-go v0.1.1-alpha/go.mod h1:q2Pq4P0AY/59RGibT4nSDnOsA4wD4XhLueFRoGYNBjk=
-github.com/instill-ai/vdp v0.1.2-alpha h1:+R0TEb/Igo1JE/UrH9ZzeO/kUuv1ZZSGLl2ypTVabxw=
-github.com/instill-ai/vdp v0.1.2-alpha/go.mod h1:lEhCVtzWPBU8dr/hpaiVKpwbByiYYpwtXMDQy6OJ+Po=
+github.com/instill-ai/protogen-go v0.1.3-alpha/go.mod h1:q2Pq4P0AY/59RGibT4nSDnOsA4wD4XhLueFRoGYNBjk=
+github.com/instill-ai/protogen-go v0.1.4-alpha h1:Je7QW9/4S1nWgAGwEtEZb0cUl0n/Fbilp/bkl99KN2o=
+github.com/instill-ai/protogen-go v0.1.4-alpha/go.mod h1:q2Pq4P0AY/59RGibT4nSDnOsA4wD4XhLueFRoGYNBjk=
+github.com/instill-ai/vdp v0.1.3-alpha.0.20220320033117-c3e0db375b45 h1:MureMU/JCh3C0QwBVVgv1HsIPRcCN+wxqpg9vnvfutU=
+github.com/instill-ai/vdp v0.1.3-alpha.0.20220320033117-c3e0db375b45/go.mod h1:xcFgWKOHkqqkKMUkc5ftoD/5/hMSVR3qNhnnHxWzxYQ=
 github.com/instill-ai/x v0.1.0-alpha h1:cXzOGkHr+u1XdB8Pn6VHYb+hZX621mvlz7qBxZ29S5Y=
 github.com/instill-ai/x v0.1.0-alpha/go.mod h1:MzD0v46c0h0t5tvE8Fmqa44rxrI75Ew5JK2iWqqZV2g=
 github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
@@ -1191,8 +1191,8 @@ golang.org/x/net v0.0.0-20210913180222-943fd674d43e/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20211013171255-e13a2654a71e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211109214657-ef0fda0de508/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
-golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
+golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/oauth2 v0.0.0-20180227000427-d7d64896b5ff/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -1322,8 +1322,9 @@ golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211209171907-798191bca915/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs=
+golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -1528,8 +1529,9 @@ google.golang.org/genproto v0.0.0-20211013025323-ce878158c4d4/go.mod h1:5CzLGKJ6
 google.golang.org/genproto v0.0.0-20211104193956-4c6863e31247/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20220118154757-00ab72f36ad5 h1:zzNejm+EgrbLfDZ6lu9Uud2IVvHySPl8vQzf04laR5Q=
 google.golang.org/genproto v0.0.0-20220118154757-00ab72f36ad5/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/genproto v0.0.0-20220317150908-0efb43f6373e h1:fNKDNuUyC4WH+inqDMpfXDdfvwfYILbsX+oskGZ8hxg=
+google.golang.org/genproto v0.0.0-20220317150908-0efb43f6373e/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E=
 google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
@@ -1564,8 +1566,9 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K
 google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k=
 google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
 google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
-google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg=
 google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
+google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M=
+google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=

internal/cache/cache.go

@@ -2,6 +2,7 @@ package cache
 
 import (
 	"github.com/go-redis/redis/v8"
+
 	"github.com/instill-ai/pipeline-backend/configs"
 )
 

internal/db/db.go

@@ -5,9 +5,10 @@ import (
 	"sync"
 	"time"
 
-	configs "github.com/instill-ai/pipeline-backend/configs"
 	"gorm.io/driver/postgres"
 	"gorm.io/gorm"
+
+	configs "github.com/instill-ai/pipeline-backend/configs"
 )
 
 var db *gorm.DB

internal/db/migrations/migration.go

@@ -6,8 +6,10 @@ import (
 	"os"
 
 	"github.com/golang-migrate/migrate/v4"
+
 	_ "github.com/golang-migrate/migrate/v4/database/postgres"
 	_ "github.com/golang-migrate/migrate/v4/source/file"
+
 	"github.com/instill-ai/pipeline-backend/configs"
 )
 

internal/temporal/temporal.go

@@ -6,13 +6,15 @@ import (
 	"strconv"
 	"strings"
 
+	"go.temporal.io/sdk/client"
+
 	"github.com/instill-ai/pipeline-backend/configs"
 	"github.com/instill-ai/pipeline-backend/internal/definition"
 	"github.com/instill-ai/pipeline-backend/internal/logger"
 	"github.com/instill-ai/pipeline-backend/pkg/model"
-	worker "github.com/instill-ai/vdp/pkg/temporal"
 	"github.com/instill-ai/x/zapadapter"
-	"go.temporal.io/sdk/client"
+
+	worker "github.com/instill-ai/vdp/pkg/temporal"
 )
 
 var c client.Client

pkg/model/pipeline.go

@@ -36,7 +36,7 @@ type Pipeline struct {
 type ListPipelineQuery struct {
 	WithRecipe bool
 	Namespace  string
-	PageSize   int32
+	PageSize   uint64
 	Cursor     uint64
 }
 

pkg/model/recipe.go

@@ -24,7 +24,7 @@ type Destination struct {
 
 type Model struct {
 	Name    string `json:"model_name,omitempty"`
-	Version int32  `json:"version,omitempty"`
+	Version uint64 `json:"version,omitempty"`
 }
 
 type LogicOperator struct {

pkg/repository/pipeline.go

@@ -4,11 +4,12 @@ import (
 	"fmt"
 	"math"
 
-	"github.com/instill-ai/pipeline-backend/internal/logger"
-	"github.com/instill-ai/pipeline-backend/pkg/model"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 	"gorm.io/gorm"
+
+	"github.com/instill-ai/pipeline-backend/internal/logger"
+	"github.com/instill-ai/pipeline-backend/pkg/model"
 )
 
 type Operations interface {