feat: define message validations from protobuf

.circleci/config.yml

@@ -88,7 +88,7 @@ jobs:
 
   generate:
     docker:
-      - image: bertychat/protoc:v5
+      - image: bertychat/protoc:v8
     working_directory: /go/src/berty.tech
     environment:
       GOPATH: /go
@@ -107,7 +107,8 @@ jobs:
             cd core
             /tmp/retry -m 5 -t 300 sh -xec ' \
               git checkout . && \
-              make generate_prepare clean && \
+              make clean && \
+              make generate_prepare && \
               make generate_local && \
               git checkout go.mod && \
               git status | cat && \

.gitignore

@@ -3,6 +3,9 @@
 # core
 core/.generated
 core/.*.generated
+core/google/
+core/gogoproto/
+core/github.com/
 android/core/core-sources.jar
 gin-bin
 

.golangci.yml

@@ -5,6 +5,7 @@ run:
     - ".*\\.pb\\.go"
     - ".*\\.gen\\.go"
     - "core/sql/migrations/.*"
+    - "core/sql/migrations/.*/.*"
     - "client/react-native/gomobile/other.go"
 
 linters-settings:

build/docker/protoc/Dockerfile

@@ -1,21 +1,24 @@
 FROM moul/protoc-gen-gotemplate:latest as pgg
 
-FROM    golang:1.11-alpine as builder
+FROM    golang:1.11.4-alpine as builder
 
+# install deps
 RUN     apk --no-cache add make git go rsync libc-dev openssh docker
 
-RUN     go get -u -v \
-        github.com/gogo/protobuf/proto \
-        github.com/gogo/protobuf/protoc-gen-gogo \
-        github.com/gogo/protobuf/gogoproto \
-        github.com/gogo/protobuf/protoc-gen-gofast \
-        github.com/gogo/protobuf/jsonpb \
-        moul.io/protoc-gen-gotemplate \
-        golang.org/x/tools/cmd/goimports \
-        github.com/99designs/gqlgen \
-        github.com/vektah/gorunpkg \
-        github.com/spf13/cobra
+# install compilers
+RUN     go get -u github.com/99designs/gqlgen
+RUN     go get -u github.com/gogo/protobuf/gogoproto
+RUN     go get -u github.com/gogo/protobuf/jsonpb
+RUN     go get -u github.com/gogo/protobuf/proto
+RUN     go get -u github.com/gogo/protobuf/protoc-gen-gofast
+RUN     go get -u github.com/gogo/protobuf/protoc-gen-gogo
+RUN     go get -u github.com/lyft/protoc-gen-validate
+RUN     go get -u github.com/spf13/cobra
+RUN     go get -u github.com/vektah/gorunpkg
+RUN     go get -u golang.org/x/tools/cmd/goimports
+RUN     go get -u moul.io/protoc-gen-gotemplate
 
+# gqlgen
 RUN     cd /go/src/github.com/99designs/gqlgen && git checkout v0.6.0 && go install github.com/99designs/gqlgen
 RUN     rm -f /go/bin/web-editor
 

build/docker/protoc/Makefile

@@ -1,4 +1,4 @@
-IMAGE ?= bertychat/protoc:v5
+IMAGE ?= bertychat/protoc:v8
 
 build:
 	docker build --pull -t $(IMAGE) .

core/Makefile

@@ -1,20 +1,23 @@
 rwildcard = $(foreach d,$(wildcard $1*),$(call rwildcard,$d/,$2) $(filter $(subst *,%,$2),$d))
 NAME ?= berty
+# FIXME: create a everythingButVendor helper that runs a wildcard but ignores node_modules/ and vendor/, and simplify variables below
 GOPATH ?= $(HOME)/go
 BIN ?= $(GOPATH)/bin/berty
 BUILD_MODE ?= dev
-SOURCES = $(call rwildcard, $(CODE_PATHS), *.go)
+SOURCES = $(call rwildcard, ./, *.go)
 OUR_SOURCES = $(filter-out $(call rwildcard,vendor//, *.go),$(SOURCES))
-CODE_PATHS = $(filter-out node_modules/,$(wildcard */))
-PROTOS = $(call rwildcard, $(CODE_PATHS), *.proto)
+PROTOS = $(call rwildcard, ./, *.proto)
 OUR_PROTOS = $(filter-out $(call rwildcard,vendor//, *.proto),$(PROTOS))
 PROTOC_OPTS = --proto_path=/protobuf:./vendor/github.com/gogo/protobuf:vendor:.
 SERVICE_PROTOS = $(call rwildcard, api, *.proto)
 GENERATED_FILES = \
-									$(call rwildcard $(CODE_PATHS), *.gen.go) \
-									$(call rwildcard $(CODE_PATHS), *.pb.go) \
-									$(call rwildcard $(CODE_PATHS), *.gen.js) \
-									$(call rwildcard $(CODE_PATHS), *.gen.graphql)
+	$(call rwildcard, $(CODE_PATHS), *.validate.gen.go) \
+	$(call rwildcard, $(CODE_PATHS), *.service.gen.go) \
+	$(call rwildcard, $(CODE_PATHS), *generated.gen.go) \
+	$(call rwildcard, $(CODE_PATHS), *kind.gen.go) \
+	$(call rwildcard, $(CODE_PATHS), *.pb.go) \
+	$(call rwildcard, $(CODE_PATHS), *.gen.js) \
+	$(call rwildcard, $(CODE_PATHS), *.gen.graphql)
 GENERATION_TARGETS = \
 	.kind.generated \
 	.client.generated \
@@ -28,7 +31,8 @@ GIT_SHA ?= $(shell git rev-parse HEAD)
 GIT_TAG ?= $(shell git describe --tags --always)
 GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
 GIT_COMMIT_DATE ?= $(shell git show -s --format=%ct $(GIT_SHA))
-EXT_LDFLAGS ?= -ldflags "-X berty.tech/core.GitSha=$(GIT_SHA) -X berty.tech/core.GitTag=$(GIT_TAG) -X berty.tech/core.GitBranch=$(GIT_BRANCH) -X berty.tech/core.BuildMode=$(BUILD_MODE) -X berty.tech/core.commitDate=$(GIT_COMMIT_DATE)"
+GO_ROOT_PKG ?= berty.tech/core
+EXT_LDFLAGS ?= -ldflags "-X $(GO_ROOT_PKG).GitSha=$(GIT_SHA) -X $(GO_ROOT_PKG).GitTag=$(GIT_TAG) -X $(GO_ROOT_PKG).GitBranch=$(GIT_BRANCH) -X $(GO_ROOT_PKG).BuildMode=$(BUILD_MODE) -X $(GO_ROOT_PKG).commitDate=$(GIT_COMMIT_DATE)"
 RUN_DAEMON_OPTS ?= --log-level=debug --notification
 
 TEST_PATHS ?= ./...
@@ -101,19 +105,21 @@ generate: .generated
 	docker run \
 		--user="$(shell id -u)" \
 		--volume="$(PWD)/..:/go/src/berty.tech" \
-		--workdir="/go/src/berty.tech/core" \
+		--workdir="/go/src/$(GO_ROOT_PKG)" \
 		--entrypoint="sh" \
 		--rm \
-		bertychat/protoc:v5 \
+		bertychat/protoc:v8 \
 		-xec 'make IGNORE_GQLGEN=$(IGNORE_GQLGEN) generate_local'
 	touch $@
 
 .PHONY: clean
 clean:
 	rm -f .generated $(GENERATED_FILES) $(GENERATION_TARGETS) $(BIN)
+	rm -rf vendor
 
 .PHONY: generate_local
 generate_local: $(GENERATION_TARGETS)
+	goimports -w `find . -name "*.pb.go" -or -name "*.gen.go" | grep -v /vendor/`
 	gofmt -w -s `find . -name "*.pb.go" -or -name "*.gen.go" | grep -v /vendor/`
 
 .PHONY: generate_prepare
@@ -145,7 +151,10 @@ dev-deps:
 ##
 
 %.pb.go: %.proto
-	$(PROTOC) $(PROTOC_OPTS) --gofast_out=plugins=grpc:"$(GOPATH)/src" "$(dir $<)"/*.proto
+	$(PROTOC) $(PROTOC_OPTS) \
+		--gofast_out="plugins=grpc:$(GOPATH)/src" \
+		--gotemplate_out=debug=false,all=true,single-package-mode=false,template_dir=./pkg/validate:. \
+		"$(dir $<)"/*.proto
 
 # FIXME: use an automated rule that matches every client generated files
 .client.generated: $(PROTOS)
@@ -169,15 +178,18 @@ dev-deps:
 .gql.generated: $(PROTOS) .pb.generated
 	mkdir -p ./api/node/graphql
 	$(PROTOC) $(PROTOC_OPTS) --gotemplate_out=debug=false,all=true,template_dir=./api/node:./api/node ./api/node/service.proto
-	goimports -w ./api/node/*.gen.go ./api/node/graphql/*.gen.go
+	goimports -w `find api/node api -name '*.gen.go'`
+
 	@# gqlgen
 	cd ./api/node/graphql; test "$(IGNORE_GQLGEN)" = "1" || $(GQLGEN) -c gqlgen.gen.yml -v gen
 	test "$(IGNORE_GQLGEN)" = "1" || cp ./api/node/graphql/service.gen.graphql  ../client/react-native/common/schema.graphql
 	test "$(IGNORE_GQLGEN)" = "1" || prettier-eslint --trailing-comma all --single-quote --no-semi --write ./api/node/graphql/enums.gen.js
 	test "$(IGNORE_GQLGEN)" = "1" || cp ./api/node/graphql/enums.gen.js ../client/react-native/common/graphql/enums.gen.js
-	test "$(IGNORE_GQLGEN)" = "0" || git checkout api/node/graphql/models/generated.gen.go api/node/graphql/graph/generated/generated.gen.go
+	test "$(IGNORE_GQLGEN)" = "0" || git checkout api/node/graphql/models/generated.gen.go api/node/graphql/graph/generated/generated.gen.go || true
+
 	@# everything worked as expected
 	touch $@
 
 .pb.generated: $(patsubst %.proto,%.pb.go,$(OUR_PROTOS))
+	rm -rf google/ # remove directory generated by protoc-gen-gotemplate for proto deps
 	touch $@

core/api/client/jsonclient/{{.File.Package}}.{{.Service.Name|lower}}.gen.go.tmpl

@@ -9,7 +9,7 @@ import (
         "encoding/json"
         "berty.tech/core/{{.File.Name|dir}}"
         {{- range .File.Dependency -}}{{- $dependency := . | getProtoFile}}
-        "{{regexReplaceAll "\"([^;]*).*\"" ($dependency.GoPkg|string) "${1}" }}"
+        {{$dependency.GoPkg}}
         {{ end -}}
 	"berty.tech/core/api/client"
         "google.golang.org/grpc"

core/api/client/{{.File.Package}}.{{.Service.Name|lower}}.gen.go.tmpl

@@ -7,7 +7,7 @@ package client
 import (
        "context"
         {{- range .File.Dependency -}}{{- $dependency := . | getProtoFile}}
-        "{{regexReplaceAll "\"([^;]*).*\"" ($dependency.GoPkg|string) "${1}" }}"
+        {{$dependency.GoPkg}}
         {{ end -}}
 	"berty.tech/core/api/client"
 	"berty.tech/core/errorcodes"