idl paths per module

README.md

@@ -237,7 +237,7 @@ import (
 
 	"github.com/uber/zanzibar/examples/example-gateway/build/endpoints/contacts/module"
 	"github.com/uber/zanzibar/examples/example-gateway/build/endpoints/contacts/workflow"
-	contacts "github.com/uber/zanzibar/examples/example-gateway/build/gen-code/endpoints/contacts/contacts"
+	contacts "github.com/uber/zanzibar/examples/example-gateway/build/gen-code/endpoints-idl/endpoints/contacts/contacts"
 
 	zanzibar "github.com/uber/zanzibar/runtime"
 	"go.uber.org/zap"
@@ -425,7 +425,7 @@ package fixture
 
 import (
 	mc "github.com/uber/zanzibar/examples/example-gateway/build/clients/contacts/mock-client"
-	gen "github.com/uber/zanzibar/examples/example-gateway/build/gen-code/clients/contacts/contacts"
+	gen "github.com/uber/zanzibar/examples/example-gateway/build/gen-code/clients-idl/clients/contacts/contacts"
 )
 
 var saveContactsFixtures = &mc.SaveContactsScenarios{

benchmarks/benchserver/main.go

@@ -29,7 +29,7 @@ import (
 	"go.uber.org/zap/zapcore"
 
 	baz "github.com/uber/zanzibar/examples/example-gateway/build/clients/baz"
-	clientsBazBaz "github.com/uber/zanzibar/examples/example-gateway/build/gen-code/clients/baz/baz"
+	clientsBazBaz "github.com/uber/zanzibar/examples/example-gateway/build/gen-code/clients-idl/clients/baz/baz"
 	testBackend "github.com/uber/zanzibar/test/lib/test_backend"
 )
 

codegen/client.go

@@ -112,7 +112,7 @@ func newClientSpec(
 	h *PackageHelper,
 	annotate bool,
 ) (*ClientSpec, error) {
-	thriftFile := filepath.Join(h.ThriftIDLPath(), config.IDLFile)
+	thriftFile := filepath.Join(h.ThriftIDLPath(), h.GetModuleIdlSubDir(false), config.IDLFile)
 	mspec, err := NewModuleSpec(thriftFile, annotate, false, h)
 
 	if err != nil {
@@ -252,7 +252,7 @@ func newGRPCClientSpec(
 	instance *ModuleInstance,
 	h *PackageHelper,
 ) (*ClientSpec, error) {
-	protoFile := filepath.Join(h.ThriftIDLPath(), config.IDLFile)
+	protoFile := filepath.Join(h.ThriftIDLPath(), h.GetModuleIdlSubDir(false), config.IDLFile)
 	protoSpec, err := NewProtoModuleSpec(protoFile, false, h)
 	if err != nil {
 		return nil, errors.Wrapf(

codegen/client_test.go

@@ -434,6 +434,7 @@ func newTestPackageHelper(t *testing.T) *PackageHelper {
 		CopyrightHeader: "copyright",
 		GenCodePackage:  packageRoot + "/build/gen-code",
 		TraceKey:        "trace-key",
+		ModuleIdlSubDir: map[string]string{"endpoints": "endpoints-idl", "default": "clients-idl"},
 	}
 
 	h, err := NewPackageHelper(
@@ -488,7 +489,7 @@ func doNewClientSpecTest(t *testing.T, rawConfig []byte, clientType string) {
 	}
 	h := newTestPackageHelper(t)
 
-	idlFile := filepath.Join(h.ThriftIDLPath(), "clients/bar/bar.thrift")
+	idlFile := filepath.Join(h.ThriftIDLPath(), h.GetModuleIdlSubDir(false), "clients/bar/bar.thrift")
 	expectedSpec := &ClientSpec{
 		ModuleSpec:         nil,
 		YAMLFile:           instance.YAMLFileName,
@@ -537,7 +538,7 @@ func TestGRPCClientNewClientSpec(t *testing.T) {
 	}
 	h := newTestPackageHelper(t)
 
-	idlFile := filepath.Join(h.ThriftIDLPath(), "clients/echo/echo.proto")
+	idlFile := filepath.Join(h.ThriftIDLPath(), h.GetModuleIdlSubDir(false), "clients/echo/echo.proto")
 	expectedSpec := &ClientSpec{
 		ModuleSpec:         nil,
 		YAMLFile:           instance.YAMLFileName,

codegen/gateway.go

@@ -325,7 +325,7 @@ func NewEndpointSpec(
 	}
 
 	thriftFile := filepath.Join(
-		h.ThriftIDLPath(), endpointConfigObj["thriftFile"].(string),
+		h.ThriftIDLPath(), h.GetModuleIdlSubDir(true), endpointConfigObj["thriftFile"].(string),
 	)
 
 	mspec, err := NewModuleSpec(thriftFile, endpointType == "http", true, h)

codegen/package.go

@@ -21,6 +21,7 @@
 package codegen
 
 import (
+	"fmt"
 	"path"
 	"path/filepath"
 	"strings"
@@ -30,8 +31,10 @@ import (
 )
 
 const (
-	_thriftSuffix = ".thrift"
-	_protoSuffix  = ".proto"
+	_thriftSuffix          = ".thrift"
+	_protoSuffix           = ".proto"
+	_endpointModuleName    = "endpoints"
+	_defaultModuleFallback = "default"
 )
 
 // PackageHelper manages the mapping from thrift file to generated type code and service code.
@@ -42,6 +45,8 @@ type PackageHelper struct {
 	configRoot string
 	// The absolute root directory containing thrift files
 	thriftRootDir string
+	// moduleIdlSubDir defines subdir for idl per module
+	moduleIdlSubDir map[string]string
 	// The go package name of where all the generated structs are
 	genCodePackage string
 	// The absolute directory to put the generated service code
@@ -70,7 +75,7 @@ type PackageHelper struct {
 	defaultDependencies map[string][]string
 }
 
-//NewDefaultPackageHelperOptions returns a new default PackageHelperOptions, all optional fields are set as default.
+// NewDefaultPackageHelperOptions returns a new default PackageHelperOptions, all optional fields are set as default.
 func NewDefaultPackageHelperOptions() *PackageHelperOptions {
 	return &PackageHelperOptions{}
 }
@@ -80,6 +85,8 @@ func NewDefaultPackageHelperOptions() *PackageHelperOptions {
 type PackageHelperOptions struct {
 	// relative path to the idl dir, defaults to "./idl"
 	RelThriftRootDir string
+	// subdir for idl per module
+	ModuleIdlSubDir map[string]string
 	// relative path to the target dir that will contain generated code, defaults to "./build"
 	RelTargetGenDir string
 	// relative path to the middleware config dir, defaults to ""
@@ -123,6 +130,10 @@ func (p *PackageHelperOptions) relThriftRootDir() string {
 	return "./idl"
 }
 
+func (p *PackageHelperOptions) moduleIdlSubDir() map[string]string {
+	return p.ModuleIdlSubDir
+}
+
 func (p *PackageHelperOptions) relMiddlewareConfigDir() string {
 	return p.RelMiddlewareConfigDir
 }
@@ -199,6 +210,11 @@ func NewPackageHelper(
 		return nil, errors.Wrapf(err, "cannot load default middlewares")
 	}
 
+	moduleIdlSubDir := options.moduleIdlSubDir()
+	if len(moduleIdlSubDir) == 0 {
+		moduleIdlSubDir = map[string]string{_endpointModuleName: ".", _defaultModuleFallback: "."}
+	}
+
 	p := &PackageHelper{
 		packageRoot:            packageRoot,
 		configRoot:             absConfigRoot,
@@ -215,6 +231,7 @@ func NewPackageHelper(
 		moduleSearchPaths:      options.ModuleSearchPaths,
 		defaultDependencies:    options.DefaultDependencies,
 		defaultHeaders:         options.DefaultHeaders,
+		moduleIdlSubDir:        moduleIdlSubDir,
 	}
 	return p, nil
 }
@@ -328,6 +345,22 @@ func (p PackageHelper) getRelativeFileName(idlFile string) (string, error) {
 	return idlFile[idx+len(p.thriftRootDir):], nil
 }
 
+// GetModuleIdlSubDir returns subdir for idl per module
+func (p PackageHelper) GetModuleIdlSubDir(isEndpoint bool) string {
+	className := p.getModuleClass(isEndpoint)
+	if subDir, ok := p.moduleIdlSubDir[className]; ok {
+		return subDir
+	}
+	panic(fmt.Sprintf("unrecognized module %s", className))
+}
+
+func (p PackageHelper) getModuleClass(isEndpoint bool) string {
+	if isEndpoint {
+		return _endpointModuleName
+	}
+	return _defaultModuleFallback
+}
+
 // TargetClientsInitPath returns where the clients init should go
 func (p PackageHelper) TargetClientsInitPath() string {
 	return path.Join(p.targetGenDir, "clients", "clients.go")

codegen/package_test.go

@@ -36,7 +36,7 @@ var fooThrift = filepath.Join(
 	os.Getenv("GOPATH"),
 	"/src/github.com/uber/zanzibar/",
 	"examples/example-gateway/idl/",
-	"clients/foo/foo.thrift")
+	"clients-idl/clients/foo/foo.thrift")
 
 var testCopyrightHeader = `// Copyright (c) 2018 Uber Technologies, Inc.
 //
@@ -88,7 +88,8 @@ func TestImportPath(t *testing.T) {
 	h := newPackageHelper(t)
 	p, err := h.TypeImportPath(fooThrift)
 	assert.Nil(t, err, "should not return error")
-	assert.Equal(t, "github.com/uber/zanzibar/examples/example-gateway/build/gen-code/clients/foo/foo", p, "wrong type import path")
+	assert.Equal(t, "github.com/uber/zanzibar/examples/example-gateway/build/gen-code/clients-idl/clients/foo/foo",
+		p, "wrong type import path")
 	_, err = h.TypeImportPath("/Users/xxx/go/src/github.com/uber/zanzibar/examples/example-gateway/build/idl/github.com/uber/zanzibar/clients/foo/foo.go")
 	assert.Error(t, err, "should return error for not a thrift file")
 	_, err = h.TypeImportPath("/Users/xxx/go/src/github.com/uber/zanzibar/examples/example-gateway/build/zanzibar/clients/foo/foo.thrift")
@@ -99,7 +100,7 @@ func TestTypePackageName(t *testing.T) {
 	h := newPackageHelper(t)
 	packageName, err := h.TypePackageName(fooThrift)
 	assert.Nil(t, err, "should not return error")
-	assert.Equal(t, "clientsFooFoo", packageName, "wrong package name")
+	assert.Equal(t, "clientsIDlClientsFooFoo", packageName, "wrong package name")
 	_, err = h.TypeImportPath("/Users/xxx/go/src/github.com/uber/zanzibar/examples/example-gateway/build/idl/github.com/uber/zanzibar/clients/foo/foo.txt")
 	assert.Error(t, err, "should return error for not a thrift file")
 }

codegen/runner/pre-steps.sh

@@ -32,7 +32,10 @@ THRIFTRW_SRCS="$(echo "$THRIFTRW_SRCS" | xargs -n1 | sort | uniq)"
 
 DIRNAME="$(dirname "$0")"
 EASY_JSON_RAW_DIR="$DIRNAME/../../scripts/easy_json"
-EASY_JSON_DIR="$(cd "$EASY_JSON_RAW_DIR";pwd)"
+EASY_JSON_DIR="$(
+	cd "$EASY_JSON_RAW_DIR"
+	pwd
+)"
 EASY_JSON_FILE="$EASY_JSON_DIR/easy_json.go"
 EASY_JSON_BINARY="$EASY_JSON_DIR/easy_json"
 RESOLVE_THRIFT_FILE="$DIRNAME/../../scripts/resolve_thrift/main.go"
@@ -42,22 +45,28 @@ RESOLVE_I64_BINARY="$DIRNAME/../../scripts/resolve_i64/resolve_i64"
 
 if [[ -d "$DIRNAME/../../vendor" ]]; then
 	THRIFTRW_RAW_DIR="$DIRNAME/../../vendor/go.uber.org/thriftrw"
-	THRIFTRW_DIR="$(cd "$THRIFTRW_RAW_DIR";pwd)"
+	THRIFTRW_DIR="$(
+		cd "$THRIFTRW_RAW_DIR"
+		pwd
+	)"
 	THRIFTRW_MAIN_FILE="$THRIFTRW_DIR/main.go"
 	THRIFTRW_BINARY="$THRIFTRW_DIR/thriftrw"
 else
 	THRIFTRW_RAW_DIR="$DIRNAME/../../../../../go.uber.org/thriftrw"
-	THRIFTRW_DIR="$(cd "$THRIFTRW_RAW_DIR";pwd)"
+	THRIFTRW_DIR="$(
+		cd "$THRIFTRW_RAW_DIR"
+		pwd
+	)"
 	THRIFTRW_MAIN_FILE="$THRIFTRW_DIR/main.go"
 	THRIFTRW_BINARY="$THRIFTRW_DIR/thriftrw"
 fi
 
 start=$(date +%s)
-echo "$start" > .TMP_ZANZIBAR_TIMESTAMP_FILE.txt
+echo "$start" >.TMP_ZANZIBAR_TIMESTAMP_FILE.txt
 
 go build -o "$THRIFTRW_BINARY" "$THRIFTRW_MAIN_FILE"
 end=$(date +%s)
-runtime=$((end-start))
+runtime=$((end - start))
 echo "Compiled thriftrw : +$runtime"
 
 echo "Generating Go code from Thrift files"
@@ -87,15 +96,15 @@ Mgoogle/protobuf/wrappers.proto=github.com/gogo/protobuf/types,\
 echo "Generating Go code from Proto files"
 found_protos=$(find "$IDL_DIR" -name "*.proto")
 for proto_file in ${found_protos}; do
-  # We are not generating clients here, just (de)serializers.
-  proto_path="$ABS_IDL_DIR"
-  gencode_path="$ABS_GENCODE_DIR"
-  mkdir -p "$gencode_path"
-  proto_file="$PWD/$proto_file"
-  protoc --proto_path="$proto_path" \
-        --proto_path=./vendor/github.com/gogo/protobuf/protobuf \
-        --gogoslick_out="$GOGO_WKT_COMPATIBILITY":"$gencode_path" \
-        "$proto_file"
+	# We are not generating clients here, just (de)serializers.
+	proto_path="$ABS_IDL_DIR"
+	gencode_path="$ABS_GENCODE_DIR"
+	mkdir -p "$gencode_path"
+	proto_file="$PWD/$proto_file"
+	protoc --proto_path="$proto_path" \
+		--proto_path=./vendor/github.com/gogo/protobuf/protobuf \
+		--gogoslick_out="$GOGO_WKT_COMPATIBILITY":"$gencode_path" \
+		"$proto_file"
 done
 
 echo "Generating YARPC clients from Proto files"
@@ -125,23 +134,31 @@ for config_file in ${config_files}; do
 			proto_path="$ABS_IDL_DIR"
 			gencode_path="$ABS_GENCODE_DIR"
 			mkdir -p "$gencode_path"
-			proto_file="$ABS_IDL_DIR/$proto_file"
+			if [[ ${config_file} != *"/selective-gateway"* ]]; then
+				module_prefix="clients-idl"
+				if [[ ${config_file} == *"/endpoints"* ]]; then
+					module_prefix="endpoints-idl"
+				fi
+				proto_file="$ABS_IDL_DIR/$module_prefix/$proto_file"
+			else
+				proto_file="$ABS_IDL_DIR/$proto_file"
+			fi
 			protoc --proto_path="$proto_path" \
-            --proto_path=./vendor/github.com/gogo/protobuf/protobuf \
-            --gogoslick_out="$GOGO_WKT_COMPATIBILITY":"$gencode_path" \
-            --yarpc-go_out="$gencode_path" \
-            "$proto_file"
+				--proto_path=./vendor/github.com/gogo/protobuf/protobuf \
+				--gogoslick_out="$GOGO_WKT_COMPATIBILITY":"$gencode_path" \
+				--yarpc-go_out="$gencode_path" \
+				"$proto_file"
 		fi
 	done
 done
 
 end=$(date +%s)
-runtime=$((end-start))
+runtime=$((end - start))
 echo "Generated structs : +$runtime"
 
 go build -o "$EASY_JSON_BINARY" "$EASY_JSON_FILE"
 end=$(date +%s)
-runtime=$((end-start))
+runtime=$((end - start))
 echo "Compiled easyjson : +$runtime"
 
 go build -o "$RESOLVE_THRIFT_BINARY" "$RESOLVE_THRIFT_FILE"
@@ -178,13 +195,23 @@ for config_file in ${config_files}; do
 		[[ ${found_thrifts} == *${thrift_file}* ]] && continue
 		found_thrifts+=" $thrift_file"
 
-		thrift_file="$IDL_DIR/$thrift_file"
+		if [[ ${config_file} != *"/selective-gateway"* ]]; then
+			module_prefix="clients-idl"
+			#echo $config_file
+			if [[ ${config_file} == *"/endpoints"* ]]; then
+				module_prefix="endpoints-idl"
+			fi
+			thrift_file="$IDL_DIR/$module_prefix/$thrift_file"
+		else
+			thrift_file="$IDL_DIR/$thrift_file"
+		fi
+
 		gen_code_dir=$(
-		"$RESOLVE_THRIFT_BINARY" "$thrift_file" "$ANNOPREFIX" | \
-			sed "s|$ABS_IDL_DIR\/\(.*\)\/.*.thrift|$ABS_GENCODE_DIR/\1|" | \
-			sort | uniq | xargs
+			"$RESOLVE_THRIFT_BINARY" "$thrift_file" "$ANNOPREFIX" |
+				sed "s|$ABS_IDL_DIR\/\(.*\)\/.*.thrift|$ABS_GENCODE_DIR/\1|" |
+				sort | uniq | xargs
 		)
-		"$RESOLVE_I64_BINARY" "$thrift_file" "/idl/"  "json.type"
+		"$RESOLVE_I64_BINARY" "$thrift_file" "/idl/" "json.type"
 		target_dirs+=" $gen_code_dir"
 	done
 done
@@ -201,5 +228,5 @@ $(find "${target_dirs[@]}" -name "*.go" | \
 goimports -w "$BUILD_DIR/gen-code/"
 
 end=$(date +%s)
-runtime=$((end-start))
+runtime=$((end - start))
 echo "Generated structs : +$runtime"

codegen/runner/runner.go

@@ -122,8 +122,12 @@ func main() {
 		config.MustGetStruct("defaultHeaders", &defaultHeaders)
 	}
 
+	moduleIdlSubDir := map[string]string{}
+	config.MustGetStruct("moduleIdlSubDir", &moduleIdlSubDir)
+	fmt.Printf("moduleIdlSubDir paths %#v\n", moduleIdlSubDir)
 	options := &codegen.PackageHelperOptions{
 		RelThriftRootDir:              config.MustGetString("thriftRootDir"),
+		ModuleIdlSubDir:               moduleIdlSubDir,
 		RelTargetGenDir:               config.MustGetString("targetGenDir"),
 		RelMiddlewareConfigDir:        relMiddlewareConfigDir,
 		RelDefaultMiddlewareConfigDir: relDefaultMiddlewareConfigDir,