Merge 1666773 into 4669d3b

backend/repository/client_update_test.go

@@ -51,7 +51,8 @@ func testUpdateClientConfig(t *testing.T, requestFile string, clientName string)
 	req := &ClientConfig{}
 	err := readJSONFile(requestFile, req)
 	assert.NoError(t, err, "Failed to unmarshal client config.")
-	tempDir, err := copyExample(t)
+	tempDir, err := copyExample("")
+	t.Logf("Temp dir is created at %s\n", tempDir)
 	if !assert.NoError(t, err, "Failed to copy example.") {
 		return
 	}
@@ -87,8 +88,8 @@ func testUpdateClientConfig(t *testing.T, requestFile string, clientName string)
 	testlib.CompareGoldenFile(t, productionJSONExpFile, productionJSON)
 }
 
-func copyExample(t *testing.T) (string, error) {
-	tempDir, err := ioutil.TempDir("", "zanzibar")
+func copyExample(localRoot string) (string, error) {
+	tempDir, err := ioutil.TempDir(localRoot, "zanzibar")
 	if err != nil {
 		return "", err
 	}
@@ -110,7 +111,6 @@ func copyExample(t *testing.T) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	t.Logf("Temp dir is created at %s\n", tempDir)
 	return tempExample, nil
 }
 

backend/repository/data/client_config_updated_expected.json

@@ -0,0 +1,30 @@
+{
+	"name": "baz",
+	"type": "tchannel",
+	"config": {
+		"thriftFile": "clients/baz/baz.thrift",
+		"thriftFileSha": "e0729d2985ec0aeb95891cba969f1fadecedb648",
+		"exposedMethods": {
+			"Call": "SimpleService::call",
+			"Compare": "SimpleService::compare",
+			"DeliberateDiffNoop": "SimpleService::sillyNoop",
+			"EchoBinary": "SecondService::echoBinary",
+			"EchoBool": "SecondService::echoBool",
+			"EchoDouble": "SecondService::echoDouble",
+			"EchoEnum": "SecondService::echoEnum",
+			"EchoI16": "SecondService::echoI16",
+			"EchoI32": "SecondService::echoI32",
+			"EchoI64": "SecondService::echoI64",
+			"EchoI8": "SecondService::echoI8",
+			"EchoString": "SecondService::echoString",
+			"EchoStringList": "SecondService::echoStringList",
+			"EchoStringMap": "SecondService::echoStringMap",
+			"EchoStringSet": "SecondService::echoStringSet",
+			"EchoStructList": "SecondService::echoStructList",
+			"EchoStructMap": "SecondService::echoStructMap",
+			"EchoStructSet": "SecondService::echoStructSet",
+			"EchoTypedef": "SecondService::echoTypedef",
+			"Ping": "SimpleService::ping"
+		}
+	}
+}

backend/repository/data/endpoint_config_updated_expected.json

@@ -0,0 +1,15 @@
+{
+	"endpointId": "baz",
+	"endpointType": "http",
+	"handleId": "compare",
+	"thriftFile": "endpoints/baz/baz.thrift",
+	"thriftFileSha": "v1",
+	"thriftMethodName": "SimpleService::compare",
+	"workflowType": "tchannelClient",
+	"clientID": "baz",
+	"clientMethod": "Compare",
+	"testFixtures": [],
+	"middlewares": [],
+	"reqHeaderMap": {},
+	"resHeaderMap": {}
+}

backend/repository/data/thrift_service_expected.json

@@ -0,0 +1,16 @@
+{
+	"GoogleNowService": {
+		"Name": "GoogleNowService",
+		"Path": "clients/googlenow/googlenow.thrift",
+		"Methods": [
+			{
+				"Name": "addCredentials",
+				"Type": "http"
+			},
+			{
+				"Name": "checkCredentials",
+				"Type": "http"
+			}
+		]
+	}
+}

backend/repository/endpoint_update_test.go

@@ -33,7 +33,8 @@ import (
 const endpointUpdateRequestDir = "../../examples/example-gateway/endpoints"
 
 func TestUpdateEndpoint(t *testing.T) {
-	tempDir, err := copyExample(t)
+	tempDir, err := copyExample("")
+	t.Logf("Temp dir is created at %s\n", tempDir)
 	if !assert.NoError(t, err, "Failed to copy example.") {
 		return
 	}

backend/repository/manager.go

@@ -0,0 +1,287 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package repository
+
+import (
+	"io/ioutil"
+	"path/filepath"
+
+	"go.uber.org/thriftrw/compile"
+
+	"github.com/pkg/errors"
+	"github.com/uber/zanzibar/codegen"
+)
+
+const (
+	runtimePrefix = "edge-gateways-runtime"
+)
+
+// Manager manages all remote repositories.
+type Manager struct {
+	// Maps from gateway ID to its repository.
+	// This map is created once and read-only afterwards.
+	RepoMap map[string]*Repository
+
+	// root directory for local copies of remote repositories.
+	localRootDir string
+
+	// IDL-registry repository.
+	idlRegistry IDLRegistry
+
+	// TODO: Add a Middleware schema repository
+}
+
+// NewManager creates a manager for remote git repositories.
+func NewManager(
+	repoMap map[string]*Repository,
+	localRoot string,
+	idlRegistry IDLRegistry,
+) *Manager {
+	manager := &Manager{
+		RepoMap:      repoMap,
+		localRootDir: localRoot,
+		idlRegistry:  idlRegistry,
+	}
+	return manager
+}
+
+// NewRuntimeRepository creates a repository for running Zanzibar with new configurations.
+func (m *Manager) NewRuntimeRepository(gatewayID string) (*Repository, error) {
+	r, ok := m.RepoMap[gatewayID]
+	if !ok {
+		return nil, errors.Errorf("gateway %q not found", gatewayID)
+	}
+	root, err := ioutil.TempDir(m.localRootDir, runtimePrefix)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create runtime directory")
+	}
+	return NewRepository(root, r.Remote(), r.fetcher, r.refreshInterval)
+}
+
+// IDLThriftService returns thrift services contained in a thrift file in IDL-registry.
+func (m *Manager) IDLThriftService(path string) (map[string]*ThriftService, error) {
+	if err := m.idlRegistry.Update(); err != nil {
+		return nil, err
+	}
+
+	localRootDir := m.idlRegistry.RootDir()
+	thriftRootDir := m.idlRegistry.ThriftRootDir()
+	packageHelper, err := codegen.NewPackageHelper(
+		"idl-registry",          // packgeRoot
+		localRootDir,            // configDirName
+		"",                      // middlewareConfig
+		thriftRootDir,           // thriftRootDir
+		"idl-registry/gen-code", // genCodePackage
+		"./build",               // targetGenDir
+		"",                      // copyrightHeader
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	// Parse service module as tchannel service.
+	thriftAbsPath := filepath.Join(localRootDir, thriftRootDir, path)
+	mspec, err := codegen.NewModuleSpec(thriftAbsPath, false, false, packageHelper)
+	if err != nil {
+		return nil, err
+	}
+	serviceType := TCHANNEL
+	// Parse HTTP annotations.
+	if _, err := codegen.NewModuleSpec(thriftAbsPath, true, false, packageHelper); err == nil {
+		serviceType = HTTP
+	}
+	serviceMap := map[string]*ThriftService{}
+	for _, service := range mspec.Services {
+		tservice := &ThriftService{
+			Name: service.Name,
+			Path: path,
+		}
+		tservice.Methods = make([]ThriftMethod, len(service.Methods))
+		for i, method := range service.Methods {
+			tservice.Methods[i].Name = method.Name
+			tservice.Methods[i].Type = serviceType
+		}
+		serviceMap[service.Name] = tservice
+	}
+	return serviceMap, nil
+}
+
+// ThriftList returns the full list of thrift files in a gateway.
+func (m *Manager) ThriftList(gateway string) (map[string]*ThriftMeta, error) {
+	repo, ok := m.RepoMap[gateway]
+	if !ok {
+		return nil, errors.Errorf("gateway %s is not found", gateway)
+	}
+	cfg, err := repo.GatewayConfig()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get gateway config")
+	}
+	return repo.ThriftConfig(cfg.ThriftRootDir)
+}
+
+// ThriftFile returns the content and meta data of a file in a gateway.
+func (m *Manager) ThriftFile(gateway, path string) (*ThriftMeta, error) {
+	repo, ok := m.RepoMap[gateway]
+	if !ok {
+		return nil, errors.Errorf("gateway %s is not found", gateway)
+	}
+	cfg, err := repo.GatewayConfig()
+	if err != nil {
+		return nil, err
+	}
+	thriftRootDir := cfg.ThriftRootDir
+	content, err := repo.ReadFile(filepath.Join(thriftRootDir, path))
+	if err != nil {
+		return nil, errors.Wrapf(err, "faile to read thrift file content: %s", path)
+	}
+	version, err := repo.ThriftFileVersion(path)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get thrift file version")
+	}
+	return &ThriftMeta{
+		Path:    path,
+		Version: version,
+		Content: string(content),
+	}, nil
+}
+
+// UpdateThriftFiles update thrift files to their master version in the IDL-registry.
+func (m *Manager) UpdateThriftFiles(r *Repository, paths []string) error {
+	if len(paths) == 0 {
+		return nil
+	}
+	newMeta, err := m.thriftMetaInIDLRegistry(paths)
+	if err != nil {
+		return errors.Wrap(err, "failed to get thrift files from IDL-registry")
+	}
+
+	if err := r.WriteThriftFileAndConfig(newMeta); err != nil {
+		return errors.Wrapf(err, "failed to update current thrift config")
+	}
+	return nil
+}
+
+// UpdateClients updates configurations for a list of clients.
+func (m *Manager) UpdateClients(r *Repository, clientCfgDir string, req []ClientConfig) error {
+	for i := range req {
+		thrift := req[i].ThriftFile
+		// Adds non-exsiting file into the repository.
+		version, versionErr := r.ThriftFileVersion(thrift)
+		if versionErr != nil {
+			if err := m.UpdateThriftFiles(r, []string{thrift}); err != nil {
+				return errors.Wrapf(err, "failed to add thrift file %s into temp repository", thrift)
+			}
+		}
+		if err := r.UpdateClientConfigs(&req[i], clientCfgDir, version); err != nil {
+			return errors.Wrapf(err, "failed to add thrift file %s into temp repository", thrift)
+		}
+	}
+	return nil
+}
+
+// UpdateEndpoints updates configurations for a list of endpoints.
+func (m *Manager) UpdateEndpoints(r *Repository, endpointCfgDir string, req []EndpointConfig) error {
+	for i := range req {
+		thrift := req[i].ThriftFile
+		// Adds non-exsiting file into the repository.
+		version, versionErr := r.ThriftFileVersion(thrift)
+		if versionErr != nil {
+			if err := m.UpdateThriftFiles(r, []string{thrift}); err != nil {
+				return errors.Wrapf(err, "failed to add thrift file %s into temp repository", thrift)
+			}
+		}
+		if err := r.WriteEndpointConfig(endpointCfgDir, &req[i], version); err != nil {
+			return errors.Wrapf(err, "failed to add thrift file %s into temp repository", thrift)
+		}
+	}
+	return nil
+}
+
+// UpdateRequest is the request to update thrift files, clients and endpoints.
+type UpdateRequest struct {
+	ThriftFiles     []string         `json:"thrift_files"`
+	ClientUpdates   []ClientConfig   `json:"client_updates"`
+	EndpointUpdates []EndpointConfig `json:"endpoint_updates"`
+}
+
+// UpdateAll writes the updates for thrift files, clients and endpoints.
+func (m *Manager) UpdateAll(r *Repository, clientCfgDir, endpointCfgDir string, req *UpdateRequest) error {
+	if err := m.UpdateThriftFiles(r, req.ThriftFiles); err != nil {
+		return errors.Wrap(err, "failed to update thrift files")
+	}
+	if err := m.UpdateClients(r, clientCfgDir, req.ClientUpdates); err != nil {
+		return errors.Wrap(err, "failed to update clients")
+	}
+	if err := m.UpdateEndpoints(r, endpointCfgDir, req.EndpointUpdates); err != nil {
+		return errors.Wrap(err, "failed to update endpoints")
+	}
+	return nil
+}
+
+// thriftMetaInIDLRegistry returns meta for a set of thrift file in IDL-registry.
+func (m *Manager) thriftMetaInIDLRegistry(paths []string) (map[string]*ThriftMeta, error) {
+	meta := make(map[string]*ThriftMeta, len(paths))
+	idlRootAbsPath := filepath.Join(m.idlRegistry.RootDir(), m.idlRegistry.ThriftRootDir())
+	for _, path := range paths {
+		module, err := compile.Compile(filepath.Join(idlRootAbsPath, path))
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to parse thrift file %s", path)
+		}
+		if err := addThriftDependencies(idlRootAbsPath, module, meta); err != nil {
+			return nil, errors.Wrapf(err, "failed to add dependencies for thrift file %s", path)
+		}
+	}
+	for path := range meta {
+		tm, err := m.idlRegistry.ThriftMeta(path, true)
+		if err != nil {
+			return nil, err
+		}
+		meta[path] = tm
+	}
+	return meta, nil
+}
+
+func addThriftDependencies(idlRoot string, module *compile.Module, meta map[string]*ThriftMeta) error {
+	relPath, err := filepath.Rel(idlRoot, module.ThriftPath)
+	if err != nil {
+		return errors.Wrapf(err, "failed to find relative path for thrift file %q under dir %q",
+			module.ThriftPath, idlRoot)
+	}
+	if _, ok := meta[relPath]; ok {
+		return nil
+	}
+	// Add the thrift file to meta.
+	meta[relPath] = nil
+	for _, includedModule := range module.Includes {
+		if err := addThriftDependencies(idlRoot, includedModule.Module, meta); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// ReadFile returns the content of a file in a relativePath in a repository.
+func (r *Repository) ReadFile(relativePath string) ([]byte, error) {
+	r.RLock()
+	defer r.RUnlock()
+	path := filepath.Join(r.localDir, relativePath)
+	return ioutil.ReadFile(path)
+}