[orc8r] Create standalone specs at build time (#5400)

Signed-off-by: Andy Lee Khuu <andykhuu@stanford.edu>

cwf/cloud/go/go.sum

@@ -683,6 +683,7 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a h1:TwMENskLwU2NnWBzrJGEWHqSiGUkO/B4rfyhwqDxDYQ=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=

fbinternal/cloud/go/go.sum

@@ -712,6 +712,7 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a h1:TwMENskLwU2NnWBzrJGEWHqSiGUkO/B4rfyhwqDxDYQ=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=

feg/cloud/go/go.sum

@@ -709,6 +709,7 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a h1:TwMENskLwU2NnWBzrJGEWHqSiGUkO/B4rfyhwqDxDYQ=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=

lte/cloud/go/go.sum

@@ -699,6 +699,7 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a h1:TwMENskLwU2NnWBzrJGEWHqSiGUkO/B4rfyhwqDxDYQ=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=

orc8r/cloud/Makefile

@@ -27,13 +27,18 @@ SWAGGER_V1_ROOT := $(SWAGGER_ROOT)/v1
 SWAGGER_V1_YML := $(SWAGGER_ROOT)/v1/swagger.yml
 SWAGGER_V1_CONFIG := $(MAGMA_ROOT)/orc8r/cloud/docker/controller/goswagger-config.yml
 SWAGGER_V1_SPECS_DIR := $(MAGMA_ROOT)/orc8r/cloud/swagger/specs
-SWAGGER_V1_COMMON_DIR := common
+SWAGGER_V1_PARTIAL_SPECS_DIR := $(SWAGGER_V1_SPECS_DIR)/partial
+SWAGGER_V1_COMMON_DIR := $(SWAGGER_V1_SPECS_DIR)/common
+SWAGGER_V1_STANDALONE_DIR := $(SWAGGER_V1_SPECS_DIR)/standalone
 
 export SWAGGER_ROOT
 export SWAGGER_COMMON
 export SWAGGER_V1_ROOT
 export SWAGGER_V1_CONFIG
 export SWAGGER_V1_SPECS_DIR
+export SWAGGER_V1_COMMON_DIR
+export SWAGGER_V1_PARTIAL_SPECS_DIR
+export SWAGGER_V1_STANDALONE_DIR
 
 COVER_DIR := $(MAGMA_ROOT)/orc8r/cloud/coverage
 export COVER_DIR
@@ -132,13 +137,14 @@ lint_tools:
 ######################
 
 swagger: swagger_directories $(SWAGGER_LIST)
-	cp $(MAGMA_ROOT)/orc8r/cloud/go/models/swagger-common.yml $(SWAGGER_V1_SPECS_DIR)/$(SWAGGER_V1_COMMON_DIR)/$(SWAGGER_COMMON)
-	combine_swagger --in=$(SWAGGER_V1_SPECS_DIR) --common=$(SWAGGER_V1_SPECS_DIR)/$(SWAGGER_V1_COMMON_DIR)/$(SWAGGER_COMMON) --out=$(SWAGGER_V1_YML)
+	cp $(MAGMA_ROOT)/orc8r/cloud/go/models/swagger-common.yml $(SWAGGER_V1_COMMON_DIR)/$(SWAGGER_COMMON)
+	combine_swagger --in=$(SWAGGER_V1_PARTIAL_SPECS_DIR) --common=$(SWAGGER_V1_COMMON_DIR)/$(SWAGGER_COMMON) --out=$(SWAGGER_V1_YML)
 	swagger validate $(SWAGGER_V1_YML)
 
 swagger_directories:
 	rm -rf $(SWAGGER_V1_SPECS_DIR)
-	mkdir -p $(SWAGGER_V1_SPECS_DIR)/$(SWAGGER_V1_COMMON_DIR)
+	mkdir -p $(SWAGGER_V1_COMMON_DIR) $(SWAGGER_V1_PARTIAL_SPECS_DIR) $(SWAGGER_V1_STANDALONE_DIR)
+
 
 $(SWAGGER_LIST): %_swagger:
 	make -C $*/cloud/go swagger_tools copy_swagger_files

orc8r/cloud/go/models/swagger-common.yml

@@ -41,12 +41,6 @@ schemes:
   - https
   - http
 
-tags:
-  - name: Networks
-    description: Everything about networks
-  - name: Gateways
-    description: Provision and manage gateways
-
 paths:
   /foo:
     get:

orc8r/cloud/go/module.mk

@@ -43,9 +43,9 @@ gen::
 #
 # For example
 #	- Before: lte/cloud/go/services/policydb/obsidian/models/swagger.v1.yml
-#	- After: orc8r/cloud/swagger/specs/policydb.swagger.v1.yml
+#	- After: orc8r/cloud/swagger/specs/partial/policydb.swagger.v1.yml
 copy_swagger_files:
-	for f in $$(find . -name swagger.v1.yml) ; do cp $$f $${SWAGGER_V1_SPECS_DIR}/$$(echo $$f | sed -r 's/.*\/services\/([^\/]*)\/obsidian\/models\/(swagger\.v1\.yml)/\1.\2/g') ; done
+	for f in $$(find . -name swagger.v1.yml) ; do cp $$f $${SWAGGER_V1_PARTIAL_SPECS_DIR}/$$(echo $$f | sed -r 's/.*\/services\/([^\/]*)\/obsidian\/models\/(swagger\.v1\.yml)/\1.\2/g') ; done
 
 lint:
 	golangci-lint run

orc8r/cloud/go/obsidian/swagger/handlers/handlers.go

@@ -74,9 +74,9 @@ func GetCombinedSpecHandler(yamlCommon string) echo.HandlerFunc {
 	}
 }
 
-// GetSpecHandler returns a routing handler which creates and serves the
-// raw YAML spec of a particular service.
-func GetSpecHandler(yamlCommon string) echo.HandlerFunc {
+// GetSpecHandler returns a routing handler which serves a standalone raw YAML
+// spec of a particular service.
+func GetSpecHandler() echo.HandlerFunc {
 	return func(c echo.Context) error {
 		service, ok, err := getServiceName(c.Param("service"))
 		if err != nil {
@@ -86,12 +86,12 @@ func GetSpecHandler(yamlCommon string) echo.HandlerFunc {
 			return obsidian.HttpError(errors.New("service not found"), http.StatusNotFound)
 		}
 
-		combined, err := swagger.GetServiceSpec(yamlCommon, service)
+		yamlSpec, err := swagger.GetServiceSpec(service)
 		if err != nil {
 			return obsidian.HttpError(err, http.StatusInternalServerError)
 		}
 
-		return c.String(http.StatusOK, combined)
+		return c.String(http.StatusOK, yamlSpec)
 	}
 }
 
@@ -133,7 +133,7 @@ func registerSpecHandlers(e *echo.Echo, trailSlashMiddleware echo.MiddlewareFunc
 	}
 	e.GET(obsidian.StaticURLPrefix+"/v1/spec", nil, trailSlashMiddleware)
 
-	e.GET(obsidian.StaticURLPrefix+"/v1/spec/:service", GetSpecHandler(yamlCommon))
+	e.GET(obsidian.StaticURLPrefix+"/v1/spec/:service", GetSpecHandler())
 
 	return nil
 }

orc8r/cloud/go/obsidian/swagger/handlers/handlers_test.go

@@ -22,21 +22,21 @@ import (
 	"magma/orc8r/cloud/go/obsidian/swagger/protos"
 	"magma/orc8r/cloud/go/obsidian/tests"
 	"magma/orc8r/cloud/go/orc8r"
-	swagger_lib "magma/orc8r/cloud/go/swagger"
+	spec "magma/orc8r/cloud/go/swagger"
 	"magma/orc8r/cloud/go/test_utils"
 	"magma/orc8r/lib/go/registry"
 
 	"github.com/labstack/echo"
 	"github.com/stretchr/testify/assert"
 )
 
-func Test_GenerateCombinedSpecHandler(t *testing.T) {
+func Test_GetCombinedSpecHandler(t *testing.T) {
 	e := echo.New()
 	testURLRoot := "/magma/v1"
 
-	commonTag := swagger_lib.TagDefinition{Name: "Tag Common"}
-	commonSpec := swagger_lib.Spec{
-		Tags: []swagger_lib.TagDefinition{commonTag},
+	commonTag := spec.TagDefinition{Name: "Tag Common"}
+	commonSpec := spec.Spec{
+		Tags: []spec.TagDefinition{commonTag},
 	}
 	yamlCommon := marshalToYAML(t, commonSpec)
 
@@ -51,21 +51,21 @@ func Test_GenerateCombinedSpecHandler(t *testing.T) {
 	}
 	tests.RunUnitTest(t, e, tc)
 
-	tags := []swagger_lib.TagDefinition{
+	tags := []spec.TagDefinition{
 		{Name: "Tag 1"},
 		{Name: "Tag 2"},
 		{Name: "Tag 3"},
 	}
 	services := []string{"test_spec_service1", "test_spec_service2", "test_spec_service3"}
-	expectedSpec := swagger_lib.Spec{
-		Tags: []swagger_lib.TagDefinition{tags[0], tags[1], tags[2], commonTag},
+	expectedSpec := spec.Spec{
+		Tags: []spec.TagDefinition{tags[0], tags[1], tags[2], commonTag},
 	}
 
 	// Clean up registry
 	defer registry.RemoveServicesWithLabel(orc8r.SwaggerSpecLabel)
 
 	for i, s := range services {
-		registerServicer(t, s, tags[i])
+		registerServicer(t, s, tags[i], spec.TagDefinition{})
 	}
 
 	// Success with registered servicers
@@ -80,54 +80,48 @@ func Test_GenerateCombinedSpecHandler(t *testing.T) {
 	tests.RunUnitTest(t, e, tc)
 }
 
-func Test_GenerateSpecHandler(t *testing.T) {
+func Test_GetSpecHandler(t *testing.T) {
 	e := echo.New()
 	testURLRoot := "/magma/v1"
 
-	commonTag := swagger_lib.TagDefinition{Name: "Tag Common"}
-	commonSpec := swagger_lib.Spec{
-		Tags: []swagger_lib.TagDefinition{commonTag},
-	}
-	yamlCommon := marshalToYAML(t, commonSpec)
-
 	// Fail with invalid service name.
 	tc := tests.Test{
 		Method:         "GET",
 		URL:            testURLRoot,
 		Payload:        nil,
 		ParamNames:     []string{"service"},
 		ParamValues:    []string{"invalid_test_spec_service"},
-		Handler:        handlers.GetSpecHandler(yamlCommon),
+		Handler:        handlers.GetSpecHandler(),
 		ExpectedStatus: 404,
 		ExpectedError:  "service not found",
 	}
 	tests.RunUnitTest(t, e, tc)
 
 	// Success with valid service name.
-	tag := swagger_lib.TagDefinition{Name: "Tag 1"}
-	expected := swagger_lib.Spec{
-		Tags: []swagger_lib.TagDefinition{tag, commonTag},
+	tag := spec.TagDefinition{Name: "Tag 1"}
+	expected := spec.Spec{
+		Tags: []spec.TagDefinition{tag},
 	}
 
 	// Clean up registry
 	defer registry.RemoveServicesWithLabel(orc8r.SwaggerSpecLabel)
 
-	registerServicer(t, "test_spec_service1", tag)
+	registerServicer(t, "test_spec_service1", spec.TagDefinition{}, tag)
 
 	tc = tests.Test{
 		Method:         "GET",
 		URL:            testURLRoot,
 		Payload:        nil,
 		ParamNames:     []string{"service"},
 		ParamValues:    []string{"test_spec_service1"},
-		Handler:        handlers.GetSpecHandler(yamlCommon),
+		Handler:        handlers.GetSpecHandler(),
 		ExpectedStatus: 200,
 		ExpectedResult: expected,
 	}
 	tests.RunUnitTest(t, e, tc)
 }
 
-func Test_GenerateSpecUIHandler(t *testing.T) {
+func Test_GetUIHandler(t *testing.T) {
 	e := echo.New()
 	testURLRoot := "/magma/v1"
 
@@ -166,7 +160,7 @@ func Test_GenerateSpecUIHandler(t *testing.T) {
 	// Clean up registry
 	defer registry.RemoveServicesWithLabel(orc8r.SwaggerSpecLabel)
 
-	registerServicer(t, "test_spec_service2", swagger_lib.TagDefinition{})
+	registerServicer(t, "test_spec_service2", spec.TagDefinition{}, spec.TagDefinition{})
 
 	tc = tests.Test{
 		Method:         "GET",
@@ -181,21 +175,28 @@ func Test_GenerateSpecUIHandler(t *testing.T) {
 	tests.RunUnitTest(t, e, tc)
 }
 
-func registerServicer(t *testing.T, service string, tag swagger_lib.TagDefinition) {
+func registerServicer(
+	t *testing.T,
+	service string,
+	partialTag spec.TagDefinition,
+	standaloneTag spec.TagDefinition,
+) {
 	labels := map[string]string{
 		orc8r.SwaggerSpecLabel: "true",
 	}
 
 	srv, lis := test_utils.NewTestOrchestratorService(t, orc8r.ModuleName, service, labels, nil)
-	spec := swagger_lib.Spec{Tags: []swagger_lib.TagDefinition{tag}}
+	partialSpec := spec.Spec{Tags: []spec.TagDefinition{partialTag}}
+	standaloneSpec := spec.Spec{Tags: []spec.TagDefinition{standaloneTag}}
 
-	yamlSpec := marshalToYAML(t, spec)
-	protos.RegisterSwaggerSpecServer(srv.GrpcServer, swagger.NewSpecServicer(yamlSpec))
+	partialYamlSpec := marshalToYAML(t, partialSpec)
+	standaloneYamlSpec := marshalToYAML(t, standaloneSpec)
+	protos.RegisterSwaggerSpecServer(srv.GrpcServer, swagger.NewSpecServicer(partialYamlSpec, standaloneYamlSpec))
 
 	go srv.RunTest(lis)
 }
 
-func marshalToYAML(t *testing.T, spec swagger_lib.Spec) string {
+func marshalToYAML(t *testing.T, spec spec.Spec) string {
 	data, err := spec.MarshalBinary()
 	assert.NoError(t, err)
 	return string(data)

orc8r/cloud/go/obsidian/swagger/poll.go

@@ -36,9 +36,9 @@ func GetCombinedSpec(yamlCommon string) (string, error) {
 
 	var yamlSpecs []string
 	for _, s := range servicers {
-		yamlSpec, err := s.GetSpec()
+		yamlSpec, err := s.GetPartialSpec()
 		if err != nil {
-			// Swallow GetSpec error because the polling should continue
+			// Swallow error because the polling should continue
 			// even if it fails to receive from a single servicer
 			err = errors.Wrapf(err, "get Swagger spec from %s service", s.GetService())
 			glog.Error(err)
@@ -58,24 +58,15 @@ func GetCombinedSpec(yamlCommon string) (string, error) {
 	return combined, nil
 }
 
-// GetServiceSpec polls a service for its spec and combines it with the
-// common spec to return a combined spec.
-func GetServiceSpec(yamlCommon string, service string) (string, error) {
+// GetServiceSpec returns a service's standalone spec.
+func GetServiceSpec(service string) (string, error) {
 	remoteSpec := NewRemoteSpec(service)
-	yamlSpec, err := remoteSpec.GetSpec()
+	yamlSpec, err := remoteSpec.GetStandaloneSpec()
 	if err != nil {
 		return "", err
 	}
 
-	combined, warnings, err := spec.Combine(yamlCommon, []string{yamlSpec})
-	if err != nil {
-		return "", err
-	}
-	if warnings != nil {
-		glog.Infof("Some Swagger spec traits were overwritten or unable to be read: %+v", warnings)
-	}
-
-	return combined, nil
+	return yamlSpec, nil
 }
 
 // GetCommonSpec returns the Swagger common spec as a YAML string.