OpenAPI 3.0 upgrade, swagger tool chain update

CLI/klish/clish_start

@@ -7,7 +7,7 @@ then
  export SYSTEM_NAME="${HOSTNAME%%.*}"
 fi
 
-export PYTHONPATH=/usr/sbin/cli:/usr/sbin/cli/scripts:/usr/sbin/:/usr/sbin/lib/swagger_client_py
+export PYTHONPATH=/usr/sbin/cli:/usr/sbin/cli/scripts:/usr/sbin
 export RENDERER_TEMPLATE_PATH=$SONIC_CLI_ROOT/render-templates
 export CLISH_PATH=$SONIC_CLI_ROOT/command-tree
 export LD_LIBRARY_PATH=/usr/local/lib:$SONIC_CLI_ROOT/.libs:$LD_LIBRARY_PATH

debian/sonic-mgmt-framework.install

@@ -1,5 +1,4 @@
 build/rest_server/rest_server       usr/sbin
 build/rest_server/generate_cert     usr/sbin
 build/rest_server/dist/ui           etc/rest_server
-build/swagger_client_py/*_client    usr/sbin/lib/swagger_client_py
 build/cli                           usr/sbin

go.sum

@@ -62,6 +62,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
+github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo=
+github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=

models/openapi_codegen.mk

@@ -22,54 +22,49 @@ TOPDIR := ..
 BUILD_DIR := $(TOPDIR)/build
 CODEGEN_TOOLS_DIR := $(TOPDIR)/tools/swagger_codegen
 
-CODEGEN_VER	:= 2.4.5
-CODEGEN_JAR := $(CODEGEN_TOOLS_DIR)/swagger-codegen-cli-$(CODEGEN_VER).jar
+CODEGEN_VER	:= 4.2.3
+CODEGEN_JAR := $(CODEGEN_TOOLS_DIR)/openapi-generator-cli-$(CODEGEN_VER).jar
 
 SERVER_BUILD_DIR	:= $(BUILD_DIR)/rest_server
 SERVER_CODEGEN_DIR	:= $(SERVER_BUILD_DIR)/codegen
 SERVER_DIST_DIR		:= $(SERVER_BUILD_DIR)/dist
 SERVER_DIST_INIT	:= $(SERVER_DIST_DIR)/.init_done
-SERVER_DIST_GO		:= $(SERVER_DIST_DIR)/swagger
+SERVER_DIST_GO		:= $(SERVER_DIST_DIR)/openapi
 SERVER_DIST_UI		:= $(SERVER_DIST_DIR)/ui
 SERVER_DIST_UI_HOME	:= $(SERVER_DIST_DIR)/ui/index.html
+RESTCONF_MD_INDEX	:= $(BUILD_DIR)/restconf_md/index.md
 
 # Load codegen preferences
 include codegen.config
 
 YANGAPI_DIR     := $(TOPDIR)/build/yaml
-YANGAPI_SPECS   := $(wildcard $(YANGAPI_DIR)/*.yaml)
+YANGAPI_SPECS   := $(shell find $(YANGAPI_DIR) -name '*.yaml' 2> /dev/null)
 YANGAPI_NAMES   := $(filter-out $(YANGAPI_EXCLUDES), $(basename $(notdir $(YANGAPI_SPECS))))
-YANGAPI_SERVERS := $(addsuffix /.yangapi_done, $(addprefix $(SERVER_CODEGEN_DIR)/, $(YANGAPI_NAMES)))
+YANGAPI_SERVERS := $(addsuffix /.yangapi_copy_done, $(addprefix $(SERVER_CODEGEN_DIR)/, $(YANGAPI_NAMES)))
 
 OPENAPI_DIR   := openapi
 OPENAPI_SPECS := $(shell find $(OPENAPI_DIR) -name '*.yaml' | sort)
 OPENAPI_NAMES := $(filter-out $(OPENAPI_EXCLUDES), $(basename $(notdir $(OPENAPI_SPECS))))
 OPENAPI_SERVERS := $(addsuffix /.openapi_done, $(addprefix $(SERVER_CODEGEN_DIR)/, $(OPENAPI_NAMES)))
 
-PY_YANGAPI_NAMES       := $(filter $(YANGAPI_NAMES), $(PY_YANGAPI_CLIENTS))
-PY_OPENAPI_NAMES       := $(filter $(OPENAPI_NAMES), $(PY_OPENAPI_CLIENTS))
-PY_CLIENT_CODEGEN_DIR  := $(BUILD_DIR)/swagger_client_py
-PY_CLIENT_TARGETS := $(addsuffix .yangapi_client_done, $(addprefix $(PY_CLIENT_CODEGEN_DIR)/, $(PY_YANGAPI_NAMES)))
-PY_CLIENT_TARGETS += $(addsuffix .openapi_client_done, $(addprefix $(PY_CLIENT_CODEGEN_DIR)/, $(PY_OPENAPI_NAMES)))
-
 UIGEN_DIR  = $(TOPDIR)/tools/ui_gen
 UIGEN_SRCS = $(shell find $(UIGEN_DIR) -type f)
 
 JAVA ?= java
 
-all: go-server py-client
+all: go-server
 
 go-server-init: $(SERVER_DIST_INIT)
 
-go-server: $(YANGAPI_SERVERS) $(OPENAPI_SERVERS) $(SERVER_DIST_INIT) $(SERVER_DIST_UI_HOME)
+go-server: $(YANGAPI_SERVERS) $(OPENAPI_SERVERS) $(SERVER_DIST_INIT) $(SERVER_DIST_UI_HOME) $(RESTCONF_MD_INDEX)
 
 $(SERVER_DIST_UI_HOME): $(YANGAPI_SERVERS) $(OPENAPI_SERVERS) $(UIGEN_SRCS)
 	@echo "+++ Generating  landing page for Swagger UI +++"
 	$(UIGEN_DIR)/src/uigen.py
 
-py-client: $(PY_CLIENT_TARGETS) | $(PY_CLIENT_CODEGEN_DIR)/.
-	@echo $(basename $(^F)) > $(PY_CLIENT_CODEGEN_DIR)/py_client
-
+$(RESTCONF_MD_INDEX): $(YANGAPI_SERVERS) $(OPENAPI_SERVERS)
+	@echo "+++ Generating index page for RESTCONF documents +++"
+	$(TOPDIR)/tools/restconf_doc_tools/index.py --mdDir $(BUILD_DIR)/restconf_md
 
 .SECONDEXPANSION:
 
@@ -86,22 +81,14 @@ py-client: $(PY_CLIENT_TARGETS) | $(PY_CLIENT_CODEGEN_DIR)/.
 #======================================================================
 $(CODEGEN_JAR): | $$(@D)/.
 	cd $(@D) && \
-	wget https://repo1.maven.org/maven2/io/swagger/swagger-codegen-cli/$(CODEGEN_VER)/$(@F)
+	wget https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/$(CODEGEN_VER)/$(@F)
 
 #======================================================================
-# Generate swagger server in GO language for Yang generated OpenAPIs
-# specs.
+# copy yang specs
 #======================================================================
-%/.yangapi_done: $(YANGAPI_DIR)/$$(*F).yaml | $$(@D)/. $(CODEGEN_JAR) $(SERVER_DIST_INIT)
-	@echo "+++ Generating GO server for Yang API $$(basename $(@D)).yaml +++"
-	$(JAVA) -jar $(CODEGEN_JAR) generate \
-		--lang go-server \
-		--input-spec $(YANGAPI_DIR)/$$(basename $(@D)).yaml \
-		--template-dir $(CODEGEN_TOOLS_DIR)/go-server/templates-yang \
-		--output $(@D)
-	cp $(@D)/go/api_* $(SERVER_DIST_GO)/
-	cp $(@D)/go/routers.go $(SERVER_DIST_GO)/routers_$$(basename $(@D)).go
-	cp $(@D)/api/swagger.yaml $(SERVER_DIST_UI)/$$(basename $(@D)).yaml
+%/.yangapi_copy_done: $(YANGAPI_DIR)/$$(*F).yaml | $$(@D)/. $(CODEGEN_JAR) $(SERVER_DIST_INIT)
+	@echo "+++ Copying $$(basename $(@D)).yaml +++"
+	cp $(YANGAPI_DIR)/$$(basename $(@D)).yaml $(SERVER_DIST_UI)/$$(basename $(@D)).yaml
 	touch $@
 
 #======================================================================
@@ -110,13 +97,14 @@ $(CODEGEN_JAR): | $$(@D)/.
 %/.openapi_done: $(OPENAPI_DIR)/$$(*F).yaml | $$(@D)/. $(CODEGEN_JAR) $(SERVER_DIST_INIT)
 	@echo "+++ Generating GO server for OpenAPI $$(basename $(@D)).yaml +++"
 	$(JAVA) -jar $(CODEGEN_JAR) generate \
-		--lang go-server \
+		-g go-server \
 		--input-spec $(OPENAPI_DIR)/$$(basename $(@D)).yaml \
 		--template-dir $(CODEGEN_TOOLS_DIR)/go-server/templates-nonyang \
 		--output $(@D)
+	rm -rf  $(@D)/go/api_*service.go
 	cp $(@D)/go/api_* $(@D)/go/model_* $(SERVER_DIST_GO)/
 	cp $(@D)/go/routers.go $(SERVER_DIST_GO)/routers_$$(basename $(@D)).go
-	cp $(@D)/api/swagger.yaml $(SERVER_DIST_UI)/$$(basename $(@D)).yaml
+	cp $(@D)/api/openapi.yaml $(SERVER_DIST_UI)/$$(basename $(@D)).yaml
 	touch $@
 
 #======================================================================
@@ -127,44 +115,14 @@ $(SERVER_DIST_INIT): | $$(@D)/.
 	cp -r $(CODEGEN_TOOLS_DIR)/go-server/src/* $(@D)/
 	touch $@
 
-#======================================================================
-# Generate swagger client in Python for yang generated OpenAPI specs
-#======================================================================
-%.yangapi_client_done: $(YANGAPI_DIR)/$$(*F).yaml | $(CODEGEN_JAR) $$(@D)/.
-	@echo "+++++ Generating Python client for $(*F).yaml +++++"
-	$(JAVA) -jar $(CODEGEN_JAR) generate \
-		-DpackageName=$(subst -,_,$(*F))_client \
-		--lang python \
-		--input-spec $(YANGAPI_DIR)/$(*F).yaml \
-		--template-dir $(CODEGEN_TOOLS_DIR)/py-client/templates \
-		--output $(@D)
-	touch $@
-
-#======================================================================
-# Generate swagger client in Python for handcoded OpenAPI specs
-#======================================================================
-%.openapi_client_done: $(OPENAPI_DIR)/$$(*F).yaml | $(CODEGEN_JAR) $$(@D)/.
-	@echo "+++++ Generating Python client for $(*F).yaml +++++"
-	$(JAVA) -jar $(CODEGEN_JAR) generate \
-		-DpackageName=$(subst -,_,$(*F))_client \
-		--lang python \
-		--input-spec $(OPENAPI_DIR)/$(*F).yaml \
-		--template-dir $(CODEGEN_TOOLS_DIR)/py-client/templates \
-		--output $(@D)
-	touch $@
-
 #======================================================================
 # Cleanups
 #======================================================================
 
-clean-server:
+clean:
 	$(RM) -r $(SERVER_DIST_DIR)
 	$(RM) -r $(SERVER_CODEGEN_DIR)
-
-clean-client:
-	$(RM) -r $(PY_CLIENT_CODEGEN_DIR)
-
-clean: clean-server clean-client
+	$(RM) $(SERVER_DIST_UI_HOME)
 
 cleanall: clean
 	$(RM) $(CODEGEN_JAR)

models/yang_to_openapi.mk

@@ -22,6 +22,8 @@ BUILD_DIR := $(TOPDIR)/build
 
 MGMT_COMMON_DIR             ?= $(TOPDIR)/../sonic-mgmt-common
 YANGAPI_DIR                 := $(BUILD_DIR)/yaml
+MD_DIR                      := $(TOPDIR)/build/restconf_md
+SERVER_DIST_DIR             := $(BUILD_DIR)/rest_server/dist/openapi
 YANGDIR                     := $(or $(wildcard $(MGMT_COMMON_DIR)/build/yang), $(MGMT_COMMON_DIR)/models/yang)
 YANGDIR_COMMON              := $(YANGDIR)/common
 YANGDIR_EXTENSIONS          := $(YANGDIR)/extensions
@@ -38,25 +40,35 @@ TOOLS_DIR        := $(TOPDIR)/tools
 PYANG_PLUGIN_DIR := $(TOOLS_DIR)/pyang/pyang_plugins
 PYANG ?= pyang
 
-OPENAPI_GEN_PRE  := $(YANGAPI_DIR)/.
+OPENAPI_GEN_PRE  := $(YANGAPI_DIR)/. $(MD_DIR)/. $(YANGAPI_DIR)/.openapi_gen_ut
 
-all: $(YANGAPI_DIR)/.done $(YANGAPI_DIR)/.sonic_done
+all: $(YANGAPI_DIR)/.openapi_gen_ut $(YANGAPI_DIR)/.done $(YANGAPI_DIR)/.sonic_done
 
 .PRECIOUS: %/.
 %/.:
 	mkdir -p $@
 
+#======================================================================
+# Unit tests for OpenAPI generator 
+#======================================================================
+$(YANGAPI_DIR)/.openapi_gen_ut: $(PYANG_PLUGIN_DIR)/openapi.py | $(YANGAPI_DIR)/.
+	$(MAKE) -C $(TOOLS_DIR)/openapi_tests
+	touch $@
+
 #======================================================================
 # Generate YAML files for Yang modules
 #======================================================================
-$(YANGAPI_DIR)/.done:  $(YANG_MOD_FILES) $(YANG_COMMON_FILES) | $(OPENAPI_GEN_PRE)
+$(YANGAPI_DIR)/.done: $(YANG_MOD_FILES) $(YANG_COMMON_FILES) | $(OPENAPI_GEN_PRE)
 	@echo "+++++ Generating YAML files for Yang modules +++++"
-	mkdir -p $(YANGAPI_DIR)
 	$(PYANG) \
 		-f swaggerapi \
-		--outdir $(@D) \
+		--outdir $(YANGAPI_DIR) \
 		--plugindir $(PYANG_PLUGIN_DIR) \
-		-p $(YANGDIR_COMMON):$(YANGDIR) \
+		--with-md-doc \
+		--md-outdir $(MD_DIR) \
+		--with-serverstub \
+		--stub-outdir $(SERVER_DIST_DIR) \
+		-p $(YANGDIR_COMMON):$(YANGDIR):$(YANGDIR_EXTENSIONS) \
 		$(YANG_MOD_FILES)
 	@echo "+++++ Generation of  YAML files for Yang modules completed +++++"
 	touch $@
@@ -68,9 +80,13 @@ $(YANGAPI_DIR)/.sonic_done: $(SONIC_YANG_MOD_FILES) $(SONIC_YANG_COMMON_FILES) |
 	@echo "+++++ Generating YAML files for Sonic Yang modules +++++"
 	$(PYANG) \
 		-f swaggerapi \
-		--outdir $(@D) \
+		--with-md-doc \
+		--outdir $(YANGAPI_DIR) \
+		--md-outdir $(MD_DIR) \
 		--plugindir $(PYANG_PLUGIN_DIR) \
-		-p $(YANGDIR_SONIC_COMMON):$(YANGDIR_SONIC):$(YANGDIR_COMMON) \
+		--with-serverstub \
+		--stub-outdir $(SERVER_DIST_DIR) \
+		-p $(YANGDIR_COMMON):$(YANGDIR_SONIC_COMMON):$(YANGDIR_SONIC) \
 		$(SONIC_YANG_MOD_FILES)
 	@echo "+++++ Generation of  YAML files for Sonic Yang modules completed +++++"
 	touch $@
@@ -81,4 +97,7 @@ $(YANGAPI_DIR)/.sonic_done: $(SONIC_YANG_MOD_FILES) $(SONIC_YANG_COMMON_FILES) |
 
 clean:
 	$(RM) -r $(YANGAPI_DIR)
+	$(RM) -r $(MD_DIR)
+
+cleanall: clean
 

rest/main/main.go

@@ -34,7 +34,7 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/Azure/sonic-mgmt-framework/build/rest_server/dist/swagger"
+	"github.com/Azure/sonic-mgmt-framework/build/rest_server/dist/openapi"
 	"github.com/Azure/sonic-mgmt-framework/rest/server"
 	"github.com/golang/glog"
 	"github.com/pkg/profile"
@@ -85,7 +85,7 @@ func main() {
 		prof.Stop()
 	}()
 
-	swagger.Load()
+	openapi.Load()
 
 	rtrConfig := server.RouterConfig{}
 	if clientAuth == "user" {

tools/codegen/go-server/src/openapi/routes.go

@@ -0,0 +1,24 @@
+////////////////////////////////////////////////////////////////////////////////
+//                                                                            //
+//  Copyright 2019 Broadcom. The term Broadcom refers to Broadcom Inc. and/or //
+//  its subsidiaries.                                                         //
+//                                                                            //
+//  Licensed under the Apache License, Version 2.0 (the "License");           //
+//  you may not use this file except in compliance with the License.          //
+//  You may obtain a copy of the License at                                   //
+//                                                                            //
+//     http://www.apache.org/licenses/LICENSE-2.0                             //
+//                                                                            //
+//  Unless required by applicable law or agreed to in writing, software       //
+//  distributed under the License is distributed on an "AS IS" BASIS,         //
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  //
+//  See the License for the specific language governing permissions and       //
+//  limitations under the License.                                            //
+//                                                                            //
+////////////////////////////////////////////////////////////////////////////////
+
+package openapi
+
+// Load function loads OpenAPI generated routes into REST server.
+func Load() {
+}

tools/codegen/go-server/templates-yang/controllers-api.j2

@@ -0,0 +1,41 @@
+
+package openapi
+
+import (
+	"net/http"
+
+	"github.com/Azure/sonic-mgmt-framework/rest/server"
+)
+
+{% for operationId in OpIds %}
+{% set path = OpIdDict[operationId]["path"] %}
+{% set pathEntry = OpIdDict[operationId]["obj"] %}
+{% set method = OpIdDict[operationId]["method"] %}
+func {{ operationId }}(w http.ResponseWriter, r *http.Request) {
+    rc, r := server.GetContext(r)
+    rc.Name = "{{ operationId }}"
+	{% if method in ["post", "put", "patch"] and "requestBody" in pathEntry %}
+	{% for consume in pathEntry["requestBody"]["content"].keys() %}
+    rc.Consumes.Add("{{ consume }}")
+	{% endfor %}
+	{% endif %}
+	{% set content = dict() %}
+	{% if method == "get" %}
+	{% set content = pathEntry["responses"]["200"]["content"] %}
+	{% endif %}
+	{% if 'x-rpc' in pathEntry and 'content' in pathEntry["responses"]["204"] %}
+	{% set content = pathEntry["responses"]["204"]["content"] %}
+	{% endif %}
+	{% for produce in content.keys() %}
+    rc.Produces.Add("{{ produce }}")
+	{% endfor %}
+	{% if 'x-params' in pathEntry.keys() %}
+	{% set varMappings = pathEntry['x-params']['varMapping'] %}
+    rc.PMap = server.NameMap{ {% for varMapping in varMappings %}"{{ varMapping['uriName'] }}":"{{ varMapping['yangName'] }}", {% endfor %} }
+	{% endif %}
+	server.Process(w, r)
+}
+{% if not loop.last %}
+
+{% endif %}
+{% endfor %}

tools/codegen/go-server/templates-yang/routers.j2

@@ -0,0 +1,22 @@
+
+package openapi
+
+import (
+	"github.com/Azure/sonic-mgmt-framework/rest/server"
+)
+
+func init() {
+
+{% for operationId in OpIds %}
+{% set path = OpIdDict[operationId]["path"] %}
+{% set pathEntry = OpIdDict[operationId]["obj"] %}
+{% set method = OpIdDict[operationId]["method"] %}
+	server.AddRoute(
+		"{{ operationId }}",
+		"{{ method|capitalize }}",
+		"{{ path }}",
+		{{ operationId }},
+	)
+
+{% endfor %}
+}

tools/openapi_tests/Makefile

@@ -0,0 +1,29 @@
+.PHONY: all test-rpc test-data-nodes
+
+TOPDIR             := ../..
+PYANG_PLUGINS_DIR  := $(TOPDIR)/tools/pyang/pyang_plugins
+PYANG ?= pyang
+
+all: test-rpc test-data-nodes test-complex-model test-complex-model-no-oneof
+
+# This will cover RPC statements with only input, only output and with
+# both input and output
+test-rpc:
+	$(PYANG) -f swaggerapi --plugindir $(PYANG_PLUGINS_DIR) test-rpc.yang | diff test-rpc.yang.expect -
+
+# This will cover container, list, leaf and leaf-lists with both
+# simple and nested hierarchies.
+# Also this will cover data type testing such as leafref, enum and string with pattern
+# simple string, integer types, leaf with default values, mandatory statements etc.
+test-data-nodes:
+	$(PYANG) -f swaggerapi --plugindir $(PYANG_PLUGINS_DIR) test-data-nodes.yang | diff test-data-nodes.yang.expect -
+
+# This will cover some complex YANGs, with many nested hierarchies
+# Also with choice-case statements, Union data types, range with min,max etc
+# Test with one-oneof
+test-complex-model:
+	$(PYANG) -f swaggerapi --plugindir $(PYANG_PLUGINS_DIR) --with-oneof ietf-snmp.yang ietf-snmp-community.yang | diff ietf-snmp.yang.expect -
+
+# Test without one-oneof
+test-complex-model-no-oneof:
+	$(PYANG) -f swaggerapi --plugindir $(PYANG_PLUGINS_DIR) ietf-snmp.yang ietf-snmp-community.yang | diff ietf-snmp.no-oneof.yang.expect -