Skip to content

Commit

Permalink
OpenAPI 3.0 upgrade, swagger tool chain update
Browse files Browse the repository at this point in the history
- Added tests for openAPI spec generator.
- OpenAPI spec generator is enhanced to generate rest-server.
stubs, this replaces the OpenAPI-generator from commnity.
- Removed openAPI client generation.
- Added Restconf document generator.
- Upgraded specs to openAPI 3.0.

Co-authored-by: Sachin Holla sachin.holla@broadcom.com
  • Loading branch information
faraazbrcm committed Jan 12, 2024
1 parent ca0656c commit ab7fe0b
Show file tree
Hide file tree
Showing 44 changed files with 38,504 additions and 465 deletions.
2 changes: 1 addition & 1 deletion CLI/klish/clish_start
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ then
export SYSTEM_NAME="${HOSTNAME%%.*}"
fi

export PYTHONPATH=/usr/sbin/cli:/usr/sbin/cli/scripts:/usr/sbin/:/usr/sbin/lib/swagger_client_py
export PYTHONPATH=/usr/sbin/cli:/usr/sbin/cli/scripts:/usr/sbin
export RENDERER_TEMPLATE_PATH=$SONIC_CLI_ROOT/render-templates
export CLISH_PATH=$SONIC_CLI_ROOT/command-tree
export LD_LIBRARY_PATH=/usr/local/lib:$SONIC_CLI_ROOT/.libs:$LD_LIBRARY_PATH
Expand Down
1 change: 0 additions & 1 deletion debian/sonic-mgmt-framework.install
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
build/rest_server/rest_server usr/sbin
build/rest_server/generate_cert usr/sbin
build/rest_server/dist/ui etc/rest_server
build/swagger_client_py/*_client usr/sbin/lib/swagger_client_py
build/cli usr/sbin
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo=
github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
Expand Down
84 changes: 21 additions & 63 deletions models/openapi_codegen.mk
Original file line number Diff line number Diff line change
Expand Up @@ -22,54 +22,49 @@ TOPDIR := ..
BUILD_DIR := $(TOPDIR)/build
CODEGEN_TOOLS_DIR := $(TOPDIR)/tools/swagger_codegen

CODEGEN_VER := 2.4.5
CODEGEN_JAR := $(CODEGEN_TOOLS_DIR)/swagger-codegen-cli-$(CODEGEN_VER).jar
CODEGEN_VER := 4.2.3
CODEGEN_JAR := $(CODEGEN_TOOLS_DIR)/openapi-generator-cli-$(CODEGEN_VER).jar

SERVER_BUILD_DIR := $(BUILD_DIR)/rest_server
SERVER_CODEGEN_DIR := $(SERVER_BUILD_DIR)/codegen
SERVER_DIST_DIR := $(SERVER_BUILD_DIR)/dist
SERVER_DIST_INIT := $(SERVER_DIST_DIR)/.init_done
SERVER_DIST_GO := $(SERVER_DIST_DIR)/swagger
SERVER_DIST_GO := $(SERVER_DIST_DIR)/openapi
SERVER_DIST_UI := $(SERVER_DIST_DIR)/ui
SERVER_DIST_UI_HOME := $(SERVER_DIST_DIR)/ui/index.html
RESTCONF_MD_INDEX := $(BUILD_DIR)/restconf_md/index.md

# Load codegen preferences
include codegen.config

YANGAPI_DIR := $(TOPDIR)/build/yaml
YANGAPI_SPECS := $(wildcard $(YANGAPI_DIR)/*.yaml)
YANGAPI_SPECS := $(shell find $(YANGAPI_DIR) -name '*.yaml' 2> /dev/null)
YANGAPI_NAMES := $(filter-out $(YANGAPI_EXCLUDES), $(basename $(notdir $(YANGAPI_SPECS))))
YANGAPI_SERVERS := $(addsuffix /.yangapi_done, $(addprefix $(SERVER_CODEGEN_DIR)/, $(YANGAPI_NAMES)))
YANGAPI_SERVERS := $(addsuffix /.yangapi_copy_done, $(addprefix $(SERVER_CODEGEN_DIR)/, $(YANGAPI_NAMES)))

OPENAPI_DIR := openapi
OPENAPI_SPECS := $(shell find $(OPENAPI_DIR) -name '*.yaml' | sort)
OPENAPI_NAMES := $(filter-out $(OPENAPI_EXCLUDES), $(basename $(notdir $(OPENAPI_SPECS))))
OPENAPI_SERVERS := $(addsuffix /.openapi_done, $(addprefix $(SERVER_CODEGEN_DIR)/, $(OPENAPI_NAMES)))

PY_YANGAPI_NAMES := $(filter $(YANGAPI_NAMES), $(PY_YANGAPI_CLIENTS))
PY_OPENAPI_NAMES := $(filter $(OPENAPI_NAMES), $(PY_OPENAPI_CLIENTS))
PY_CLIENT_CODEGEN_DIR := $(BUILD_DIR)/swagger_client_py
PY_CLIENT_TARGETS := $(addsuffix .yangapi_client_done, $(addprefix $(PY_CLIENT_CODEGEN_DIR)/, $(PY_YANGAPI_NAMES)))
PY_CLIENT_TARGETS += $(addsuffix .openapi_client_done, $(addprefix $(PY_CLIENT_CODEGEN_DIR)/, $(PY_OPENAPI_NAMES)))

UIGEN_DIR = $(TOPDIR)/tools/ui_gen
UIGEN_SRCS = $(shell find $(UIGEN_DIR) -type f)

JAVA ?= java

all: go-server py-client
all: go-server

go-server-init: $(SERVER_DIST_INIT)

go-server: $(YANGAPI_SERVERS) $(OPENAPI_SERVERS) $(SERVER_DIST_INIT) $(SERVER_DIST_UI_HOME)
go-server: $(YANGAPI_SERVERS) $(OPENAPI_SERVERS) $(SERVER_DIST_INIT) $(SERVER_DIST_UI_HOME) $(RESTCONF_MD_INDEX)

$(SERVER_DIST_UI_HOME): $(YANGAPI_SERVERS) $(OPENAPI_SERVERS) $(UIGEN_SRCS)
@echo "+++ Generating landing page for Swagger UI +++"
$(UIGEN_DIR)/src/uigen.py

py-client: $(PY_CLIENT_TARGETS) | $(PY_CLIENT_CODEGEN_DIR)/.
@echo $(basename $(^F)) > $(PY_CLIENT_CODEGEN_DIR)/py_client

$(RESTCONF_MD_INDEX): $(YANGAPI_SERVERS) $(OPENAPI_SERVERS)
@echo "+++ Generating index page for RESTCONF documents +++"
$(TOPDIR)/tools/restconf_doc_tools/index.py --mdDir $(BUILD_DIR)/restconf_md

.SECONDEXPANSION:

Expand All @@ -86,22 +81,14 @@ py-client: $(PY_CLIENT_TARGETS) | $(PY_CLIENT_CODEGEN_DIR)/.
#======================================================================
$(CODEGEN_JAR): | $$(@D)/.
cd $(@D) && \
wget https://repo1.maven.org/maven2/io/swagger/swagger-codegen-cli/$(CODEGEN_VER)/$(@F)
wget https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/$(CODEGEN_VER)/$(@F)

#======================================================================
# Generate swagger server in GO language for Yang generated OpenAPIs
# specs.
# copy yang specs
#======================================================================
%/.yangapi_done: $(YANGAPI_DIR)/$$(*F).yaml | $$(@D)/. $(CODEGEN_JAR) $(SERVER_DIST_INIT)
@echo "+++ Generating GO server for Yang API $$(basename $(@D)).yaml +++"
$(JAVA) -jar $(CODEGEN_JAR) generate \
--lang go-server \
--input-spec $(YANGAPI_DIR)/$$(basename $(@D)).yaml \
--template-dir $(CODEGEN_TOOLS_DIR)/go-server/templates-yang \
--output $(@D)
cp $(@D)/go/api_* $(SERVER_DIST_GO)/
cp $(@D)/go/routers.go $(SERVER_DIST_GO)/routers_$$(basename $(@D)).go
cp $(@D)/api/swagger.yaml $(SERVER_DIST_UI)/$$(basename $(@D)).yaml
%/.yangapi_copy_done: $(YANGAPI_DIR)/$$(*F).yaml | $$(@D)/. $(CODEGEN_JAR) $(SERVER_DIST_INIT)
@echo "+++ Copying $$(basename $(@D)).yaml +++"
cp $(YANGAPI_DIR)/$$(basename $(@D)).yaml $(SERVER_DIST_UI)/$$(basename $(@D)).yaml
touch $@

#======================================================================
Expand All @@ -110,13 +97,14 @@ $(CODEGEN_JAR): | $$(@D)/.
%/.openapi_done: $(OPENAPI_DIR)/$$(*F).yaml | $$(@D)/. $(CODEGEN_JAR) $(SERVER_DIST_INIT)
@echo "+++ Generating GO server for OpenAPI $$(basename $(@D)).yaml +++"
$(JAVA) -jar $(CODEGEN_JAR) generate \
--lang go-server \
-g go-server \
--input-spec $(OPENAPI_DIR)/$$(basename $(@D)).yaml \
--template-dir $(CODEGEN_TOOLS_DIR)/go-server/templates-nonyang \
--output $(@D)
rm -rf $(@D)/go/api_*service.go
cp $(@D)/go/api_* $(@D)/go/model_* $(SERVER_DIST_GO)/
cp $(@D)/go/routers.go $(SERVER_DIST_GO)/routers_$$(basename $(@D)).go
cp $(@D)/api/swagger.yaml $(SERVER_DIST_UI)/$$(basename $(@D)).yaml
cp $(@D)/api/openapi.yaml $(SERVER_DIST_UI)/$$(basename $(@D)).yaml
touch $@

#======================================================================
Expand All @@ -127,44 +115,14 @@ $(SERVER_DIST_INIT): | $$(@D)/.
cp -r $(CODEGEN_TOOLS_DIR)/go-server/src/* $(@D)/
touch $@

#======================================================================
# Generate swagger client in Python for yang generated OpenAPI specs
#======================================================================
%.yangapi_client_done: $(YANGAPI_DIR)/$$(*F).yaml | $(CODEGEN_JAR) $$(@D)/.
@echo "+++++ Generating Python client for $(*F).yaml +++++"
$(JAVA) -jar $(CODEGEN_JAR) generate \
-DpackageName=$(subst -,_,$(*F))_client \
--lang python \
--input-spec $(YANGAPI_DIR)/$(*F).yaml \
--template-dir $(CODEGEN_TOOLS_DIR)/py-client/templates \
--output $(@D)
touch $@

#======================================================================
# Generate swagger client in Python for handcoded OpenAPI specs
#======================================================================
%.openapi_client_done: $(OPENAPI_DIR)/$$(*F).yaml | $(CODEGEN_JAR) $$(@D)/.
@echo "+++++ Generating Python client for $(*F).yaml +++++"
$(JAVA) -jar $(CODEGEN_JAR) generate \
-DpackageName=$(subst -,_,$(*F))_client \
--lang python \
--input-spec $(OPENAPI_DIR)/$(*F).yaml \
--template-dir $(CODEGEN_TOOLS_DIR)/py-client/templates \
--output $(@D)
touch $@

#======================================================================
# Cleanups
#======================================================================

clean-server:
clean:
$(RM) -r $(SERVER_DIST_DIR)
$(RM) -r $(SERVER_CODEGEN_DIR)

clean-client:
$(RM) -r $(PY_CLIENT_CODEGEN_DIR)

clean: clean-server clean-client
$(RM) $(SERVER_DIST_UI_HOME)

cleanall: clean
$(RM) $(CODEGEN_JAR)
Expand Down
35 changes: 27 additions & 8 deletions models/yang_to_openapi.mk
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ BUILD_DIR := $(TOPDIR)/build

MGMT_COMMON_DIR ?= $(TOPDIR)/../sonic-mgmt-common
YANGAPI_DIR := $(BUILD_DIR)/yaml
MD_DIR := $(TOPDIR)/build/restconf_md
SERVER_DIST_DIR := $(BUILD_DIR)/rest_server/dist/openapi
YANGDIR := $(or $(wildcard $(MGMT_COMMON_DIR)/build/yang), $(MGMT_COMMON_DIR)/models/yang)
YANGDIR_COMMON := $(YANGDIR)/common
YANGDIR_EXTENSIONS := $(YANGDIR)/extensions
Expand All @@ -38,25 +40,35 @@ TOOLS_DIR := $(TOPDIR)/tools
PYANG_PLUGIN_DIR := $(TOOLS_DIR)/pyang/pyang_plugins
PYANG ?= pyang

OPENAPI_GEN_PRE := $(YANGAPI_DIR)/.
OPENAPI_GEN_PRE := $(YANGAPI_DIR)/. $(MD_DIR)/. $(YANGAPI_DIR)/.openapi_gen_ut

all: $(YANGAPI_DIR)/.done $(YANGAPI_DIR)/.sonic_done
all: $(YANGAPI_DIR)/.openapi_gen_ut $(YANGAPI_DIR)/.done $(YANGAPI_DIR)/.sonic_done

.PRECIOUS: %/.
%/.:
mkdir -p $@

#======================================================================
# Unit tests for OpenAPI generator
#======================================================================
$(YANGAPI_DIR)/.openapi_gen_ut: $(PYANG_PLUGIN_DIR)/openapi.py | $(YANGAPI_DIR)/.
$(MAKE) -C $(TOOLS_DIR)/openapi_tests
touch $@

#======================================================================
# Generate YAML files for Yang modules
#======================================================================
$(YANGAPI_DIR)/.done: $(YANG_MOD_FILES) $(YANG_COMMON_FILES) | $(OPENAPI_GEN_PRE)
$(YANGAPI_DIR)/.done: $(YANG_MOD_FILES) $(YANG_COMMON_FILES) | $(OPENAPI_GEN_PRE)
@echo "+++++ Generating YAML files for Yang modules +++++"
mkdir -p $(YANGAPI_DIR)
$(PYANG) \
-f swaggerapi \
--outdir $(@D) \
--outdir $(YANGAPI_DIR) \
--plugindir $(PYANG_PLUGIN_DIR) \
-p $(YANGDIR_COMMON):$(YANGDIR) \
--with-md-doc \
--md-outdir $(MD_DIR) \
--with-serverstub \
--stub-outdir $(SERVER_DIST_DIR) \
-p $(YANGDIR_COMMON):$(YANGDIR):$(YANGDIR_EXTENSIONS) \
$(YANG_MOD_FILES)
@echo "+++++ Generation of YAML files for Yang modules completed +++++"
touch $@
Expand All @@ -68,9 +80,13 @@ $(YANGAPI_DIR)/.sonic_done: $(SONIC_YANG_MOD_FILES) $(SONIC_YANG_COMMON_FILES) |
@echo "+++++ Generating YAML files for Sonic Yang modules +++++"
$(PYANG) \
-f swaggerapi \
--outdir $(@D) \
--with-md-doc \
--outdir $(YANGAPI_DIR) \
--md-outdir $(MD_DIR) \
--plugindir $(PYANG_PLUGIN_DIR) \
-p $(YANGDIR_SONIC_COMMON):$(YANGDIR_SONIC):$(YANGDIR_COMMON) \
--with-serverstub \
--stub-outdir $(SERVER_DIST_DIR) \
-p $(YANGDIR_COMMON):$(YANGDIR_SONIC_COMMON):$(YANGDIR_SONIC) \
$(SONIC_YANG_MOD_FILES)
@echo "+++++ Generation of YAML files for Sonic Yang modules completed +++++"
touch $@
Expand All @@ -81,4 +97,7 @@ $(YANGAPI_DIR)/.sonic_done: $(SONIC_YANG_MOD_FILES) $(SONIC_YANG_COMMON_FILES) |

clean:
$(RM) -r $(YANGAPI_DIR)
$(RM) -r $(MD_DIR)

cleanall: clean

4 changes: 2 additions & 2 deletions rest/main/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import (
"syscall"
"time"

"github.com/Azure/sonic-mgmt-framework/build/rest_server/dist/swagger"
"github.com/Azure/sonic-mgmt-framework/build/rest_server/dist/openapi"
"github.com/Azure/sonic-mgmt-framework/rest/server"
"github.com/golang/glog"
"github.com/pkg/profile"
Expand Down Expand Up @@ -85,7 +85,7 @@ func main() {
prof.Stop()
}()

swagger.Load()
openapi.Load()

rtrConfig := server.RouterConfig{}
if clientAuth == "user" {
Expand Down
Empty file removed tools/.gitkeep
Empty file.
24 changes: 24 additions & 0 deletions tools/codegen/go-server/src/openapi/routes.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
////////////////////////////////////////////////////////////////////////////////
// //
// Copyright 2019 Broadcom. The term Broadcom refers to Broadcom Inc. and/or //
// its subsidiaries. //
// //
// Licensed under the Apache License, Version 2.0 (the "License"); //
// you may not use this file except in compliance with the License. //
// You may obtain a copy of the License at //
// //
// http://www.apache.org/licenses/LICENSE-2.0 //
// //
// Unless required by applicable law or agreed to in writing, software //
// distributed under the License is distributed on an "AS IS" BASIS, //
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. //
// See the License for the specific language governing permissions and //
// limitations under the License. //
// //
////////////////////////////////////////////////////////////////////////////////

package openapi

// Load function loads OpenAPI generated routes into REST server.
func Load() {
}
41 changes: 41 additions & 0 deletions tools/codegen/go-server/templates-yang/controllers-api.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@

package openapi

import (
"net/http"

"github.com/Azure/sonic-mgmt-framework/rest/server"
)

{% for operationId in OpIds %}
{% set path = OpIdDict[operationId]["path"] %}
{% set pathEntry = OpIdDict[operationId]["obj"] %}
{% set method = OpIdDict[operationId]["method"] %}
func {{ operationId }}(w http.ResponseWriter, r *http.Request) {
rc, r := server.GetContext(r)
rc.Name = "{{ operationId }}"
{% if method in ["post", "put", "patch"] and "requestBody" in pathEntry %}
{% for consume in pathEntry["requestBody"]["content"].keys() %}
rc.Consumes.Add("{{ consume }}")
{% endfor %}
{% endif %}
{% set content = dict() %}
{% if method == "get" %}
{% set content = pathEntry["responses"]["200"]["content"] %}
{% endif %}
{% if 'x-rpc' in pathEntry and 'content' in pathEntry["responses"]["204"] %}
{% set content = pathEntry["responses"]["204"]["content"] %}
{% endif %}
{% for produce in content.keys() %}
rc.Produces.Add("{{ produce }}")
{% endfor %}
{% if 'x-params' in pathEntry.keys() %}
{% set varMappings = pathEntry['x-params']['varMapping'] %}
rc.PMap = server.NameMap{ {% for varMapping in varMappings %}"{{ varMapping['uriName'] }}":"{{ varMapping['yangName'] }}", {% endfor %} }
{% endif %}
server.Process(w, r)
}
{% if not loop.last %}

{% endif %}
{% endfor %}
22 changes: 22 additions & 0 deletions tools/codegen/go-server/templates-yang/routers.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@

package openapi

import (
"github.com/Azure/sonic-mgmt-framework/rest/server"
)

func init() {

{% for operationId in OpIds %}
{% set path = OpIdDict[operationId]["path"] %}
{% set pathEntry = OpIdDict[operationId]["obj"] %}
{% set method = OpIdDict[operationId]["method"] %}
server.AddRoute(
"{{ operationId }}",
"{{ method|capitalize }}",
"{{ path }}",
{{ operationId }},
)

{% endfor %}
}
29 changes: 29 additions & 0 deletions tools/openapi_tests/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
.PHONY: all test-rpc test-data-nodes

TOPDIR := ../..
PYANG_PLUGINS_DIR := $(TOPDIR)/tools/pyang/pyang_plugins
PYANG ?= pyang

all: test-rpc test-data-nodes test-complex-model test-complex-model-no-oneof

# This will cover RPC statements with only input, only output and with
# both input and output
test-rpc:
$(PYANG) -f swaggerapi --plugindir $(PYANG_PLUGINS_DIR) test-rpc.yang | diff test-rpc.yang.expect -

# This will cover container, list, leaf and leaf-lists with both
# simple and nested hierarchies.
# Also this will cover data type testing such as leafref, enum and string with pattern
# simple string, integer types, leaf with default values, mandatory statements etc.
test-data-nodes:
$(PYANG) -f swaggerapi --plugindir $(PYANG_PLUGINS_DIR) test-data-nodes.yang | diff test-data-nodes.yang.expect -

# This will cover some complex YANGs, with many nested hierarchies
# Also with choice-case statements, Union data types, range with min,max etc
# Test with one-oneof
test-complex-model:
$(PYANG) -f swaggerapi --plugindir $(PYANG_PLUGINS_DIR) --with-oneof ietf-snmp.yang ietf-snmp-community.yang | diff ietf-snmp.yang.expect -

# Test without one-oneof
test-complex-model-no-oneof:
$(PYANG) -f swaggerapi --plugindir $(PYANG_PLUGINS_DIR) ietf-snmp.yang ietf-snmp-community.yang | diff ietf-snmp.no-oneof.yang.expect -

0 comments on commit ab7fe0b

Please sign in to comment.