Merge branch 'main' into pr-sai-api-cases-1-review

.gitmodules

@@ -5,4 +5,4 @@
 [submodule "test/SAI-Challenger"]
 	path = test/SAI-Challenger
 	url = https://github.com/opencomputeproject/SAI-Challenger
-	branch = multiple-api-support
+	branch = main

.wordlist.txt

@@ -81,12 +81,14 @@ CLI
 CloudStorm
 CNIP
 codebase
+collaterals
 compat
 Compat
 conf
 CONFDB
 confgen
 config
+configs
 Containerlab
 CP
 CreatedHalfOpenFlow
@@ -157,6 +159,8 @@ downcasting
 DPDK
 DPU
 dpu
+dpugen
+DPUGEN
 DPUControlMsgs
 DPUs
 drawio
@@ -246,6 +250,7 @@ IDL
 iface
 iflow
 impactful
+incrementing
 inband
 InbfromLB
 INIT
@@ -350,6 +355,8 @@ NVidia
 NVMe
 observability
 OCP
+OID
+OIDs
 ok
 Oleksandr
 onboardsupport
@@ -383,6 +390,7 @@ Pensando
 performant
 pingmesh
 PIR
+pluggable
 PLVision's
 pmon
 PNA
@@ -403,6 +411,7 @@ prereq
 Pritsak
 PrivateAddress
 programmability
+programmatically
 protobuf
 provids
 PRs
@@ -413,6 +422,7 @@ py
 PyPi
 pytest
 PyTest
+PyTests
 pytests
 Pytest's
 Pyunit
@@ -455,6 +465,9 @@ runtime
 rx
 SAI
 sai
+saic
+SAIC
+saichallenger
 saigen
 sairedis
 SAIRPC
@@ -495,12 +508,14 @@ SoC
 SoCs
 Sommers
 SONiC
+SONiC's
 SpoofingPacket
 SRC
 src
 STATEDB
 stateful
 statemachine
+stdout
 struct
 subclassed
 subdirectories
@@ -552,6 +567,7 @@ TGen
 Tgen
 Tgens
 th
+throughs
 ToC
 TODO
 TODOs
@@ -584,10 +600,12 @@ Unpair
 untracked
 upcasting
 upstreaming
+utils
 vcpus
 veth
 VFP
 vip
+vips
 VIP
 virsh
 virt

dash-pipeline/Makefile

@@ -429,7 +429,7 @@ run-saithrift-ptftests: disable-ipv6
 	$(DOCKER_RUN_SAITHRIFT_CLIENT) \
 	-w /tests/ \
 	$(DOCKER_SAITHRIFT_CLIENT_IMG) \
-	./run-functional-tests.sh
+	./functional/ptf/run-tests.sh
 
 # Run "dev" PTF tests mounted from host
 # Note - if add ixia-c usage to PTF tests, add dependency deploy-ixiac
@@ -438,7 +438,7 @@ run-saithrift-dev-ptftests: disable-ipv6
 	-v $(PWD)/../test/test-cases/:/tests-dev/ \
 	-w /tests-dev/ \
 	$(DOCKER_SAITHRIFT_CLIENT_IMG) \
-	./run-functional-tests.sh
+	./functional/ptf/run-tests.sh
 
 run-saithrift-client-bash:
 	$(DOCKER_RUN_SAITHRIFT_CLIENT) \
@@ -592,7 +592,7 @@ DOCKER_SAI_CHALLENGER_CLIENT_BLDR_IMG = $(DOCKER_SAI_CHALLENGER_CLIENT_BLDR_IMG_
 
 docker-saichallenger-client-bldr:
 	{ [ x$(ENABLE_DOCKER_PULL) == xy ] && docker pull $(DOCKER_SAI_CHALLENGER_CLIENT_BLDR_IMG); } || \
-		{ pushd $(SAI_CHALLENGER_PATH) && ./build.sh -i client && popd; \
+		{ pushd $(SAI_CHALLENGER_PATH) && git submodule update --init && ./build.sh -i client && popd; \
 		docker build \
 			-f dockerfiles/Dockerfile.saichallenger-client-bldr \
 			-t $(DOCKER_SAI_CHALLENGER_CLIENT_BLDR_IMG) \
@@ -609,16 +609,17 @@ docker-publish-saichallenger-client-bldr:
 docker-pull-saichallenger-client-bldr:
 	docker pull $(DOCKER_SAI_CHALLENGER_CLIENT_BLDR_IMG)
 
-docker-saichallenger-client:
-	cd $(SAI_CHALLENGER_PATH) && ./build.sh -i client
+docker-saichallenger-client: docker-saichallenger-client-bldr
 	docker build \
 		-f dockerfiles/Dockerfile.saichallenger-client \
 		-t $(DOCKER_SAI_CHALLENGER_CLIENT_IMG) \
 		.
 
 DOCKER_RUN_SAI_CHALLENGER_CLIENT=docker run \
 	-v $(SAI_CHALLENGER_PATH):/sai-challenger \
-	-v $(SAI_CHALLENGER_PATH)/../test-cases/scale/saic/:/sai-challenger/dash_tests \
+	-v $(SAI_CHALLENGER_PATH)/../test-cases/scale/saic/:/sai-challenger/dash_tests/scale \
+	-v $(SAI_CHALLENGER_PATH)/../test-cases/functional/saic/:/sai-challenger/dash_tests/functional \
+	-v $(SAI_CHALLENGER_PATH)/../test-cases/utils/:/sai-challenger/dash_tests/utils \
 	-v $(PWD)/../:/dash \
 	--cap-add=NET_ADMIN \
 	--device /dev/net/tun:/dev/net/tun \
@@ -642,12 +643,28 @@ run-saichallenger-client-bash: deploy-ixiac
 kill-saichallenger-client:
 	-docker kill $(CONTAINER_SAI_CHALLENGER_CLIENT_NAME)
 
-run-saichallenger-tests: deploy-ixiac
+run-saichallenger-tests: run-saichallenger-functional-tests run-saichallenger-scale-tests
+
+run-saichallenger-functional-tests: deploy-ixiac
 	$(DOCKER_RUN_SAI_CHALLENGER_CLIENT) \
-	-w /sai-challenger/dash_tests \
+	-w /sai-challenger/dash_tests/functional \
+	$(DOCKER_FLAGS) \
+	$(DOCKER_SAI_CHALLENGER_CLIENT_IMG) \
+	./run-tests.sh --setup=$(SAI_CHALLENGER_SETUP_FILE) $(SAI_CHALLENGER_TEST)
+
+run-saichallenger-scale-tests: deploy-ixiac
+	$(DOCKER_RUN_SAI_CHALLENGER_CLIENT) \
+	-w /sai-challenger/dash_tests/scale \
+	$(DOCKER_FLAGS) \
+	$(DOCKER_SAI_CHALLENGER_CLIENT_IMG) \
+	./run-tests.sh --setup=$(SAI_CHALLENGER_SETUP_FILE) $(SAI_CHALLENGER_TEST)
+
+run-saichallenger-tutorials: deploy-ixiac
+	$(DOCKER_RUN_SAI_CHALLENGER_CLIENT) \
+	-w /sai-challenger/dash_tests/functional \
 	$(DOCKER_FLAGS) \
 	$(DOCKER_SAI_CHALLENGER_CLIENT_IMG) \
-	./run_tests.sh --setup=$(SAI_CHALLENGER_SETUP_FILE) $(SAI_CHALLENGER_TEST)
+	./run-tests.sh --setup=$(SAI_CHALLENGER_SETUP_FILE) tutorial
 
 ###############################
 # ENVIRONMENT SETUP TARGETS

dash-pipeline/SAI/sai_api_gen.py

@@ -184,6 +184,7 @@ def generate_sai_apis(program, ignore_tables):
     for table in tables:
         sai_table_data = dict()
         sai_table_data['keys'] = []
+        sai_table_data['ipaddr_family_attr'] = 'false'
         sai_table_data[ACTIONS_TAG] = []
         sai_table_data[ACTION_PARAMS_TAG] = []
 
@@ -221,6 +222,13 @@ def generate_sai_apis(program, ignore_tables):
                         key2["v4_or_v6_id"] = key['id']
                         break
 
+        for key in sai_table_data['keys']:
+            if (key['match_type'] == 'exact' and key['sai_key_type'] == 'sai_ip_address_t') or \
+                (key['match_type'] == 'ternary' and key['sai_key_type'] == 'sai_ip_address_t') or \
+                (key['match_type'] == 'lpm' and key['sai_lpm_type'] == 'sai_ip_prefix_t') or \
+                (key['match_type'] == 'list' and key['sai_list_type'] == 'sai_ip_prefix_list_t'):
+                    sai_table_data['ipaddr_family_attr'] = 'true'
+
         param_names = []
         for action in table[ACTION_REFS_TAG]:
             action_id = action["id"]

dash-pipeline/SAI/templates/saiapi.cpp.j2

@@ -193,7 +193,7 @@ sai_status_t sai_create_{{ table.name }}(
     return 0;
 ErrRet:
     delete matchActionEntry;
-    RemoveFromTable({{ table.name }}_id);
+    RemoveFromTable(*{{ table.name }}_id);
     return -1;
 }
 

dash-pipeline/SAI/templates/saiapi.h.j2

@@ -225,6 +225,17 @@ typedef enum _sai_{{ table.name }}_attr_t
 
 {% endif %}
 {% endif %}
+{% endif %}
+{% if table.ipaddr_family_attr == 'true' %}
+    /**
+     * @brief IP address family for resource accounting
+     *
+     * @type sai_ip_addr_family_t
+     * @flags READ_ONLY
+     * @isresourcetype true
+     */
+    SAI_{{ table.name | upper }}_ATTR_IP_ADDR_FAMILY,
+
 {% endif %}
     /**
      * @brief End of attributes

dash-pipeline/dockerfiles/DOCKER_SAI_CHALLENGER_CLIENT_BLDR_IMG.env

@@ -2,4 +2,4 @@
 # Changing this will cause build/publish to occur in CI actions
 export DASH_ACR_REGISTRY=sonicdash.azurecr.io
 export DOCKER_SAI_CHALLENGER_CLIENT_BLDR_IMG_NAME=${DASH_ACR_REGISTRY}/dash-saichallenger-client-bldr
-export DOCKER_SAI_CHALLENGER_CLIENT_BLDR_IMG_CTAG?=221209
+export DOCKER_SAI_CHALLENGER_CLIENT_BLDR_IMG_CTAG?=221228

dash-pipeline/dockerfiles/Dockerfile.saichallenger-client

@@ -1,4 +1,4 @@
-FROM sonicdash.azurecr.io/dash-saichallenger-client-bldr:221109
+FROM sonicdash.azurecr.io/dash-saichallenger-client-bldr:221228
 
 ENV SAI_CHALLENGER_PATH /sai-challenger
 ENV DASH_PATH /dash
@@ -20,7 +20,6 @@ RUN cd /saithrift-0.9 && \
         rm -rf thrift-0.11.0 && \
     cd /SAI/test/ptf && \
         python3 setup.py install && \
-    ln -s ${SAI_CHALLENGER_PATH} /usr/local/lib/python3.7/dist-packages/saichallenger && \
     ln -s ${DASH_PATH}/test/test-cases/scale/saic ${SAI_CHALLENGER_PATH}/dash_tests
 
 CMD ["/usr/bin/supervisord"]

dash-pipeline/dockerfiles/Dockerfile.saichallenger-client-bldr

@@ -1,3 +1,4 @@
+# Requires <url of sai-challenger branch> <commit sha> or something
 FROM sc-client
 
 ADD tests/ /tests/

documentation/general/sdn-pipeline-basic-elements.md

@@ -338,6 +338,8 @@ corresponding flow is created in the flow table.
 
 ### VM to VM (in VNET) communication
 
+Please note that VNET to VNET is seen below.  VNET Peering is identical and no datapath changes are needed.  VNET Global Peering uses IPv6 as the tunnel and uses the same tunnel format.  
+
 ![VMtoVM](./images/sdn/sdn-packet-transforms-vm-to-vm.svg)
 
 ### Internal Load balancer (in VNET) communication