DASH Test framework extension.

.github/workflows/dash-bmv2-ci.yml

@@ -11,7 +11,8 @@ on:
       - 'test/**.sh'
       - 'test/**.yml'
       - 'dash-pipeline/**'
-      - '!dash-pipeline/docker files/Dockerfile.*'
+      - '!dash-pipeline/dockerfiles/Dockerfile.*'
+      - 'dash-pipeline/dockerfiles/*.env'
       - '!dash-pipeline/.dockerignore'
       - '!dash-pipeline/**.md'
       - '!dash-pipeline/**.svg'
@@ -27,7 +28,8 @@ on:
       - 'test/**.sh'
       - 'test/**.yml'
       - 'dash-pipeline/**'
-      - '!dash-pipeline/docker files/Dockerfile.*'
+      - '!dash-pipeline/dockerfiles/Dockerfile.*'
+      - 'dash-pipeline/dockerfiles/*.env'
       - '!dash-pipeline/.dockerignore'
       - '!dash-pipeline/**.md'
       - '!dash-pipeline/**.svg'
@@ -82,4 +84,6 @@ jobs:
       run:  DOCKER_FLAGS=$docker_fg_flags make deploy-ixiac
     - name: Run Pytests
       run:  DOCKER_FLAGS=$docker_fg_root_flags make run-saithrift-pytests
+    - name: Run SAI-Challenger Tests
+      run:  DOCKER_FLAGS=$docker_fg_root_flags make run-saichallenger-tests
 

.github/workflows/dash-saichallenger-client-docker-acr.yml

@@ -0,0 +1,54 @@
+name: DASH-docker-saichallenger-client-build-publish-acr
+
+on:
+  push:
+    branches: [ "**" ]
+    paths:
+      - '.github/workflows/dash-saichallenger-client-docker-acr.yml'
+      - 'dash-pipeline/Makefile'
+      - 'dash-pipeline/dockerfiles/Dockerfile.saichallenger-client'
+      - 'dash-pipeline/.dockerignore'
+      - 'dash-pipeline/dockerfiles/.dockerignore'
+  pull_request:
+    branches: [ "**" ]
+    paths:
+      - '.github/workflows/dash-saichallenger-client-docker-acr.yml'
+      - 'dash-pipeline/Makefile'
+      - 'dash-pipeline/dockerfiles/Dockerfile.saichallenger-client'
+      - 'dash-pipeline/.dockerignore'
+      - 'dash-pipeline/dockerfiles/.dockerignore'
+  workflow_dispatch:
+
+jobs:
+  build:
+    # Can only publish from within DASH repo (need credentials from secrets)
+    if: github.repository == 'Azure/DASH'
+    name: Build and publish dash-saichallenger-client image
+    runs-on: ubuntu-20.04
+    env:
+      docker_fg_flags: -u root --privileged
+      docker_bg_flags: -d -u root --privileged
+    defaults:
+      run:
+        working-directory: ./dash-pipeline
+    steps:
+    - uses: actions/checkout@v3
+    - name: Pull docker p4c image
+      run:  make docker-pull-dash-p4c
+    - name: Build P4 software switch (bmv2) and P4Info
+      run:  DOCKER_FLAGS=$docker_fg_flags make p4
+    - name: Install SAI submodule
+      run:  git submodule update --init
+    - name: Generate SAI API
+      run:  DOCKER_FLAGS=$docker_fg_flags make sai
+    - name: Generate SAI-Thrift client and server code and libs
+      run:  DOCKER_FLAGS=$docker_fg_flags make saithrift-server
+    - uses: azure/docker-login@v1
+      with:
+        login-server: sonicdash.azurecr.io
+        username: ${{ secrets.DASH_ACR_USERNAME }}
+        password: ${{ secrets.DASH_ACR_PASSWORD }}    
+    - name: Build SAI-Challenger client docker image
+      run: DOCKER_FLAGS=$docker_fg_flags make docker-saichallenger-client
+    - name: Publish SAI-Challenger client docker image
+      run:  make docker-publish-saichallenger-client

.github/workflows/dash-saichallenger-client-docker.yml

@@ -0,0 +1,45 @@
+name: DASH-docker-saichallenger-client-img
+
+on:
+  push:
+    branches: [ "**" ]
+    paths:
+      - '.github/workflows/dash-saichallenger-client-docker.yml'
+      - 'dash-pipeline/Makefile'
+      - 'dash-pipeline/dockerfiles/Dockerfile.saichallenger-client'
+      - 'dash-pipeline/.dockerignore'
+      - 'dash-pipeline/dockerfiles/.dockerignore'
+  pull_request:
+    branches: [ "**" ]
+    paths:
+      - '.github/workflows/dash-saichallenger-client-docker.yml'
+      - 'dash-pipeline/Makefile'
+      - 'dash-pipeline/dockerfiles/Dockerfile.saichallenger-client'
+      - 'dash-pipeline/.dockerignore'
+      - 'dash-pipeline/dockerfiles/.dockerignore'
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Build dash-saichallenger-client-image
+    runs-on: ubuntu-20.04
+    env:
+      docker_fg_flags: -u root --privileged
+      docker_bg_flags: -d -u root --privileged
+    defaults:
+      run:
+        working-directory: ./dash-pipeline
+    steps:
+    - uses: actions/checkout@v3
+    - name: Pull docker p4c image
+      run:  make docker-pull-dash-p4c
+    - name: Build P4 software switch (bmv2) and P4Info
+      run:  DOCKER_FLAGS=$docker_fg_flags make p4
+    - name: Install SAI submodule
+      run:  git submodule update --init
+    - name: Generate SAI API
+      run:  DOCKER_FLAGS=$docker_fg_flags make sai
+    - name: Generate SAI-Thrift client and server code and libs
+      run:  DOCKER_FLAGS=$docker_fg_flags make saithrift-server
+    - name: Build SAI-Challenger client docker image
+      run:  DOCKER_FLAGS=$docker_fg_flags make docker-saichallenger-client

.github/workflows/dash-saithrift-client-bldr-docker-acr.yml

@@ -49,8 +49,8 @@ jobs:
       with:
         login-server: sonicdash.azurecr.io
         username: ${{ secrets.DASH_ACR_USERNAME }}
-        password: ${{ secrets.DASH_ACR_PASSWORD }}    
-    - name: Build dash-saithrift-client-bldr image 
+        password: ${{ secrets.DASH_ACR_PASSWORD }}
+    - name: Build dash-saithrift-client-bldr image
       run:  DOCKER_FLAGS=$docker_fg_flags make docker-saithrift-client-bldr
     - name: Publish dash-saithrift-client-bldr docker image
       run: make docker-publish-saithrift-client-bldr

.gitmodules

@@ -2,3 +2,7 @@
 	path = dash-pipeline/SAI/SAI
 	url = https://github.com/reshmaintel/SAI.git
 	branch = dash-ptf-ci
+[submodule "test/SAI-Challenger"]
+	path = test/SAI-Challenger
+	url = https://github.com/opencomputeproject/SAI-Challenger
+	branch = multiple-api-support

.wordlist.txt

@@ -296,6 +296,7 @@ LLDP
 lldp
 loadbalancer
 lookups
+LOOKUPs
 loopback
 LPM
 lts
@@ -304,6 +305,7 @@ makefile
 Makefile
 Makefiles
 makefiles
+MAPs
 MatchedHalfOpenFlow
 MatchedOtherFlow
 MatchedTcpFlow
@@ -421,6 +423,7 @@ qos
 Radv
 rdpty
 reachability
+reconvergence
 README
 READMEs
 README's
@@ -452,12 +455,14 @@ runtime
 rx
 SAI
 sai
+saigen
 sairedis
 SAIRPC
 saiserver
 saithrift
 sata
 scalability
+scalable
 Scalable
 scapy
 Schemas

dash-pipeline/Makefile

@@ -6,15 +6,15 @@ mkfile_dir := $(dir $(mkfile_path))
 # "All" type targets for convenience
 all:p4 sai saithrift-server docker-saithrift-client test
 
-run-all-tests:run-libsai-test run-saithrift-client-tests
+run-all-tests:run-libsai-test run-saithrift-client-tests run-saichallenger-tests
 run-saithrift-client-tests: run-saithrift-ptftests run-saithrift-pytests
 run-saithrift-client-dev-tests: run-saithrift-dev-ptftests run-saithrift-dev-pytests
 
 .PHONY:clean
 clean: kill-all p4-clean sai-clean test-clean network-clean saithrift-server-clean
 	rm -rf $(P4_OUTDIR)
 
-kill-all: kill-saithrift-server kill-switch undeploy-ixiac
+kill-all: kill-saithrift-server kill-switch undeploy-ixiac kill-saichallenger-client
 
 PWD := $(realpath $(mkfile_dir))
 DASH_USER ?=dashuser
@@ -49,6 +49,10 @@ include dockerfiles/DOCKER_SAITHRIFT_CLIENT_BLDR_IMG.env
 # TODO: consider some other tagging scheme
 DOCKER_SAITHRIFT_CLIENT_IMG ?= local/dash-saithrift-client:latest
 
+# Base image with test frameworks, DASH client libs not installed
+# include file defines DOCKER_SAI_CHALLENGER_CLIENT_IMG
+include dockerfiles/DOCKER_SAI_CHALLENGER_CLIENT_IMG.env
+
 # Set differently in CI scripts as needed, e.g. run switch container in -d mode
 DOCKER_FLAGS ?=-it
 
@@ -122,7 +126,7 @@ DOCKER_RUN_SAITHRIFT_BLDR =\
 
 sai: sai-clean sai-headers sai-meta libsai
 
-sai-headers: p4 | SAI/SAI 
+sai-headers: p4 | SAI/SAI
 	@echo "Generate SAI library headers and implementation..."
 	mkdir -p SAI/lib && chmod -R o+w SAI && \
 	$(DOCKER_RUN) \
@@ -131,7 +135,7 @@ sai-headers: p4 | SAI/SAI
 		-w /SAI $(DOCKER_SAITHRIFT_BLDR_IMG) \
 	    ./generate_dash_api.sh
 
-sai-meta: 
+sai-meta:
 	@echo "Generate SAI metadata..."
 	# hack - remove scripts which cause Git ownership failures in CI pipelines
 	# We don't need them, they're to check that SAI headers didn't experience enum changes etc.
@@ -146,7 +150,7 @@ sai-meta:
 	    make
 
 # TODO - add SAI header dependencies
-libsai: 
+libsai:
 	@echo "build libsai.so..."
 	$(DOCKER_RUN) \
 		$(DOCKER_FLAGS) \
@@ -155,7 +159,7 @@ libsai:
 		$(DOCKER_BMV2_BLDR_IMG) \
 	    make
 
-libsai-clean: 
+libsai-clean:
 	-rm -rf SAI/lib/*
 
 .PHONY:sai-clean
@@ -354,7 +358,7 @@ docker-publish-saithrift-client-bldr:
 	docker push $(DOCKER_SAITHRIFT_CLIENT_BLDR_IMG)
 
 ###############################
-		
+
 # Client image, rebuild any time SAI interface changes
 # TODO - add sai header (inc/ and experimental) dependencies
 docker-saithrift-client:
@@ -512,6 +516,73 @@ deploy-ixiac:
 undeploy-ixiac:
 	cd ../test/third-party/traffic_gen && ./undeploy_ixiac.sh
 
+###############################
+# SAI-CHALLENGER TARGETS
+###############################
+
+SAI_CHALLENGER_PATH = $(PWD)/../test/SAI-Challenger
+
+CONTAINER_SAI_CHALLENGER_CLIENT_NAME = dash-saichallenger-client-$(USER)
+
+# Add passing parameters to the run-saichallenger-tests target.
+ifeq (run-saichallenger-tests,$(firstword $(MAKECMDGOALS)))
+  SAI_CHALLENGER_SETUP_FILE := $(wordlist 2, 2, $(MAKECMDGOALS))
+  SAI_CHALLENGER_TEST := $(wordlist 3, 3, $(MAKECMDGOALS))
+  $(eval $(SAI_CHALLENGER_SETUP_FILE):;@:)
+  $(eval $(SAI_CHALLENGER_TEST):;@:)
+endif
+# If setup file's isn't passed, set it to the default.
+ifeq ($(SAI_CHALLENGER_SETUP_FILE),)
+SAI_CHALLENGER_SETUP_FILE := sai_dpu_client_server_snappi.json
+endif
+# If test's name isn't passed, set it to the default.
+ifeq ($(SAI_CHALLENGER_TEST),)
+SAI_CHALLENGER_TEST := test_sai_vnet_*.py
+endif
+
+docker-saichallenger-client:
+	cd $(SAI_CHALLENGER_PATH) && ./build.sh -i client
+	docker build \
+		-f dockerfiles/Dockerfile.saichallenger-client \
+		-t $(DOCKER_SAI_CHALLENGER_CLIENT_IMG) \
+		.
+
+docker-publish-saichallenger-client:
+	@echo "Publish $(DOCKER_SAI_CHALLENGER_CLIENT_IMG) - requires credentials, can only do from DASH repo, not a fork"
+	docker push $(DOCKER_SAI_CHALLENGER_CLIENT_IMG)
+
+DOCKER_RUN_SAI_CHALLENGER_CLIENT=docker run \
+	-v $(SAI_CHALLENGER_PATH):/sai-challenger \
+	-v $(SAI_CHALLENGER_PATH)/../test-cases/scale/saic/:/sai-challenger/dash_tests \
+	-v $(PWD)/../:/dash \
+	--cap-add=NET_ADMIN \
+	--device /dev/net/tun:/dev/net/tun \
+	--rm \
+	--network=host \
+	--name $(CONTAINER_SAI_CHALLENGER_CLIENT_NAME)
+
+run-saichallenger-client: deploy-ixiac
+	$(DOCKER_RUN_SAI_CHALLENGER_CLIENT) \
+	-d \
+	--name $(CONTAINER_SAI_CHALLENGER_CLIENT_NAME) \
+	$(DOCKER_SAI_CHALLENGER_CLIENT_IMG)
+
+run-saichallenger-client-bash: deploy-ixiac
+	$(DOCKER_RUN_SAI_CHALLENGER_CLIENT) \
+	-w /sai-challenger/dash_tests \
+	$(DOCKER_FLAGS) \
+	$(DOCKER_SAI_CHALLENGER_CLIENT_IMG) \
+	/bin/bash
+
+kill-saichallenger-client:
+	-docker kill $(CONTAINER_SAI_CHALLENGER_CLIENT_NAME)
+
+run-saichallenger-tests: deploy-ixiac
+	$(DOCKER_RUN_SAI_CHALLENGER_CLIENT) \
+	-w /sai-challenger/dash_tests \
+	$(DOCKER_FLAGS) \
+	$(DOCKER_SAI_CHALLENGER_CLIENT_IMG) \
+	./run_vnet_tests.sh --setup=$(SAI_CHALLENGER_SETUP_FILE) $(SAI_CHALLENGER_TEST)
 
 ###############################
 # ENVIRONMENT SETUP TARGETS