diff --git a/.github/workflows/dash-bmv2-ci.yml b/.github/workflows/dash-bmv2-ci.yml index d951ab851..4b9c6c729 100644 --- a/.github/workflows/dash-bmv2-ci.yml +++ b/.github/workflows/dash-bmv2-ci.yml @@ -40,8 +40,9 @@ jobs: name: Build and Test DASH Pipeline runs-on: ubuntu-20.04 env: - docker_fg_flags: -u root --privileged - docker_bg_flags: -d -u root --privileged + docker_fg_flags: --privileged + docker_fg_root_flags: --privileged -u root + docker_bg_flags: -d --privileged defaults: run: working-directory: ./dash-pipeline @@ -76,9 +77,9 @@ jobs: - name: Run saithrift server run: DOCKER_FLAGS=$docker_bg_flags make run-saithrift-server - name: Run PTF Tests - run: DOCKER_FLAGS=$docker_fg_flags make run-saithrift-ptftests + run: DOCKER_FLAGS=$docker_fg_root_flags make run-saithrift-ptftests - name: Deploy ixia-c Traffic Generator run: DOCKER_FLAGS=$docker_fg_flags make deploy-ixiac - name: Run Pytests - run: DOCKER_FLAGS=$docker_fg_flags make run-saithrift-pytests + run: DOCKER_FLAGS=$docker_fg_root_flags make run-saithrift-pytests diff --git a/dash-pipeline/Makefile b/dash-pipeline/Makefile index 73f45f806..237fbaed4 100644 --- a/dash-pipeline/Makefile +++ b/dash-pipeline/Makefile @@ -1,45 +1,44 @@ SHELL = /bin/bash # "All" type targets for convenience -all:fix-perms p4 sai saithrift-server docker-saithrift-client test +all:p4 sai saithrift-server docker-saithrift-client test run-all-tests:run-libsai-test deploy-ixiac run-saithrift-client-tests run-saithrift-client-tests: run-saithrift-ptftests run-saithrift-pytests run-saithrift-client-dev-tests: run-saithrift-dev-ptftests run-saithrift-dev-pytests .PHONY:clean -clean: fix-perms kill-all p4-clean sai-clean test-clean network-clean saithrift-server-clean - rm -rf bmv2/dash_pipeline.bmv2 +clean: kill-all p4-clean sai-clean test-clean network-clean saithrift-server-clean + rm -rf $(P4_OUTDIR) kill-all: kill-saithrift-server kill-switch undeploy-ixiac - -.PHONY: fix-perms -fix-perms: - @echo "### Applying Fix permissions workaround..." - sudo chmod -R o+rw SAI/ bmv2/ tests/ - PWD := $(shell pwd) +DASH_USER ?=dashuser +DASH_GROUP ?=dashusers +DASH_UID ?=4321 +DASH_GUID?=4321 +DASH_HOST?=dash # Default Docker images, override from env var # Needed Libraries only DOCKER_GRPC_IMG ?=chrissommers/dash-grpc:1.43.2 -# Slimmed-down version bmv2 backend only 632MB - works: -DOCKER_P4C_BMV2_IMG ?=chrissommers/dash-p4c-bmv2:220819 +# Slimmed-down version bmv2 backend only 632MB: +DOCKER_P4C_BMV2_IMG ?=chrissommers/dash-p4c-bmv2:220825 -# Builds sai-P4rt clients to run inside bmvs process -DOCKER_BMV2_BLDR_IMG ?=chrissommers/dash-bmv2-bldr:220630 +# Builds SAI-P4RT clients to run inside bmv2 process +DOCKER_BMV2_BLDR_IMG ?=chrissommers/dash-bmv2-bldr:220819 # Runs bmv2 process - consider slimmer if don't need sai/p4rt clients inside DOCKER_BMV2_RUN_IMG ?=$(DOCKER_BMV2_BLDR_IMG) # Compiles sai headers, libsai, saithrift server & client/server libs -DOCKER_SAITHRIFT_BLDR_IMG ?=chrissommers/dash-saithrift-bldr:220719 +DOCKER_SAITHRIFT_BLDR_IMG ?=chrissommers/dash-saithrift-bldr:220819 # Base image with test frameworks, DASH client libs not installed -DOCKER_SAITHRIFT_CLIENT_BLDR_IMG ?=chrissommers/dash-saithrift-client-bldr:220723 +DOCKER_SAITHRIFT_CLIENT_BLDR_IMG ?=chrissommers/dash-saithrift-client-bldr:220819 # Client image built locally with saithrift client libs + PTF & Pytest frameworks # TODO: consider some other tagging scheme @@ -70,24 +69,26 @@ sai-submodule: # P4 Source code compile TARGETS ###################################### -P4_ARTIFACTS=bmv2/dash_pipeline.bmv2/dash_pipeline.json bmv2/dash_pipeline.bmv2/dash_pipeline_p4rt.txt +P4_SRC=$(wildcard bmv2/*.p4) +P4_MAIN=bmv2/dash_pipeline.p4 +P4_OUTDIR=bmv2/dash_pipeline.bmv2 + +P4_ARTIFACTS=$(P4_OUTDIR)/dash_pipeline.json $(P4_OUTDIR)/dash_pipeline_p4rt.txt -p4: fix-perms bmv2/dash_pipeline.bmv2/dash_pipeline.json +p4: $(P4_OUTDIR)/dash_pipeline.json .PHONY:p4-clean p4-clean: - -rm -rf bmv2/dash_pipeline.bmv2 - -P4_SRC=$(wildcard bmv2/*.p4) -P4_MAIN=bmv2/dash_pipeline.p4 -P4_OUTDIR=bmv2/dash_pipeline.bmv2 + -rm -rf $(P4_OUTDIR) # Compile P4 into bmv2 .json fle and P4info for SAI header autogeneration $(P4_ARTIFACTS): $(P4_SRC) @echo "Compile P4 program $(P4_MAIN) ..." mkdir -p $(P4_OUTDIR) && \ + chmod o+w $(P4_OUTDIR) && \ docker run \ --rm \ --name dash-p4c-$(USER) \ + -u $(DASH_USER) \ $(DOCKER_FLAGS) \ -v $(PWD)/bmv2:/bmv2 \ -w / \ @@ -114,20 +115,18 @@ DOCKER_RUN_SAITHRIFT_BLDR =\ # TODO - create separate rules for headers, libsai.so .PHONY:sai -sai: fix-perms sai-clean sai-headers sai-meta libsai +sai: sai-clean sai-headers sai-meta libsai -sai-headers: fix-perms p4 | SAI/SAI +sai-headers: p4 | SAI/SAI @echo "Generate SAI library headers and implementation..." - mkdir -p SAI/lib + mkdir -p SAI/lib && chmod -R o+w SAI && \ $(DOCKER_RUN) \ $(DOCKER_FLAGS) \ --name build_sai-$(USER) \ -w /SAI $(DOCKER_SAITHRIFT_BLDR_IMG) \ ./generate_dash_api.sh - # Fixup perms from generate_dash_api using sudo to call python,so subsequent scripts can access - sudo chmod -R a+rw $(PWD)/SAI/SAI -sai-meta: fix-perms +sai-meta: @echo "Generate SAI metadata..." # hack - remove scripts which cause Git ownership failures in CI pipelines # We don't need them, they're to check that SAI headers didn't experience enum changes etc. @@ -140,10 +139,9 @@ sai-meta: fix-perms -v $(PWD)/..:/dash -w /dash/dash-pipeline/SAI/SAI/meta \ $(DOCKER_SAITHRIFT_BLDR_IMG) \ make - sudo chmod -R o+rw SAI/ # TODO - add SAI header dependencies -libsai: fix-perms +libsai: @echo "build libsai.so..." $(DOCKER_RUN) \ $(DOCKER_FLAGS) \ @@ -152,11 +150,11 @@ libsai: fix-perms $(DOCKER_BMV2_BLDR_IMG) \ make -libsai-clean: fix-perms - -rm -rf SAI/lib/libsai.so +libsai-clean: + -rm -rf SAI/lib/* .PHONY:sai-clean -sai-clean: fix-perms SAI/SAI libsai-clean +sai-clean: SAI/SAI libsai-clean saithrift-server-clean @echo "Restoring SAI subdirectories to baseline..." rm -rf SAI/SAI/inc SAI/SAI/experimental SAI/SAI/meta cd SAI/SAI && git checkout -- inc experimental meta @@ -176,8 +174,8 @@ run-switch: network $(DOCKER_RUN) \ --name simple_switch-$(USER) \ -u root \ - -v $(PWD)/bmv2/dash_pipeline.bmv2/dash_pipeline.json:/etc/dash/dash_pipeline.json \ - -v $(PWD)/bmv2/dash_pipeline.bmv2/dash_pipeline_p4rt.txt:/etc/dash/dash_pipeline_p4rt.txt \ + -v $(PWD)/$(P4_OUTDIR)/dash_pipeline.json:/etc/dash/dash_pipeline.json \ + -v $(PWD)/$(P4_OUTDIR)/dash_pipeline_p4rt.txt:/etc/dash/dash_pipeline_p4rt.txt \ $(DOCKER_BMV2_RUN_IMG) \ env LD_LIBRARY_PATH=/usr/local/lib \ simple_switch_grpc \ @@ -194,7 +192,7 @@ kill-switch: # SAI-THRIFT SERVER TARGETS ############################### -saithrift-server:fix-perms +saithrift-server: $(DOCKER_RUN_SAITHRIFT_BLDR) \ make $@ @@ -204,8 +202,8 @@ DOCKER_RUN_SAITHRIFT_SRVR =\ $(DOCKER_FLAGS) \ --net=host \ --name dash-saithrift-server-$(USER) \ - -v $(PWD)/bmv2/dash_pipeline.bmv2/dash_pipeline.json:/etc/dash/dash_pipeline.json \ - -v $(PWD)/bmv2/dash_pipeline.bmv2/dash_pipeline_p4rt.txt:/etc/dash/dash_pipeline_p4rt.txt \ + -v $(PWD)/$(P4_OUTDIR)/dash_pipeline.json:/etc/dash/dash_pipeline.json \ + -v $(PWD)/$(P4_OUTDIR)/dash_pipeline_p4rt.txt:/etc/dash/dash_pipeline_p4rt.txt \ -v $(PWD)/SAI:/SAI \ -v $(PWD)/SAI/SAI/meta:/meta \ -w /SAI/rpc/usr/sbin \ @@ -226,6 +224,7 @@ run-saithrift-server-bash: saithrift-server-clean: $(DOCKER_RUN_SAITHRIFT_BLDR) \ make $@ + rm -rf SAI/rpc ############################### # SAI TEST TARGETS @@ -240,6 +239,7 @@ test-clean: .PHONY:libsai-test libsai-test:SAI/SAI SAI/lib/libsai.so + chmod -R a+w tests/libsai $(DOCKER_RUN) \ --name dash-build-test-$(USER) \ -w /tests/libsai $(DOCKER_BMV2_BLDR_IMG) \ @@ -286,10 +286,11 @@ docker-bmv2-bldr: docker build \ -f dockerfiles/Dockerfile.bmv2-bldr \ -t $(DOCKER_BMV2_BLDR_IMG) \ - --build-arg user=$(USER) \ - --build-arg uid=$(shell id -u) \ - --build-arg guid=$(shell id -g) \ - --build-arg hostname=$(shell echo $$HOSTNAME) \ + --build-arg user=$(DASH_USER) \ + --build-arg group=$(DASH_GROUP) \ + --build-arg uid=$(DASH_UID) \ + --build-arg guid=$(DASH_GUID) \ + --build-arg hostname=$(DASH_HOST) \ --build-arg available_processors=$(shell nproc) \ dockerfiles @@ -303,10 +304,11 @@ docker-saithrift-bldr: docker build \ -f dockerfiles/Dockerfile.saithrift-bldr \ -t $(DOCKER_SAITHRIFT_BLDR_IMG) \ - --build-arg user=$(USER) \ - --build-arg uid=$(shell id -u) \ - --build-arg guid=$(shell id -g) \ - --build-arg hostname=$(shell echo $$HOSTNAME) \ + --build-arg user=$(DASH_USER) \ + --build-arg group=$(DASH_GROUP) \ + --build-arg uid=$(DASH_UID) \ + --build-arg guid=$(DASH_GUID) \ + --build-arg hostname=$(DASH_HOST) \ --build-arg available_processors=$(shell nproc) \ dockerfiles @@ -322,10 +324,11 @@ docker-saithrift-client-bldr: docker build \ -f dockerfiles/Dockerfile.saithrift-client-bldr \ -t $(DOCKER_SAITHRIFT_CLIENT_BLDR_IMG) \ - --build-arg user=$(USER) \ - --build-arg uid=$(shell id -u) \ - --build-arg guid=$(shell id -g) \ - --build-arg hostname=$(shell echo $$HOSTNAME) \ + --build-arg user=$(DASH_USER) \ + --build-arg group=$(DASH_GROUP) \ + --build-arg uid=$(DASH_UID) \ + --build-arg guid=$(DASH_GUID) \ + --build-arg hostname=$(DASH_HOST) \ --build-arg available_processors=$(shell nproc) \ ./tests @@ -394,14 +397,26 @@ run-saithrift-client-bash: ############################### +# docker-dash-p4c: +# docker build \ +# -f dockerfiles/Dockerfile.p4c-bmv2 \ +# -t $(DOCKER_P4C_BMV2_IMG) \ +# --build-arg user=$(USER) \ +# --build-arg uid=$(shell id -u) \ +# --build-arg guid=$(shell id -g) \ +# --build-arg hostname=$(shell echo $$HOSTNAME) \ +# --build-arg available_processors=$(shell nproc) \ +# dockerfiles + docker-dash-p4c: docker build \ -f dockerfiles/Dockerfile.p4c-bmv2 \ -t $(DOCKER_P4C_BMV2_IMG) \ - --build-arg user=$(USER) \ - --build-arg uid=$(shell id -u) \ - --build-arg guid=$(shell id -g) \ - --build-arg hostname=$(shell echo $$HOSTNAME) \ + --build-arg user=$(DASH_USER) \ + --build-arg group=$(DASH_GROUP) \ + --build-arg uid=$(DASH_UID) \ + --build-arg guid=$(DASH_GUID) \ + --build-arg hostname=$(DASH_HOST) \ --build-arg available_processors=$(shell nproc) \ dockerfiles @@ -416,10 +431,11 @@ docker-dash-grpc: docker build \ -f dockerfiles/Dockerfile.grpc1.43.2 \ -t $(DOCKER_GRPC_IMG) \ - --build-arg user=$(USER) \ - --build-arg uid=$(shell id -u) \ - --build-arg guid=$(shell id -g) \ - --build-arg hostname=$(shell echo $$HOSTNAME) \ + --build-arg user=$(DASH_USER) \ + --build-arg group=$(DASH_GROUP) \ + --build-arg uid=$(DASH_UID) \ + --build-arg guid=$(DASH_GUID) \ + --build-arg hostname=$(DASH_HOST) \ --build-arg available_processors=$(shell nproc) \ dockerfiles @@ -473,10 +489,10 @@ veth2: /sys/class/net/veth2 sudo sysctl net.ipv6.conf.veth3.accept_ra_pinfo=0 sudo sysctl net.ipv6.conf.veth3.router_solicitations=0 +# Delete veth's, test existence to avoid needless use of sudo network-clean: - # Note, deleting one member (e.g. veth0) deletes its peer (e.g. veth1) - -sudo ip link delete dev veth0 - -sudo ip link delete dev veth2 + @-([ -e /sys/class/net/veth0 ] && sudo ip link delete dev veth0 && echo "Deleted veth0/1") || echo "No veth0, not deleting" + @-([ -e /sys/class/net/veth2 ] && sudo ip link delete dev veth2 && echo "Deleted veth2/3") || echo "No veth2, not deleting" ############################### # IXIA-C TARGETS diff --git a/dash-pipeline/SAI/generate_dash_api.sh b/dash-pipeline/SAI/generate_dash_api.sh index 440f96153..d3c73ebe2 100755 --- a/dash-pipeline/SAI/generate_dash_api.sh +++ b/dash-pipeline/SAI/generate_dash_api.sh @@ -1,5 +1,5 @@ #!/bin/bash -sudo ./sai_api_gen.py \ +./sai_api_gen.py \ /bmv2/dash_pipeline.bmv2/dash_pipeline_p4rt.json \ --ignore-tables=appliance,eni_meter,slb_decap \ dash diff --git a/dash-pipeline/SAI/saithrift/Makefile b/dash-pipeline/SAI/saithrift/Makefile index 936d380fc..0cf0c36d6 100644 --- a/dash-pipeline/SAI/saithrift/Makefile +++ b/dash-pipeline/SAI/saithrift/Makefile @@ -27,7 +27,7 @@ SAIRPC_EXTRA_LIBS="\ -lssl \ -laddress_sorting" -# Below based on: https://github.com/opencomputeproject/SAI/blob/088627dd90c3420daf96d294c661b4a152afb01e/ptf/SAI_PTF_user-guide.md +# Below based on: https://github.com/opencomputeproject/SAI/blob/master/ptf/SAI_PTF_user-guide.md # Dependencies are assumed to be installed, e.g. in the docker container saithrift-server: # Copy headers to /usr/include/sai @@ -41,6 +41,7 @@ saithrift-server: sudo cp $(LIB)/libsai.so /usr/lib @echo "Build SAI thrift server and libraries..." + mkdir -p $(RPC_INST_DIR) && chmod a+w $(RPC_INST_DIR) cd $(SAI) && export SAITHRIFTV2=y && \ export GEN_SAIRPC_OPTS="-ve" && \ @@ -57,6 +58,7 @@ saithrift-server: # Copy thrift libs from builder image onto host cp /usr/lib/libthrift*so* /usr/lib/thrift-0.11.0.tar.gz $(RPC_INST_DIR) + chmod -R a+w $(META)/generated $(META)/html $(META)/xml $(RPC_INST_DIR) saithrift-server-clean: cd $(SAI) && export SAITHRIFTV2=y && make clean diff --git a/dash-pipeline/dockerfiles/Dockerfile.bmv2-bldr b/dash-pipeline/dockerfiles/Dockerfile.bmv2-bldr index 93c8b1c1e..7dc0e64d1 100644 --- a/dash-pipeline/dockerfiles/Dockerfile.bmv2-bldr +++ b/dash-pipeline/dockerfiles/Dockerfile.bmv2-bldr @@ -63,13 +63,14 @@ COPY --from=grpc /usr/local/lib/lib*grpc*.so* \ ARG user ARG uid +ARG group ARG guid ARG hostname ENV BUILD_HOSTNAME $hostname ENV USER $user -RUN groupadd -f -r -g $guid g$user +RUN groupadd -f -r -g $guid $group RUN useradd $user -l -u $uid -g $guid -d /var/$user -m -s /bin/bash diff --git a/dash-pipeline/dockerfiles/Dockerfile.p4c-bmv2 b/dash-pipeline/dockerfiles/Dockerfile.p4c-bmv2 index 413c1a0c3..d58281953 100644 --- a/dash-pipeline/dockerfiles/Dockerfile.p4c-bmv2 +++ b/dash-pipeline/dockerfiles/Dockerfile.p4c-bmv2 @@ -45,6 +45,19 @@ COPY --from=p4lang-p4c \ /usr/local/bin/p4c-bm2-ss \ /usr/local/bin/ + +ARG user +ARG uid +ARG group +ARG guid +ARG hostname + +ENV BUILD_HOSTNAME $hostname +ENV USER $user + +RUN groupadd -f -r -g $guid $group + +RUN useradd $user -l -u $uid -g $guid -d /var/$user -m -s /bin/bash CMD bash # # Alternate approach - selective remove backends etc. diff --git a/dash-pipeline/dockerfiles/Dockerfile.saithrift-bldr b/dash-pipeline/dockerfiles/Dockerfile.saithrift-bldr index 3bee87e89..3596f2521 100644 --- a/dash-pipeline/dockerfiles/Dockerfile.saithrift-bldr +++ b/dash-pipeline/dockerfiles/Dockerfile.saithrift-bldr @@ -1,6 +1,6 @@ FROM chrissommers/dash-grpc:1.43.2 as grpc -FROM chrissommers/dash-bmv2-bldr:220630 as bmv2 +FROM chrissommers/dash-bmv2-bldr:220819 as bmv2 # amd64/ubuntu:20.04 on 2022-07-03 FROM amd64/ubuntu@sha256:b2339eee806d44d6a8adc0a790f824fb71f03366dd754d400316ae5a7e3ece3e as builder LABEL maintainer="SONiC-DASH Community " @@ -77,13 +77,14 @@ WORKDIR / ARG user ARG uid +ARG group ARG guid ARG hostname ENV BUILD_HOSTNAME $hostname ENV USER $user -RUN groupadd -f -r -g $guid g$user +RUN groupadd -f -r -g $guid $group RUN useradd $user -l -u $uid -g $guid -d /var/$user -m -s /bin/bash diff --git a/dash-pipeline/dockerfiles/Dockerfile.saithrift-client-bldr b/dash-pipeline/dockerfiles/Dockerfile.saithrift-client-bldr index c639f9352..b192a88a1 100644 --- a/dash-pipeline/dockerfiles/Dockerfile.saithrift-client-bldr +++ b/dash-pipeline/dockerfiles/Dockerfile.saithrift-client-bldr @@ -20,13 +20,14 @@ WORKDIR / ARG user ARG uid +ARG group ARG guid ARG hostname ENV BUILD_HOSTNAME $hostname ENV USER $user -RUN groupadd -f -r -g $guid g$user +RUN groupadd -f -r -g $guid $group RUN useradd $user -l -u $uid -g $guid -d /var/$user -m -s /bin/bash