diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml new file mode 100644 index 000000000..c32daf0aa --- /dev/null +++ b/.github/workflows/on-pr.yml @@ -0,0 +1,223 @@ +name: Faucet on pull-request tests + +on: [push, pull_request] + +env: + FAUCET_TEST_IMG: "faucet/tests" + SHARDARGS: "--privileged --sysctl net.ipv6.conf.all.disable_ipv6=0 --ulimit core=99999999999:99999999999 -v /var/local/lib/docker:/var/lib/docker" + FILES_CHANGED: "all" + CODECHECK_PY_VER: 3.6 + MATRIX_SHARDS: 11 + +jobs: + + unit-tests: + name: Unit tests + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.5, 3.6, 3.7, 3.8] + steps: + - name: Checkout source + uses: actions/checkout@v2.3.1 + - if: ${{ github.event.before != '0000000000000000000000000000000000000000' }} + name: Get file changes + id: file_changes + uses: trilom/file-changes-action@v1.2.4 + with: + output: ' ' + - if: ${{ steps.file_changes.outputs.files }} + name: Compare file changes + run: | + FILES_CHANGED="${{ steps.file_changes.outputs.files }}" + PY_FILES_CHANGED=$(echo "${{ steps.file_changes.outputs.files }}" | tr ' ' '\n' | grep -E ".py$" | tr '\n' ' ') + RQ_FILES_CHANGED=$(echo "${{ steps.file_changes.outputs.files }}" | tr ' ' '\n' | grep -E "requirements(.*)txt$" | tr '\n' ' ') + DOC_FILES_CHANGED=$(echo "${{ steps.file_changes.outputs.files }}" | tr ' ' '\n' | grep -E "docs/**" | tr '\n' ' ') + echo "Files changed: ${FILES_CHANGED}" + echo "Python code changed: ${PY_FILES_CHANGED}" + echo "Requirement changes: ${RQ_FILES_CHANGED}" + echo "Documentation changes: ${DOC_FILES_CHANGED}" + echo ::set-env name=FILES_CHANGED::${FILES_CHANGED} + echo ::set-env name=PY_FILES_CHANGED::${PY_FILES_CHANGED} + echo ::set-env name=RQ_FILES_CHANGED::${RQ_FILES_CHANGED} + echo ::set-env name=DOC_FILES_CHANGED::${DOC_FILES_CHANGED} + - name: Set up python-${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + ./docker/pip_deps.sh + pip3 install ./ + pip3 show faucet + - name: Running unit tests + run: ./tests/run_unit_tests.sh || exit 1 + - if: ${{ matrix.python-version == env.CODECHECK_PY_VER }} + name: Upload codecov + uses: codecov/codecov-action@v1.0.7 + - if: ${{ env.FILES_CHANGED == 'all' || env.RQ_FILES_CHANGED || env.PY_FILES_CHANGED }} + name: Pytype + run: | + cd ./tests/codecheck || exit 1 + if [[ "${{ env.FILES_CHANGED }}" == "all" || ! -z "${{ env.RQ_FILES_CHANGED }}" ]]; then + echo "Running pytype on everything" + ./pytype.sh || exit 1 + else + echo "Running pytype on ${{ env.PY_FILES_CHANGED }}" + ./pytype.sh ${{ env.PY_FILES_CHANGED }} || exit 1 + fi + + codecheck: + name: Code check + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v2.3.1 + - if: ${{ github.event.before != '0000000000000000000000000000000000000000' }} + name: Get file changes + id: file_changes + uses: trilom/file-changes-action@v1.2.4 + with: + output: ' ' + - if: ${{ steps.file_changes.outputs.files }} + name: Compare file changes + run: | + FILES_CHANGED="${{ steps.file_changes.outputs.files }}" + PY_FILES_CHANGED=$(echo "${{ steps.file_changes.outputs.files }}" | tr ' ' '\n' | grep -E ".py$" | tr '\n' ' ') + RQ_FILES_CHANGED=$(echo "${{ steps.file_changes.outputs.files }}" | tr ' ' '\n' | grep -E "requirements(.*)txt$" | tr '\n' ' ') + DOC_FILES_CHANGED=$(echo "${{ steps.file_changes.outputs.files }}" | tr ' ' '\n' | grep -E "docs/**" | tr '\n' ' ') + echo "Files changed: ${FILES_CHANGED}" + echo "Python code changed: ${PY_FILES_CHANGED}" + echo "Requirement changes: ${RQ_FILES_CHANGED}" + echo "Documentation changes: ${DOC_FILES_CHANGED}" + echo ::set-env name=FILES_CHANGED::${FILES_CHANGED} + echo ::set-env name=PY_FILES_CHANGED::${PY_FILES_CHANGED} + echo ::set-env name=RQ_FILES_CHANGED::${RQ_FILES_CHANGED} + echo ::set-env name=DOC_FILES_CHANGED::${DOC_FILES_CHANGED} + - name: Set up python-${{ env.CODECHECK_PY_VER }} + uses: actions/setup-python@v2 + with: + python-version: ${{ env.CODECHECK_PY_VER }} + - name: Install dependencies + run: | + ./docker/pip_deps.sh + - if: ${{ env.DOC_FILES_CHANGED }} + name: Build docs + run: | + cd ./docs || exit 1 + sudo apt-get install librsvg2-bin || exit 1 + make html || exit 1 + rm -rf _build + - if: ${{ env.FILES_CHANGED == 'all' || env.PY_FILES_CHANGED }} + name: Pylint + run: | + cd ./tests/codecheck || exit 1 + ./pylint.sh ${{ env.PY_FILES_CHANGED }} || exit 1 + + integration-tests: + name: Integration tests + runs-on: ubuntu-latest + strategy: + matrix: + MATRIX_SHARD: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + steps: + - name: Checkout source + uses: actions/checkout@v2.3.1 + - if: ${{ github.event.before != '0000000000000000000000000000000000000000' }} + name: Get file changes + id: file_changes + uses: trilom/file-changes-action@v1.2.4 + with: + output: ' ' + - if: ${{ steps.file_changes.outputs.files }} + name: Compare file changes + run: | + FILES_CHANGED="${{ steps.file_changes.outputs.files }}" + PY_FILES_CHANGED=$(echo "${{ steps.file_changes.outputs.files }}" | tr ' ' '\n' | grep -E ".py$" | tr '\n' ' ') + RQ_FILES_CHANGED=$(echo "${{ steps.file_changes.outputs.files }}" | tr ' ' '\n' | grep -E "requirements(.*)txt$" | tr '\n' ' ') + DOC_FILES_CHANGED=$(echo "${{ steps.file_changes.outputs.files }}" | tr ' ' '\n' | grep -E "docs/**" | tr '\n' ' ') + echo "Files changed: ${FILES_CHANGED}" + echo "Python code changed: ${PY_FILES_CHANGED}" + echo "Requirement changes: ${RQ_FILES_CHANGED}" + echo "Documentation changes: ${DOC_FILES_CHANGED}" + echo ::set-env name=FILES_CHANGED::${FILES_CHANGED} + echo ::set-env name=PY_FILES_CHANGED::${PY_FILES_CHANGED} + echo ::set-env name=RQ_FILES_CHANGED::${RQ_FILES_CHANGED} + echo ::set-env name=DOC_FILES_CHANGED::${DOC_FILES_CHANGED} + - if: ${{ env.FILES_CHANGED == 'all' || env.PY_FILES_CHANGED || env.RQ_FILES_CHANGED }} + name: Shard out tests + run: | + ALLTESTFILES="tests/integration/mininet_tests.py tests/integration/mininet_multidp_tests.py clib/clib_mininet_tests.py" + ALLTESTS=$(grep -E -o "^class (Faucet[a-zA-Z0-9]+Test)" ${ALLTESTFILES} | cut -f2 -d" " | sort) + declare -A sharded + function shard { + work=$1 + workers=$2 + i=0 + for shard in $work ; do + i=$(( i % workers )) + sharded[$i]="${sharded[$i]} $shard" + i=$(( i + 1 )) + done + } + shard "$ALLTESTS" ${{ env.MATRIX_SHARDS }} + FAUCET_TESTS="-din ${sharded[${{ matrix.MATRIX_SHARD }}]}" + echo ::set-env name=FAUCET_TESTS::${FAUCET_TESTS} + - if: ${{ env.FAUCET_TESTS }} + name: Build docker + run: docker build --pull -t ${FAUCET_TEST_IMG} -f Dockerfile.tests . || exit 1 + - if: ${{ env.FAUCET_TESTS }} + name: Setup docker + run: | + ulimit -c unlimited + echo '/var/tmp/core.%h.%e.%t' | sudo tee /proc/sys/kernel/core_pattern + sudo modprobe openvswitch + sudo modprobe ebtables + - if: ${{ env.FAUCET_TESTS }} + name: Run docker + run: sudo docker run ${SHARDARGS} -v $HOME/.cache/pip:/var/tmp/pip-cache -e FAUCET_TESTS="${FAUCET_TESTS}" -t ${FAUCET_TEST_IMG} || exit 1 + - name: Detect core dumps + run: if [ ls -1 /var/tmp/core* > /dev/null 2>&1 ]; then exit 1; fi + + sanity-testing: + name: Sanity tests + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v2.3.1 + - if: ${{ github.event.before != '0000000000000000000000000000000000000000' }} + name: Get file changes + id: file_changes + uses: trilom/file-changes-action@v1.2.4 + with: + output: ' ' + - if: ${{ steps.file_changes.outputs.files }} + name: Compare file changes + run: | + FILES_CHANGED="${{ steps.file_changes.outputs.files }}" + PY_FILES_CHANGED=$(echo "${{ steps.file_changes.outputs.files }}" | tr ' ' '\n' | grep -E ".py$" | tr '\n' ' ') + RQ_FILES_CHANGED=$(echo "${{ steps.file_changes.outputs.files }}" | tr ' ' '\n' | grep -E "requirements(.*)txt$" | tr '\n' ' ') + DOC_FILES_CHANGED=$(echo "${{ steps.file_changes.outputs.files }}" | tr ' ' '\n' | grep -E "docs/**" | tr '\n' ' ') + echo "Files changed: ${FILES_CHANGED}" + echo "Python code changed: ${PY_FILES_CHANGED}" + echo "Requirement changes: ${RQ_FILES_CHANGED}" + echo "Documentation changes: ${DOC_FILES_CHANGED}" + echo ::set-env name=FILES_CHANGED::${FILES_CHANGED} + echo ::set-env name=PY_FILES_CHANGED::${PY_FILES_CHANGED} + echo ::set-env name=RQ_FILES_CHANGED::${RQ_FILES_CHANGED} + echo ::set-env name=DOC_FILES_CHANGED::${DOC_FILES_CHANGED} + - if: ${{ env.FILES_CHANGED == 'all' || env.PY_FILES_CHANGED || env.RQ_FILES_CHANGED }} + name: Setup docker test requirements + run: | + ulimit -c unlimited + echo '/var/tmp/core.%h.%e.%t' | sudo tee /proc/sys/kernel/core_pattern + sudo modprobe openvswitch + sudo modprobe ebtables + - if: ${{ env.FILES_CHANGED == 'all' || env.PY_FILES_CHANGED || env.RQ_FILES_CHANGED }} + name: Build docker + run: docker build --pull -t ${FAUCET_TEST_IMG} -f Dockerfile.tests . || exit 1 + - if: ${{ env.FILES_CHANGED == 'all' || env.PY_FILES_CHANGED || env.RQ_FILES_CHANGED }} + name: Run docker + run: sudo docker run ${SHARDARGS} -v $HOME/.cache/pip:/var/tmp/pip-cache -e FAUCET_TESTS="-ni FaucetSanityTest FaucetStackStringOfDPUntaggedTest" -e HWTESTS="1" -t ${FAUCET_TEST_IMG} || exit 1 + - name: Detect core dumps + run: if [ ls -1 /var/tmp/core* > /dev/null 2>&1 ]; then exit 1; fi diff --git a/.github/workflows/periodic.yml b/.github/workflows/periodic.yml new file mode 100644 index 000000000..72f59714d --- /dev/null +++ b/.github/workflows/periodic.yml @@ -0,0 +1,35 @@ +name: Faucet scheduled tests + +on: + schedule: + # NZST = UTC + 12 + # Schedule to run at midnight & lunch-time + - cron: '0 0,12 * * *' + +env: + FAUCET_TEST_IMG: "faucet/tests" + SHARDARGS: "--privileged --sysctl net.ipv6.conf.all.disable_ipv6=0 --ulimit core=99999999999:99999999999 -v /var/local/lib/docker:/var/lib/docker" + +jobs: + + generative-testing: + name: Generative tests + runs-on: ubuntu-latest + strategy: + matrix: + MATRIX_SHARD: [--generative_unit, --generative_integration] + steps: + - name: Checkout source + uses: actions/checkout@v2.3.1 + - name: Setup docker test requirements + run: | + ulimit -c unlimited + echo '/var/tmp/core.%h.%e.%t' | sudo tee /proc/sys/kernel/core_pattern + sudo modprobe openvswitch + sudo modprobe ebtables + - name: Build docker + run: docker build --pull -t ${FAUCET_TEST_IMG} -f Dockerfile.tests . || exit 1 + - name: Run docker + run: sudo docker run ${SHARDARGS} -v $HOME/.cache/pip:/var/tmp/pip-cache -e FAUCET_TESTS="${{ matrix.MATRIX_SHARD }}" -t ${FAUCET_TEST_IMG} || exit 1 + - name: Detect core dumps + run: if [ ls -1 /var/tmp/core* > /dev/null 2>&1 ]; then exit 1; fi diff --git a/.travis.yml b/.travis.yml index 37c26e933..3db23b3c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,81 +1,15 @@ dist: bionic sudo: required -addons: - apt: - update: true - packages: - - librsvg2-bin - - parallel cache: pip: true stages: - - test - name: deploy if: repo = faucetsdn/faucet AND tag IS present -env: - global: - - FAUCET_TEST_IMG=faucet/tests - - MATRIX_SHARDS=6 -script: - - ./travis/runtests.sh jobs: include: - - &unit-test - stage: test - language: python - python: 3.5 - env: - - MATRIX_SHARD=unittest - - PYTYPE=true - - <<: *unit-test - python: 3.6 - env: - - MATRIX_SHARD=unittest - - BUILD_DOCS=true - - CODECOV_UPLOAD=true - - PYLINT=true - - PYTYPE=true - - <<: *unit-test - python: 3.7 - env: - - MATRIX_SHARD=unittest - - PYTYPE=true - - <<: *unit-test - python: 3.8 - env: - - MATRIX_SHARD=unittest - - PYTYPE=true - - &integration-test - stage: test - env: MATRIX_SHARD=sanity - language: bash - before_install: - - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - sudo apt-get update - - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce - services: - - docker - - <<: *integration-test - env: MATRIX_SHARD=0 - - <<: *integration-test - env: MATRIX_SHARD=1 - - <<: *integration-test - env: MATRIX_SHARD=2 - - <<: *integration-test - env: MATRIX_SHARD=3 - - <<: *integration-test - env: MATRIX_SHARD=4 - - <<: *integration-test - env: MATRIX_SHARD=5 - - <<: *integration-test - env: MATRIX_SHARD=generative-unit - - <<: *integration-test - env: MATRIX_SHARD=generative-integration - stage: deploy language: python python: 3.6 - script: skip deploy: provider: pypi on: diff --git a/docker/fuzz_config.sh b/docker/fuzz_config.sh index 4c804ed94..c4897e2ac 100755 --- a/docker/fuzz_config.sh +++ b/docker/fuzz_config.sh @@ -14,4 +14,4 @@ if [ -e "$checkfile" ]; then fi fi -AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 AFL_SKIP_CPUFREQ=1 py-afl-fuzz -x "$dictfile" -m 5000 -i "$inputfile" -o "$outputfile" -- /usr/bin/python3 /faucet-src/tests/fuzzer/fuzz_config.py +AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 AFL_SKIP_CPUFREQ=1 py-afl-fuzz -x "$dictfile" -m 5000 -i "$inputfile" -o "$outputfile" -- /usr/bin/python3 /faucet-src/tests/generative/fuzzer/fuzz_config.py diff --git a/docker/fuzz_packet.sh b/docker/fuzz_packet.sh index 18c50a691..aa4e8d0a7 100755 --- a/docker/fuzz_packet.sh +++ b/docker/fuzz_packet.sh @@ -14,4 +14,4 @@ if [ -e "$checkfile" ]; then fi fi -AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 AFL_SKIP_CPUFREQ=1 py-afl-fuzz -m 5000 -x "$dictfile" -i "$inputfile" -o "$outputfile" -- /usr/bin/python3 /faucet-src/tests/fuzzer/fuzz_packet.py +AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 AFL_SKIP_CPUFREQ=1 py-afl-fuzz -m 5000 -x "$dictfile" -i "$inputfile" -o "$outputfile" -- /usr/bin/python3 /faucet-src/tests/generative/fuzzer/fuzz_packet.py diff --git a/tests/fuzzer/config/ex1 b/tests/generative/fuzzer/config/ex1 similarity index 100% rename from tests/fuzzer/config/ex1 rename to tests/generative/fuzzer/config/ex1 diff --git a/tests/fuzzer/config/ex10 b/tests/generative/fuzzer/config/ex10 similarity index 100% rename from tests/fuzzer/config/ex10 rename to tests/generative/fuzzer/config/ex10 diff --git a/tests/fuzzer/config/ex11 b/tests/generative/fuzzer/config/ex11 similarity index 100% rename from tests/fuzzer/config/ex11 rename to tests/generative/fuzzer/config/ex11 diff --git a/tests/fuzzer/config/ex12 b/tests/generative/fuzzer/config/ex12 similarity index 100% rename from tests/fuzzer/config/ex12 rename to tests/generative/fuzzer/config/ex12 diff --git a/tests/fuzzer/config/ex14 b/tests/generative/fuzzer/config/ex14 similarity index 100% rename from tests/fuzzer/config/ex14 rename to tests/generative/fuzzer/config/ex14 diff --git a/tests/fuzzer/config/ex2 b/tests/generative/fuzzer/config/ex2 similarity index 100% rename from tests/fuzzer/config/ex2 rename to tests/generative/fuzzer/config/ex2 diff --git a/tests/fuzzer/config/ex3 b/tests/generative/fuzzer/config/ex3 similarity index 100% rename from tests/fuzzer/config/ex3 rename to tests/generative/fuzzer/config/ex3 diff --git a/tests/fuzzer/config/ex4 b/tests/generative/fuzzer/config/ex4 similarity index 100% rename from tests/fuzzer/config/ex4 rename to tests/generative/fuzzer/config/ex4 diff --git a/tests/fuzzer/config/ex5 b/tests/generative/fuzzer/config/ex5 similarity index 100% rename from tests/fuzzer/config/ex5 rename to tests/generative/fuzzer/config/ex5 diff --git a/tests/fuzzer/config/ex6 b/tests/generative/fuzzer/config/ex6 similarity index 100% rename from tests/fuzzer/config/ex6 rename to tests/generative/fuzzer/config/ex6 diff --git a/tests/fuzzer/config/ex7 b/tests/generative/fuzzer/config/ex7 similarity index 100% rename from tests/fuzzer/config/ex7 rename to tests/generative/fuzzer/config/ex7 diff --git a/tests/fuzzer/config/ex8 b/tests/generative/fuzzer/config/ex8 similarity index 100% rename from tests/fuzzer/config/ex8 rename to tests/generative/fuzzer/config/ex8 diff --git a/tests/fuzzer/config/ex9 b/tests/generative/fuzzer/config/ex9 similarity index 100% rename from tests/fuzzer/config/ex9 rename to tests/generative/fuzzer/config/ex9 diff --git a/tests/fuzzer/dict/config.dict b/tests/generative/fuzzer/dict/config.dict similarity index 100% rename from tests/fuzzer/dict/config.dict rename to tests/generative/fuzzer/dict/config.dict diff --git a/tests/fuzzer/dict/packet.dict b/tests/generative/fuzzer/dict/packet.dict similarity index 100% rename from tests/fuzzer/dict/packet.dict rename to tests/generative/fuzzer/dict/packet.dict diff --git a/tests/fuzzer/display_packet_crash.py b/tests/generative/fuzzer/display_packet_crash.py similarity index 94% rename from tests/fuzzer/display_packet_crash.py rename to tests/generative/fuzzer/display_packet_crash.py index 9d386b15c..7098debbb 100644 --- a/tests/fuzzer/display_packet_crash.py +++ b/tests/generative/fuzzer/display_packet_crash.py @@ -52,4 +52,4 @@ def main(): if len(sys.argv) == 2: main() else: - print('USAGE: python3 display_packet_crash.py ') + sys.stderr.write('USAGE: python3 display_packet_crash.py \n') diff --git a/tests/fuzzer/fake_packet.py b/tests/generative/fuzzer/fake_packet.py similarity index 74% rename from tests/fuzzer/fake_packet.py rename to tests/generative/fuzzer/fake_packet.py index 6bce13f09..6cdb19467 100644 --- a/tests/fuzzer/fake_packet.py +++ b/tests/generative/fuzzer/fake_packet.py @@ -1,12 +1,16 @@ """Fake classes for the packet fuzzer to use""" -class RyuEvent: # pylint: disable=too-few-public-methods + +class RyuEvent: # pylint: disable=too-few-public-methods """Fake ryuevent class""" + def __init__(self, msg): self.msg = msg -class Message: # pylint: disable=too-few-public-methods + +class Message: # pylint: disable=too-few-public-methods """Fake message class""" + def __init__(self, *args, **kwargs): self.datapath = kwargs['datapath'] self.cookie = kwargs['cookie'] @@ -16,7 +20,9 @@ def __init__(self, *args, **kwargs): self.match = kwargs self.args = args -class Datapath: # pylint: disable=too-few-public-methods + +class Datapath: # pylint: disable=too-few-public-methods """Fake datapath class""" + def __init__(self, dp_id): self.dp_id = dp_id diff --git a/tests/fuzzer/fuzz_config.py b/tests/generative/fuzzer/fuzz_config.py similarity index 92% rename from tests/fuzzer/fuzz_config.py rename to tests/generative/fuzzer/fuzz_config.py index f548135bd..cd469d9cf 100644 --- a/tests/fuzzer/fuzz_config.py +++ b/tests/generative/fuzzer/fuzz_config.py @@ -26,7 +26,7 @@ def create_config_file(config): def main(): logging.disable(logging.CRITICAL) - while afl.loop(ROUNDS): # pylint: disable=c-extension-no-member + while afl.loop(ROUNDS): # pylint: disable=c-extension-no-member config = sys.stdin.read() file_name = create_config_file(config) try: diff --git a/tests/fuzzer/fuzz_packet.py b/tests/generative/fuzzer/fuzz_packet.py similarity index 95% rename from tests/fuzzer/fuzz_packet.py rename to tests/generative/fuzzer/fuzz_packet.py index 703e72de0..5a69b562b 100644 --- a/tests/fuzzer/fuzz_packet.py +++ b/tests/generative/fuzzer/fuzz_packet.py @@ -30,7 +30,7 @@ def main(): valve.dp.running = True valve.dp.dyn_finalized = state - while afl.loop(ROUNDS): # pylint: disable=c-extension-no-member + while afl.loop(ROUNDS): # pylint: disable=c-extension-no-member # receive input from afl rcv = sys.stdin.read() data = None diff --git a/tests/fuzzer/packet/aoe.ex1 b/tests/generative/fuzzer/packet/aoe.ex1 similarity index 100% rename from tests/fuzzer/packet/aoe.ex1 rename to tests/generative/fuzzer/packet/aoe.ex1 diff --git a/tests/fuzzer/packet/arp.ex1 b/tests/generative/fuzzer/packet/arp.ex1 similarity index 100% rename from tests/fuzzer/packet/arp.ex1 rename to tests/generative/fuzzer/packet/arp.ex1 diff --git a/tests/fuzzer/packet/arp.ex2 b/tests/generative/fuzzer/packet/arp.ex2 similarity index 100% rename from tests/fuzzer/packet/arp.ex2 rename to tests/generative/fuzzer/packet/arp.ex2 diff --git a/tests/fuzzer/packet/asap.ex1 b/tests/generative/fuzzer/packet/asap.ex1 similarity index 100% rename from tests/fuzzer/packet/asap.ex1 rename to tests/generative/fuzzer/packet/asap.ex1 diff --git a/tests/fuzzer/packet/asap.ex2 b/tests/generative/fuzzer/packet/asap.ex2 similarity index 100% rename from tests/fuzzer/packet/asap.ex2 rename to tests/generative/fuzzer/packet/asap.ex2 diff --git a/tests/fuzzer/packet/diameter.ex1 b/tests/generative/fuzzer/packet/diameter.ex1 similarity index 100% rename from tests/fuzzer/packet/diameter.ex1 rename to tests/generative/fuzzer/packet/diameter.ex1 diff --git a/tests/fuzzer/packet/dns.ex1 b/tests/generative/fuzzer/packet/dns.ex1 similarity index 100% rename from tests/fuzzer/packet/dns.ex1 rename to tests/generative/fuzzer/packet/dns.ex1 diff --git a/tests/fuzzer/packet/dns.ex2 b/tests/generative/fuzzer/packet/dns.ex2 similarity index 100% rename from tests/fuzzer/packet/dns.ex2 rename to tests/generative/fuzzer/packet/dns.ex2 diff --git a/tests/fuzzer/packet/http.ex1 b/tests/generative/fuzzer/packet/http.ex1 similarity index 100% rename from tests/fuzzer/packet/http.ex1 rename to tests/generative/fuzzer/packet/http.ex1 diff --git a/tests/fuzzer/packet/http.ex2 b/tests/generative/fuzzer/packet/http.ex2 similarity index 100% rename from tests/fuzzer/packet/http.ex2 rename to tests/generative/fuzzer/packet/http.ex2 diff --git a/tests/fuzzer/packet/icmp.ex1 b/tests/generative/fuzzer/packet/icmp.ex1 similarity index 100% rename from tests/fuzzer/packet/icmp.ex1 rename to tests/generative/fuzzer/packet/icmp.ex1 diff --git a/tests/fuzzer/packet/icmp.ex2 b/tests/generative/fuzzer/packet/icmp.ex2 similarity index 100% rename from tests/fuzzer/packet/icmp.ex2 rename to tests/generative/fuzzer/packet/icmp.ex2 diff --git a/tests/fuzzer/packet/icmp.ex3 b/tests/generative/fuzzer/packet/icmp.ex3 similarity index 100% rename from tests/fuzzer/packet/icmp.ex3 rename to tests/generative/fuzzer/packet/icmp.ex3 diff --git a/tests/fuzzer/packet/icmp.ex4 b/tests/generative/fuzzer/packet/icmp.ex4 similarity index 100% rename from tests/fuzzer/packet/icmp.ex4 rename to tests/generative/fuzzer/packet/icmp.ex4 diff --git a/tests/fuzzer/packet/igmpv2.ex1 b/tests/generative/fuzzer/packet/igmpv2.ex1 similarity index 100% rename from tests/fuzzer/packet/igmpv2.ex1 rename to tests/generative/fuzzer/packet/igmpv2.ex1 diff --git a/tests/fuzzer/packet/ipv4.ex1 b/tests/generative/fuzzer/packet/ipv4.ex1 similarity index 100% rename from tests/fuzzer/packet/ipv4.ex1 rename to tests/generative/fuzzer/packet/ipv4.ex1 diff --git a/tests/fuzzer/packet/ipv6.ex1 b/tests/generative/fuzzer/packet/ipv6.ex1 similarity index 100% rename from tests/fuzzer/packet/ipv6.ex1 rename to tests/generative/fuzzer/packet/ipv6.ex1 diff --git a/tests/fuzzer/packet/irc.ex1 b/tests/generative/fuzzer/packet/irc.ex1 similarity index 100% rename from tests/fuzzer/packet/irc.ex1 rename to tests/generative/fuzzer/packet/irc.ex1 diff --git a/tests/fuzzer/packet/irc.ex2 b/tests/generative/fuzzer/packet/irc.ex2 similarity index 100% rename from tests/fuzzer/packet/irc.ex2 rename to tests/generative/fuzzer/packet/irc.ex2 diff --git a/tests/fuzzer/packet/irc.ex3 b/tests/generative/fuzzer/packet/irc.ex3 similarity index 100% rename from tests/fuzzer/packet/irc.ex3 rename to tests/generative/fuzzer/packet/irc.ex3 diff --git a/tests/fuzzer/packet/lacp.ex1 b/tests/generative/fuzzer/packet/lacp.ex1 similarity index 100% rename from tests/fuzzer/packet/lacp.ex1 rename to tests/generative/fuzzer/packet/lacp.ex1 diff --git a/tests/fuzzer/packet/msger.ex1 b/tests/generative/fuzzer/packet/msger.ex1 similarity index 100% rename from tests/fuzzer/packet/msger.ex1 rename to tests/generative/fuzzer/packet/msger.ex1 diff --git a/tests/fuzzer/packet/tcp.ex1 b/tests/generative/fuzzer/packet/tcp.ex1 similarity index 100% rename from tests/fuzzer/packet/tcp.ex1 rename to tests/generative/fuzzer/packet/tcp.ex1 diff --git a/tests/fuzzer/packet/tcp.ex2 b/tests/generative/fuzzer/packet/tcp.ex2 similarity index 100% rename from tests/fuzzer/packet/tcp.ex2 rename to tests/generative/fuzzer/packet/tcp.ex2 diff --git a/tests/fuzzer/packet/tcp.ex3 b/tests/generative/fuzzer/packet/tcp.ex3 similarity index 100% rename from tests/fuzzer/packet/tcp.ex3 rename to tests/generative/fuzzer/packet/tcp.ex3 diff --git a/tests/fuzzer/packet/tcp.ex4 b/tests/generative/fuzzer/packet/tcp.ex4 similarity index 100% rename from tests/fuzzer/packet/tcp.ex4 rename to tests/generative/fuzzer/packet/tcp.ex4 diff --git a/tests/fuzzer/packet/tcp.ex5 b/tests/generative/fuzzer/packet/tcp.ex5 similarity index 100% rename from tests/fuzzer/packet/tcp.ex5 rename to tests/generative/fuzzer/packet/tcp.ex5 diff --git a/tests/fuzzer/packet/udp.ex1 b/tests/generative/fuzzer/packet/udp.ex1 similarity index 100% rename from tests/fuzzer/packet/udp.ex1 rename to tests/generative/fuzzer/packet/udp.ex1 diff --git a/tests/fuzzer/packet/udp.ex2 b/tests/generative/fuzzer/packet/udp.ex2 similarity index 100% rename from tests/fuzzer/packet/udp.ex2 rename to tests/generative/fuzzer/packet/udp.ex2 diff --git a/tests/fuzzer/packet/udp.ex3 b/tests/generative/fuzzer/packet/udp.ex3 similarity index 100% rename from tests/fuzzer/packet/udp.ex3 rename to tests/generative/fuzzer/packet/udp.ex3 diff --git a/tests/fuzzer/packet/udp.ex4 b/tests/generative/fuzzer/packet/udp.ex4 similarity index 100% rename from tests/fuzzer/packet/udp.ex4 rename to tests/generative/fuzzer/packet/udp.ex4 diff --git a/tests/generative/integration/mininet_tests.py b/tests/generative/integration/mininet_tests.py index b7d55247b..12048db60 100644 --- a/tests/generative/integration/mininet_tests.py +++ b/tests/generative/integration/mininet_tests.py @@ -23,7 +23,7 @@ class FaucetFaultToleranceBaseTest(FaucetTopoTestBase): for a fault to build the full graph for the current fault. ASSUME_SYMMETRIC_PING: A simplification can assume that (h1 -> h2) implies (h2 -> h1). - Set to true assume that host connectivity is symmetric. + Set to true to assume that host connectivity is symmetric. INTERVLAN_ONLY: Set to true to test only the inter-VLAN connectivity; ignore connections between hosts on the same VLAN. Speed up the inter-VLAN testing by ignoring the intra-VLAN cases for @@ -36,7 +36,7 @@ class FaucetFaultToleranceBaseTest(FaucetTopoTestBase): IGNORE_SUBGRAPH: Assume for a topology with subgraphs, the subgraphs do not need to be tested (if they have already been tested) """ - INSTANT_FAIL = False + INSTANT_FAIL = True ASSUME_SYMMETRIC_PING = True INTERVLAN_ONLY = False @@ -330,6 +330,8 @@ class FaucetSingleFaultTolerance2DPTest(FaucetFaultToleranceBaseTest): N_DP_LINKS = 1 STACK_ROOTS = {0: 1} + ASSUME_SYMMETRIC_PING = False + class FaucetSingleFaultTolerance3DPTest(FaucetFaultToleranceBaseTest): """Run a range of fault-tolerance tests for topologies on 3 DPs""" @@ -386,7 +388,6 @@ def test_ftp2_destroying_one_of_each_link(self): self.N_DP_LINKS = 1 -@unittest.skip('Too expensive for Travis to run') class FaucetSingleFaultTolerance5DPTest(FaucetFaultToleranceBaseTest): """Run a range of fault-tolerance tests for topologies on 5 DPs""" @@ -397,7 +398,7 @@ class FaucetSingleFaultTolerance5DPTest(FaucetFaultToleranceBaseTest): STACK_ROOTS = {0: 1} -@unittest.skip('Too expensive for Travis to run') +@unittest.skip('Too computationally complex') class FaucetSingleFaultTolerance6DPTest(FaucetFaultToleranceBaseTest): """Run a range of fault-tolerance tests for topologies on 5 DPs""" @@ -408,7 +409,7 @@ class FaucetSingleFaultTolerance6DPTest(FaucetFaultToleranceBaseTest): STACK_ROOTS = {0: 1} -@unittest.skip('Too expensive for Travis to run') +@unittest.skip('Too computationally complex') class FaucetSingleFaultTolerance7DPTest(FaucetFaultToleranceBaseTest): """Run a range of fault-tolerance tests for topologies on 5 DPs""" diff --git a/tests/generative/unit/test_topology.py b/tests/generative/unit/test_topology.py index 85c94f6b0..db7de9ab4 100755 --- a/tests/generative/unit/test_topology.py +++ b/tests/generative/unit/test_topology.py @@ -219,9 +219,6 @@ def test(self): for num_dps, nl in GRAPHS.items(): chunk = 50 batch = 1 - if num_dps > 6: - # TODO: Travis can't handle much more than 6 without timing out - break for test_class in (ValveTopologyRestartTest, ): for i in range(0, len(nl), chunk): test_nl = [nl[0]] + nl[i:i + chunk] diff --git a/tests/integration/mininet_multidp_tests.py b/tests/integration/mininet_multidp_tests.py index 1d4ee13e3..9230744df 100644 --- a/tests/integration/mininet_multidp_tests.py +++ b/tests/integration/mininet_multidp_tests.py @@ -1822,79 +1822,53 @@ def test_mclag_vip_connectivity(self): self.verify_stack_up() self.verify_lag_connectivity(self.LACP_HOST) - def restart_on_down_lag_port(self, port_dp_index, cold_start_dp_index): - """Down a port on port_dpid_index, cold-start on cold_start_dp, UP previous port""" - # Bring a LACP port DOWN - chosen_dpid = self.dpids[port_dp_index] - port_no = self.host_port_maps[self.LACP_HOST][port_dp_index][0] - self.set_port_down(port_no, chosen_dpid) - self.verify_num_lag_up_ports(1, chosen_dpid) - # Cold start switch, cold-start twice to get back to initial condition - conf = self._get_faucet_conf() - interfaces_conf = conf['dps'][self.topo.switches_by_id[cold_start_dp_index]]['interfaces'] - for port, port_conf in interfaces_conf.items(): - if 'lacp' not in port_conf and 'stack' not in port_conf: - # Change host VLAN to enable cold-starting on faucet-2 - curr_vlan = port_conf['native_vlan'] - port_conf['native_vlan'] = ( - self.topo.vlan_name(1) - if curr_vlan == self.topo.vlan_name(0) else self.topo.vlan_name(0)) - # VLAN changed so just delete the host information instead of recomputing - # routes etc.. - for _id in self.host_information: - if cold_start_dp_index in self.host_port_maps[_id]: - ports = self.host_port_maps[_id][cold_start_dp_index] - if port in ports: - del self.host_information[_id] - break - break - self.reload_conf( - conf, self.faucet_config_path, restart=True, - cold_start=True, change_expected=False) - # Bring LACP port UP - self.set_port_up(port_no, chosen_dpid) - self.verify_num_lag_up_ports(2, chosen_dpid) - # Take down all of the other ports - for dp_i, ports in self.host_port_maps[self.LACP_HOST].items(): - dpid = self.topo.dpids_by_id[dp_i] - if dpid != chosen_dpid: - for port in ports: - if port != port_no: - self.set_port_down(port, dpid) - - def test_mclag_coldstart(self): - """Test LACP MCLAG after a cold start""" - lacp_host_links = [0, 0, 1, 1] - self.set_up(lacp_host_links) - self.verify_stack_up() - self.verify_lag_host_connectivity() - self.restart_on_down_lag_port(1, 1) - self.verify_lag_host_connectivity() - def test_mclag_warmstart(self): """Test LACP MCLAG after a warm start""" lacp_host_links = [0, 0, 1, 1] self.set_up(lacp_host_links) + + # Perform initial test self.verify_stack_up() self.verify_lag_host_connectivity() - self.restart_on_down_lag_port(0, 1) + + # Take down single LAG port + self.set_port_down(self.host_port_maps[self.LACP_HOST][0][0], self.dpids[0]) + self.verify_num_lag_up_ports(1, self.dpids[0]) + + # Force warm start on switch by changing native VLAN of host1 + conf = self._get_faucet_conf() + interfaces_conf = conf['dps'][self.topo.switches_by_id[0]]['interfaces'] + interfaces_conf[self.host_port_maps[1][0][0]]['native_vlan'] = self.topo.vlan_name(0) + self.reload_conf( + conf, self.faucet_config_path, restart=True, + cold_start=False, change_expected=False) + self.host_information.pop(1) + + # Set a single LAG port back UP + self.set_port_up(self.host_port_maps[self.LACP_HOST][0][0], self.dpids[0]) + self.verify_num_lag_up_ports(2, self.dpids[0]) + + # Verify connectivity self.verify_lag_host_connectivity() def test_mclag_portrestart(self): """Test LACP MCLAG after a port gets restarted""" lacp_host_links = [0, 0, 1, 1] self.set_up(lacp_host_links) + + # Perform initial test self.verify_stack_up() self.verify_lag_host_connectivity() - chosen_dpid = self.dpids[0] - port_no = self.host_port_maps[self.LACP_HOST][0][0] - self.set_port_down(port_no, chosen_dpid) - self.set_port_up(port_no, chosen_dpid) - for dp_i, ports in self.host_port_maps[self.LACP_HOST].items(): - dpid = self.topo.dpids_by_id[dp_i] - for port in ports: - if dpid != chosen_dpid and port != port_no: - self.set_port_down(port, dpid) + + # Set LAG port down + self.set_port_down(self.host_port_maps[self.LACP_HOST][0][0], self.dpids[0]) + self.verify_num_lag_up_ports(1, self.dpids[0]) + + # Set LAG port back up + self.set_port_up(self.host_port_maps[self.LACP_HOST][0][0], self.dpids[0]) + self.verify_num_lag_up_ports(2, self.dpids[0]) + + # Verify connectivity self.verify_lag_host_connectivity() diff --git a/tests/run_unit_tests.sh b/tests/run_unit_tests.sh index eee26725c..b39ec4004 100755 --- a/tests/run_unit_tests.sh +++ b/tests/run_unit_tests.sh @@ -12,4 +12,5 @@ SRCFILES="find $TESTDIR/unit/*/test_*py $TESTDIR/integration/experimental_api_te coverage erase || exit 1 $SRCFILES | xargs realpath | shuf | parallel --timeout 300 --delay 1 --bar --halt now,fail=1 -j 2 $TESTCMD || exit 1 coverage combine +coverage xml coverage report -m --fail-under=$MINCOVERAGE || exit 1 diff --git a/travis/runtests.sh b/travis/runtests.sh deleted file mode 100755 index 15c8633ca..000000000 --- a/travis/runtests.sh +++ /dev/null @@ -1,154 +0,0 @@ -#!/bin/bash - -# See https://docs.travis-ci.com/user/environment-variables/#convenience-variables -echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}" -echo "TRAVIS_COMMIT: ${TRAVIS_COMMIT}" -echo "TRAVIS_PULL_REQUEST: ${TRAVIS_PULL_REQUEST}" - -# If FILES_CHANGED is set to all, run codecheck tests on all files, -# otherwise only run on changed files listed in PY_FILES_CHANGED -FILES_CHANGED="all" -PY_FILES_CHANGED="" -RQ_FILES_CHANGED="" - -if [ ! -z "${TRAVIS_COMMIT_RANGE}" ]; then - # This isn't a new branch - - echo "TRAVIS_COMMIT_RANGE: ${TRAVIS_COMMIT_RANGE}" - - if [ "${TRAVIS_PULL_REQUEST}" == "false" ]; then - # This isn't a PR build - - # We need the individual commits to detect force pushes - COMMIT1="$(echo "${TRAVIS_COMMIT_RANGE}" | cut -f 1 -d '.')" - COMMIT2="$(echo "${TRAVIS_COMMIT_RANGE}" | cut -f 4 -d '.')" - - if [ "$(git cat-file -t "${COMMIT1}" 2>/dev/null)" == "commit" ] && [ "$(git cat-file -t "${COMMIT2}" 2>/dev/null)" == "commit" ]; then - # Both commits exist, this isn't a rewrite of history - COMMIT_RANGE="${TRAVIS_COMMIT_RANGE}" - fi - else - # This is a PR build - COMMIT_RANGE="${TRAVIS_BRANCH}...HEAD" - fi - - if [ ! -z "${COMMIT_RANGE}" ]; then - GIT_DIFF_CMD="git diff --diff-filter=ACMRT --name-only ${COMMIT_RANGE} --" - FILES_CHANGED=$(${GIT_DIFF_CMD} | tr '\n' ' ') - PY_FILES_CHANGED=$(${GIT_DIFF_CMD} | grep -E ".py$" | tr '\n' ' ') - RQ_FILES_CHANGED=$(${GIT_DIFF_CMD} | grep -E "requirements(.*)txt$" | tr '\n' ' ') - fi -fi - -if [ "${FILES_CHANGED}" != "all" ]; then - echo "These files have changed between ${COMMIT_RANGE}: ${FILES_CHANGED}" - [ ! -z "${PY_FILES_CHANGED}" ] && echo "Python code changes: ${PY_FILES_CHANGED}" - [ ! -z "${RQ_FILES_CHANGED}" ] && echo "Python requirements changes: ${RQ_FILES_CHANGED}" -fi - -if [ "${MATRIX_SHARD}" == "unittest" ] ; then - ./docker/pip_deps.sh - pip3 install ./ - pip3 show faucet - ./tests/run_unit_tests.sh || exit 1 - - if [ "${CODECOV_UPLOAD}" == "true" ] ; then - codecov || true - fi - - if [ "${BUILD_DOCS}" == "true" ] ; then - ( - cd ./docs || exit 1 - make html || exit 1 - rm -rf _build - ) - fi - - if [ "${PYLINT}" == "true" ] ; then - if [ "${FILES_CHANGED}" == "all" ] || [ ! -z "${PY_FILES_CHANGED}" ]; then - ( - cd ./tests/codecheck || exit 1 - ./pylint.sh ${PY_FILES_CHANGED} || exit 1 - ) - fi - fi - - if [ "${PYTYPE}" == "true" ] ; then - if [ "${FILES_CHANGED}" == "all" ] || [ ! -z "${PY_FILES_CHANGED}" ] || [ ! -z "${RQ_FILES_CHANGED}" ]; then - ( - cd ./tests/codecheck || exit 1 - if [ ! -z "${RQ_FILES_CHANGED}" ]; then - # When requirements change, run pytype on everything - ./pytype.sh || exit 1 - else - ./pytype.sh ${PY_FILES_CHANGED} || exit 1 - fi - ) - fi - fi - - exit 0 -elif [ "${MATRIX_SHARD}" == "sanity" ] ; then - FAUCET_TESTS="-u FaucetSanityTest" -elif [ "${MATRIX_SHARD}" == "generative-unit" ]; then - FAUCET_TESTS="--generative_unit" -elif [ "${MATRIX_SHARD}" == "generative-integration" ]; then - FAUCET_TESTS="--generative_integration" -else - ALLTESTFILES="tests/integration/mininet_tests.py tests/integration/mininet_multidp_tests.py clib/clib_mininet_tests.py" - ALLTESTS=$(grep -E -o "^class (Faucet[a-zA-Z0-9]+Test)" ${ALLTESTFILES} | cut -f2 -d" " | sort) - declare -A sharded - - function shard { - work=$1 - workers=$2 - i=0 - for shard in $work ; do - i=$(expr $i % $workers) - sharded[$i]="${sharded[$i]} $shard" - i=$(expr $i + 1) - done - } - - shard "$ALLTESTS" ${MATRIX_SHARDS} - FAUCET_TESTS="-din ${sharded[${MATRIX_SHARD}]}" -fi - -if [ "${FILES_CHANGED}" != "all" ]; then - if [ -z "${PY_FILES_CHANGED}" ] && [ -z "${RQ_FILES_CHANGED}" ]; then - echo "Not running docker tests because no code changes detected" - exit 0 - fi -fi - -docker build --pull -t ${FAUCET_TEST_IMG} -f Dockerfile.tests . || exit 1 - -SHARDARGS="--privileged --sysctl net.ipv6.conf.all.disable_ipv6=0 \ - --ulimit core=99999999999:99999999999 \ - -v /var/local/lib/docker:/var/lib/docker \ - -v $HOME/.cache/pip:/var/tmp/pip-cache" - -echo "MATRIX_SHARD: ${MATRIX_SHARD}" -echo "FAUCET_TESTS: ${FAUCET_TESTS}" -echo "SHARDARGS: ${SHARDARGS}" - -ulimit -c unlimited -echo '/var/tmp/core.%h.%e.%t' | sudo tee /proc/sys/kernel/core_pattern -sudo modprobe openvswitch -sudo modprobe ebtables - -if [ "${MATRIX_SHARD}" == "sanity" ] ; then - # Simulate hardware test switch - # TODO: run a standalone DP and also a stacked DP test to test hardware linkages. - sudo docker run $SHARDARGS -e FAUCET_TESTS="-ni FaucetSanityTest FaucetStackStringOfDPUntaggedTest" -e HWTESTS="1" -t ${FAUCET_TEST_IMG} || exit 1 -else - sudo docker run $SHARDARGS -e FAUCET_TESTS="${FAUCET_TESTS}" -t ${FAUCET_TEST_IMG} || exit 1 -fi - -if ls -1 /var/tmp/core* >/dev/null 2>&1 ; then - echo "coredumps found after tests run." - exit 1 - # TODO: automatically run gdb? -fi - -exit 0