Merge pull request #3609 from mab68/generative-actions

Move unit, integration and generative tests to github from travis

.github/workflows/on-pr.yml

@@ -0,0 +1,223 @@
+name: Faucet on pull-request tests
+
+on: [push, pull_request]
+
+env:
+  FAUCET_TEST_IMG: "faucet/tests"
+  SHARDARGS: "--privileged --sysctl net.ipv6.conf.all.disable_ipv6=0 --ulimit core=99999999999:99999999999 -v /var/local/lib/docker:/var/lib/docker"
+  FILES_CHANGED: "all"
+  CODECHECK_PY_VER: 3.6
+  MATRIX_SHARDS: 10
+
+jobs:
+
+  unit-tests:
+    name: Unit tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.5, 3.6, 3.7, 3.8]
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v2.3.1
+      - if: ${{ github.event.before != '0000000000000000000000000000000000000000' }}
+        name: Get file changes
+        id: file_changes
+        uses: trilom/file-changes-action@v1.2.4
+        with:
+          output: ' '
+      - if: ${{ steps.file_changes.outputs.files }}
+        name: Compare file changes
+        run: |
+          FILES_CHANGED="${{ steps.file_changes.outputs.files }}"
+          PY_FILES_CHANGED=$(echo "${{ steps.file_changes.outputs.files }}" | tr ' ' '\n' | grep -E ".py$" | tr '\n' ' ')
+          RQ_FILES_CHANGED=$(echo "${{ steps.file_changes.outputs.files }}" | tr ' ' '\n' | grep -E "requirements(.*)txt$" | tr '\n' ' ')
+          DOC_FILES_CHANGED=$(echo "${{ steps.file_changes.outputs.files }}" | tr ' ' '\n' | grep -E "docs/**" | tr '\n' ' ')
+          echo "Files changed: ${FILES_CHANGED}"
+          echo "Python code changed: ${PY_FILES_CHANGED}"
+          echo "Requirement changes: ${RQ_FILES_CHANGED}"
+          echo "Documentation changes: ${DOC_FILES_CHANGED}"
+          echo ::set-env name=FILES_CHANGED::${FILES_CHANGED}
+          echo ::set-env name=PY_FILES_CHANGED::${PY_FILES_CHANGED}
+          echo ::set-env name=RQ_FILES_CHANGED::${RQ_FILES_CHANGED}
+          echo ::set-env name=DOC_FILES_CHANGED::${DOC_FILES_CHANGED}
+      - name: Set up python-${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          ./docker/pip_deps.sh
+          pip3 install ./
+          pip3 show faucet
+      - name: Running unit tests
+        run: ./tests/run_unit_tests.sh || exit 1
+      - if: ${{ matrix.python-version == env.CODECHECK_PY_VER }}
+        name: Upload codecov
+        uses: codecov/codecov-action@v1.0.7
+      - if: ${{ env.FILES_CHANGED == 'all' || env.RQ_FILES_CHANGED || env.PY_FILES_CHANGED }}
+        name: Pytype
+        run: |
+          cd ./tests/codecheck || exit 1
+          if [[ "${{ env.FILES_CHANGED }}" == "all" || ! -z "${{ env.RQ_FILES_CHANGED }}" ]]; then
+            echo "Running pytype on everything"
+            ./pytype.sh || exit 1
+          else
+            echo "Running pytype on ${{ env.PY_FILES_CHANGED }}"
+            ./pytype.sh ${{ env.PY_FILES_CHANGED }} || exit 1
+          fi
+
+  codecheck:
+    name: Code check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v2.3.1
+      - if: ${{ github.event.before != '0000000000000000000000000000000000000000' }}
+        name: Get file changes
+        id: file_changes
+        uses: trilom/file-changes-action@v1.2.4
+        with:
+          output: ' '
+      - if: ${{ steps.file_changes.outputs.files }}
+        name: Compare file changes
+        run: |
+          FILES_CHANGED="${{ steps.file_changes.outputs.files }}"
+          PY_FILES_CHANGED=$(echo "${{ steps.file_changes.outputs.files }}" | tr ' ' '\n' | grep -E ".py$" | tr '\n' ' ')
+          RQ_FILES_CHANGED=$(echo "${{ steps.file_changes.outputs.files }}" | tr ' ' '\n' | grep -E "requirements(.*)txt$" | tr '\n' ' ')
+          DOC_FILES_CHANGED=$(echo "${{ steps.file_changes.outputs.files }}" | tr ' ' '\n' | grep -E "docs/**" | tr '\n' ' ')
+          echo "Files changed: ${FILES_CHANGED}"
+          echo "Python code changed: ${PY_FILES_CHANGED}"
+          echo "Requirement changes: ${RQ_FILES_CHANGED}"
+          echo "Documentation changes: ${DOC_FILES_CHANGED}"
+          echo ::set-env name=FILES_CHANGED::${FILES_CHANGED}
+          echo ::set-env name=PY_FILES_CHANGED::${PY_FILES_CHANGED}
+          echo ::set-env name=RQ_FILES_CHANGED::${RQ_FILES_CHANGED}
+          echo ::set-env name=DOC_FILES_CHANGED::${DOC_FILES_CHANGED}
+      - name: Set up python-${{ env.CODECHECK_PY_VER }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.CODECHECK_PY_VER }}
+      - name: Install dependencies
+        run: |
+          ./docker/pip_deps.sh
+      - if: ${{ env.DOC_FILES_CHANGED }}
+        name: Build docs
+        run: |
+          cd ./docs || exit 1
+          sudo apt-get install librsvg2-bin || exit 1
+          make html || exit 1
+          rm -rf _build
+      - if: ${{ env.FILES_CHANGED == 'all' || env.PY_FILES_CHANGED }}
+        name: Pylint
+        run: |
+          cd ./tests/codecheck || exit 1
+          ./pylint.sh ${{ env.PY_FILES_CHANGED }} || exit 1
+
+  integration-tests:
+    name: Integration tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        MATRIX_SHARD: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v2.3.1
+      - if: ${{ github.event.before != '0000000000000000000000000000000000000000' }}
+        name: Get file changes
+        id: file_changes
+        uses: trilom/file-changes-action@v1.2.4
+        with:
+          output: ' '
+      - if: ${{ steps.file_changes.outputs.files }}
+        name: Compare file changes
+        run: |
+          FILES_CHANGED="${{ steps.file_changes.outputs.files }}"
+          PY_FILES_CHANGED=$(echo "${{ steps.file_changes.outputs.files }}" | tr ' ' '\n' | grep -E ".py$" | tr '\n' ' ')
+          RQ_FILES_CHANGED=$(echo "${{ steps.file_changes.outputs.files }}" | tr ' ' '\n' | grep -E "requirements(.*)txt$" | tr '\n' ' ')
+          DOC_FILES_CHANGED=$(echo "${{ steps.file_changes.outputs.files }}" | tr ' ' '\n' | grep -E "docs/**" | tr '\n' ' ')
+          echo "Files changed: ${FILES_CHANGED}"
+          echo "Python code changed: ${PY_FILES_CHANGED}"
+          echo "Requirement changes: ${RQ_FILES_CHANGED}"
+          echo "Documentation changes: ${DOC_FILES_CHANGED}"
+          echo ::set-env name=FILES_CHANGED::${FILES_CHANGED}
+          echo ::set-env name=PY_FILES_CHANGED::${PY_FILES_CHANGED}
+          echo ::set-env name=RQ_FILES_CHANGED::${RQ_FILES_CHANGED}
+          echo ::set-env name=DOC_FILES_CHANGED::${DOC_FILES_CHANGED}
+      - if: ${{ env.FILES_CHANGED == 'all' || env.PY_FILES_CHANGED || env.RQ_FILES_CHANGED }}
+        name: Shard out tests
+        run: |
+            ALLTESTFILES="tests/integration/mininet_tests.py tests/integration/mininet_multidp_tests.py clib/clib_mininet_tests.py"
+            ALLTESTS=$(grep -E -o "^class (Faucet[a-zA-Z0-9]+Test)" ${ALLTESTFILES} | cut -f2 -d" " | sort)
+            declare -A sharded
+            function shard {
+              work=$1
+              workers=$2
+              i=0
+              for shard in $work ; do
+                i=$(( i % workers ))
+                sharded[$i]="${sharded[$i]} $shard"
+                i=$(( i + 1 ))
+              done
+            }
+            shard "$ALLTESTS" ${{ env.MATRIX_SHARDS }}
+            FAUCET_TESTS="-din ${sharded[${{ matrix.MATRIX_SHARD }}]}"
+            echo ::set-env name=FAUCET_TESTS::${FAUCET_TESTS}
+      - if: ${{ env.FAUCET_TESTS }}
+        name: Build docker
+        run: docker build --pull -t ${FAUCET_TEST_IMG} -f Dockerfile.tests . || exit 1
+      - if: ${{ env.FAUCET_TESTS }}
+        name: Setup docker
+        run: |
+          ulimit -c unlimited
+          echo '/var/tmp/core.%h.%e.%t' | sudo tee /proc/sys/kernel/core_pattern
+          sudo modprobe openvswitch
+          sudo modprobe ebtables
+      - if: ${{ env.FAUCET_TESTS }}
+        name: Run docker
+        run: sudo docker run ${SHARDARGS} -v $HOME/.cache/pip:/var/tmp/pip-cache -e FAUCET_TESTS="${FAUCET_TESTS}" -t ${FAUCET_TEST_IMG} || exit 1
+      - name: Detect core dumps
+        run: if [ ls -1 /var/tmp/core* > /dev/null 2>&1 ]; then exit 1; fi
+
+  sanity-testing:
+    name: Sanity tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v2.3.1
+      - if: ${{ github.event.before != '0000000000000000000000000000000000000000' }}
+        name: Get file changes
+        id: file_changes
+        uses: trilom/file-changes-action@v1.2.4
+        with:
+          output: ' '
+      - if: ${{ steps.file_changes.outputs.files }}
+        name: Compare file changes
+        run: |
+          FILES_CHANGED="${{ steps.file_changes.outputs.files }}"
+          PY_FILES_CHANGED=$(echo "${{ steps.file_changes.outputs.files }}" | tr ' ' '\n' | grep -E ".py$" | tr '\n' ' ')
+          RQ_FILES_CHANGED=$(echo "${{ steps.file_changes.outputs.files }}" | tr ' ' '\n' | grep -E "requirements(.*)txt$" | tr '\n' ' ')
+          DOC_FILES_CHANGED=$(echo "${{ steps.file_changes.outputs.files }}" | tr ' ' '\n' | grep -E "docs/**" | tr '\n' ' ')
+          echo "Files changed: ${FILES_CHANGED}"
+          echo "Python code changed: ${PY_FILES_CHANGED}"
+          echo "Requirement changes: ${RQ_FILES_CHANGED}"
+          echo "Documentation changes: ${DOC_FILES_CHANGED}"
+          echo ::set-env name=FILES_CHANGED::${FILES_CHANGED}
+          echo ::set-env name=PY_FILES_CHANGED::${PY_FILES_CHANGED}
+          echo ::set-env name=RQ_FILES_CHANGED::${RQ_FILES_CHANGED}
+          echo ::set-env name=DOC_FILES_CHANGED::${DOC_FILES_CHANGED}
+      - if: ${{ env.FILES_CHANGED == 'all' || env.PY_FILES_CHANGED || env.RQ_FILES_CHANGED }}
+        name: Setup docker test requirements
+        run: |
+          ulimit -c unlimited
+          echo '/var/tmp/core.%h.%e.%t' | sudo tee /proc/sys/kernel/core_pattern
+          sudo modprobe openvswitch
+          sudo modprobe ebtables
+      - if: ${{ env.FILES_CHANGED == 'all' || env.PY_FILES_CHANGED || env.RQ_FILES_CHANGED }}
+        name: Build docker
+        run: docker build --pull -t ${FAUCET_TEST_IMG} -f Dockerfile.tests . || exit 1
+      - if: ${{ env.FILES_CHANGED == 'all' || env.PY_FILES_CHANGED || env.RQ_FILES_CHANGED }}
+        name: Run docker
+        run: sudo docker run ${SHARDARGS} -v $HOME/.cache/pip:/var/tmp/pip-cache -e FAUCET_TESTS="-ni FaucetSanityTest FaucetStackStringOfDPUntaggedTest" -e HWTESTS="1" -t ${FAUCET_TEST_IMG} || exit 1
+      - name: Detect core dumps
+        run: if [ ls -1 /var/tmp/core* > /dev/null 2>&1 ]; then exit 1; fi

.github/workflows/periodic.yml

@@ -0,0 +1,35 @@
+name: Faucet scheduled tests
+
+on:
+  schedule:
+    # NZST = UTC + 12
+    # Schedule to run at midnight & lunch-time
+    - cron: '0 0,12 * * *'
+
+env:
+  FAUCET_TEST_IMG: "faucet/tests"
+  SHARDARGS: "--privileged --sysctl net.ipv6.conf.all.disable_ipv6=0 --ulimit core=99999999999:99999999999 -v /var/local/lib/docker:/var/lib/docker"
+
+jobs:
+
+  generative-testing:
+    name: Generative tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        MATRIX_SHARD: [--generative_unit, --generative_integration]
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v2.3.1
+      - name: Setup docker test requirements
+        run: |
+          ulimit -c unlimited
+          echo '/var/tmp/core.%h.%e.%t' | sudo tee /proc/sys/kernel/core_pattern
+          sudo modprobe openvswitch
+          sudo modprobe ebtables
+      - name: Build docker
+        run: docker build --pull -t ${FAUCET_TEST_IMG} -f Dockerfile.tests . || exit 1
+      - name: Run docker
+        run: sudo docker run ${SHARDARGS} -v $HOME/.cache/pip:/var/tmp/pip-cache -e FAUCET_TESTS="${{ matrix.MATRIX_SHARD }}" -t ${FAUCET_TEST_IMG} || exit 1
+      - name: Detect core dumps
+        run: if [ ls -1 /var/tmp/core* > /dev/null 2>&1 ]; then exit 1; fi

.travis.yml

@@ -9,73 +9,13 @@ addons:
 cache:
   pip: true
 stages:
-  - test
   - name: deploy
     if: repo = faucetsdn/faucet AND tag IS present
-env:
-  global:
-    - FAUCET_TEST_IMG=faucet/tests
-    - MATRIX_SHARDS=6
-script:
-  - ./travis/runtests.sh
 jobs:
   include:
-    - &unit-test
-      stage: test
-      language: python
-      python: 3.5
-      env:
-        - MATRIX_SHARD=unittest
-        - PYTYPE=true
-    - <<: *unit-test
-      python: 3.6
-      env:
-        - MATRIX_SHARD=unittest
-        - BUILD_DOCS=true
-        - CODECOV_UPLOAD=true
-        - PYLINT=true
-        - PYTYPE=true
-    - <<: *unit-test
-      python: 3.7
-      env:
-        - MATRIX_SHARD=unittest
-        - PYTYPE=true
-    - <<: *unit-test
-      python: 3.8
-      env:
-        - MATRIX_SHARD=unittest
-        - PYTYPE=true
-    - &integration-test
-      stage: test
-      env: MATRIX_SHARD=sanity
-      language: bash
-      before_install:
-        - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
-        - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
-        - sudo apt-get update
-        - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
-      services:
-        - docker
-    - <<: *integration-test
-      env: MATRIX_SHARD=0
-    - <<: *integration-test
-      env: MATRIX_SHARD=1
-    - <<: *integration-test
-      env: MATRIX_SHARD=2
-    - <<: *integration-test
-      env: MATRIX_SHARD=3
-    - <<: *integration-test
-      env: MATRIX_SHARD=4
-    - <<: *integration-test
-      env: MATRIX_SHARD=5
-    - <<: *integration-test
-      env: MATRIX_SHARD=generative-unit
-    - <<: *integration-test
-      env: MATRIX_SHARD=generative-integration
     - stage: deploy
       language: python
       python: 3.6
-      script: skip
       deploy:
         provider: pypi
         on:

docker/fuzz_config.sh

@@ -14,4 +14,4 @@ if [ -e "$checkfile" ]; then
     fi
 fi
 
-AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 AFL_SKIP_CPUFREQ=1 py-afl-fuzz -x "$dictfile" -m 5000 -i "$inputfile" -o "$outputfile" -- /usr/bin/python3 /faucet-src/tests/fuzzer/fuzz_config.py
+AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 AFL_SKIP_CPUFREQ=1 py-afl-fuzz -x "$dictfile" -m 5000 -i "$inputfile" -o "$outputfile" -- /usr/bin/python3 /faucet-src/tests/generative/fuzzer/fuzz_config.py

docker/fuzz_packet.sh

@@ -14,4 +14,4 @@ if [ -e "$checkfile" ]; then
     fi
 fi
 
-AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 AFL_SKIP_CPUFREQ=1 py-afl-fuzz -m 5000 -x "$dictfile" -i "$inputfile" -o "$outputfile" -- /usr/bin/python3 /faucet-src/tests/fuzzer/fuzz_packet.py
+AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 AFL_SKIP_CPUFREQ=1 py-afl-fuzz -m 5000 -x "$dictfile" -i "$inputfile" -o "$outputfile" -- /usr/bin/python3 /faucet-src/tests/generative/fuzzer/fuzz_packet.py

tests/fuzzer/display_packet_crash.py → tests/generative/fuzzer/display_packet_crash.py

@@ -52,4 +52,4 @@ def main():
     if len(sys.argv) == 2:
         main()
     else:
-        print('USAGE: python3 display_packet_crash.py <AFL_CRASH_FILE>')
+        sys.stderr.write('USAGE: python3 display_packet_crash.py <AFL_CRASH_FILE>\n')

tests/fuzzer/fake_packet.py → tests/generative/fuzzer/fake_packet.py

@@ -1,12 +1,16 @@
 """Fake classes for the packet fuzzer to use"""
 
-class RyuEvent: # pylint: disable=too-few-public-methods
+
+class RyuEvent:  # pylint: disable=too-few-public-methods
     """Fake ryuevent class"""
+
     def __init__(self, msg):
         self.msg = msg
 
-class Message: # pylint: disable=too-few-public-methods
+
+class Message:  # pylint: disable=too-few-public-methods
     """Fake message class"""
+
     def __init__(self, *args, **kwargs):
         self.datapath = kwargs['datapath']
         self.cookie = kwargs['cookie']
@@ -16,7 +20,9 @@ def __init__(self, *args, **kwargs):
         self.match = kwargs
         self.args = args
 
-class Datapath: # pylint: disable=too-few-public-methods
+
+class Datapath:  # pylint: disable=too-few-public-methods
     """Fake datapath class"""
+
     def __init__(self, dp_id):
         self.dp_id = dp_id

tests/fuzzer/fuzz_config.py → tests/generative/fuzzer/fuzz_config.py

@@ -26,7 +26,7 @@ def create_config_file(config):
 
 def main():
     logging.disable(logging.CRITICAL)
-    while afl.loop(ROUNDS): # pylint: disable=c-extension-no-member
+    while afl.loop(ROUNDS):  # pylint: disable=c-extension-no-member
         config = sys.stdin.read()
         file_name = create_config_file(config)
         try:

tests/fuzzer/fuzz_packet.py → tests/generative/fuzzer/fuzz_packet.py

@@ -30,7 +30,7 @@ def main():
             valve.dp.running = True
             valve.dp.dyn_finalized = state
 
-    while afl.loop(ROUNDS): # pylint: disable=c-extension-no-member
+    while afl.loop(ROUNDS):  # pylint: disable=c-extension-no-member
         # receive input from afl
         rcv = sys.stdin.read()
         data = None