mdns: Example tests integration

Author: suren-gabrielyan-espressif

.github/workflows/target-test.yml

@@ -23,7 +23,7 @@ jobs:
     container: espressif/idf:${{ matrix.idf_ver }}
     steps:
       - name: Checkout esp-protocols
-        uses: actions/checkout@v3
+        uses: actions/checkout@master
         with:
           path: esp-protocols
       - name: Build ${{ matrix.example }} with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }}
@@ -39,7 +39,8 @@ jobs:
     strategy: 
       matrix: 
         idf_ver: ["latest"] 
-        idf_target: ["esp32", "esp32s2", "esp32c3"] 
+        idf_target: ["esp32", "esp32s2", "esp32c3"]
+        config: ["eth_custom_netif", "eth_def", "eth_no_ipv6", "eth_socket"]
 
     runs-on: ubuntu-20.04 
     container: espressif/idf:${{ matrix.idf_ver }} 
@@ -48,36 +49,41 @@ jobs:
         uses: actions/checkout@v3 
         with: 
           submodules: recursive 
-          path: esp-protocols 
-      - name: Build ${{ matrix.example }} with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }} 
+      - name: Build ${{ matrix.example }} with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }} for ${{ matrix.config }} 
         env: 
           IDF_TARGET: ${{ matrix.idf_target }} 
         shell: bash 
+        working-directory: components/mdns/examples/
         run: | 
           . ${IDF_PATH}/export.sh 
-          cd $GITHUB_WORKSPACE/esp-protocols/components/mdns/examples/ 
-          idf.py set-target ${{ matrix.idf_target }}
-          echo "Building for default configuration"
-          idf.py build
-          rm sdkconfig
-          echo "Building for sdkconfig.ci.eth_def"
-          cat sdkconfig.ci.eth_def >> sdkconfig.defaults 
-          idf.py build 
-          rm sdkconfig sdkconfig.defaults
-          echo "Building for sdkconfig.ci.eth_custom_netif"
-          cat sdkconfig.ci.eth_custom_netif >> sdkconfig.defaults 
-          idf.py build 
-          rm sdkconfig sdkconfig.defaults
-          echo "Building for sdkconfig.ci.eth_socket"
-          cat sdkconfig.ci.eth_socket >> sdkconfig.defaults 
-          idf.py build 
-          rm sdkconfig sdkconfig.defaults
-          echo "Building for sdkconfig.ci.eth_no_ipv6"
-          cat sdkconfig.ci.eth_no_ipv6 >> sdkconfig.defaults
-          idf.py build
-          cd $GITHUB_WORKSPACE/esp-protocols/components/mdns/tests/test_apps/ 
+          rm -rf sdkconfig sdkconfig.defaults build build_${{ matrix.config }}
+          cat sdkconfig.ci.${{ matrix.config }} >> sdkconfig.defaults  
           idf.py set-target ${{ matrix.idf_target }}  
           idf.py build 
+          mv build build_${{ matrix.config }}
+      - name: Merge binaries with IDF-${{ matrix.idf_ver }} for ${{ matrix.config }}
+        working-directory: components/mdns/examples
+        env:
+          IDF_TARGET: ${{ matrix.idf_target }}
+        shell: bash
+        run: |
+          . ${IDF_PATH}/export.sh
+          cd build_${{ matrix.config }}
+          esptool.py --chip ${{ matrix.idf_target }} merge_bin --fill-flash-size 4MB -o flash_image.bin @flash_args
+          cd ../
+          cat build_${{ matrix.config }}/config/sdkconfig.json
+      - uses: actions/upload-artifact@v2
+        with:
+          name: examples_app_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }}_${{ matrix.config }}
+          path: |
+            components/mdns/examples/build_${{ matrix.config }}/bootloader/bootloader.bin
+            components/mdns/examples/build_${{ matrix.config }}/partition_table/partition-table.bin
+            components/mdns/examples/build_${{ matrix.config }}/*.bin
+            components/mdns/examples/build_${{ matrix.config }}/*.elf
+            components/mdns/examples/build_${{ matrix.config }}/flasher_args.json
+            components/mdns/examples/build_${{ matrix.config }}/config/sdkconfig.h
+            components/mdns/examples/build_${{ matrix.config }}/config/sdkconfig.json
+          if-no-files-found: error
 
   build_asio:
     strategy:
@@ -109,7 +115,6 @@ jobs:
       matrix:
         idf_ver: ["latest"]
         idf_target: ["esp32"]
-
     runs-on: ubuntu-20.04
     container: espressif/idf:${{ matrix.idf_ver }}
     steps:
@@ -137,7 +142,14 @@ jobs:
       - uses: actions/upload-artifact@v2
         with:
           name: examples_app_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }}
-          path: components/esp_websocket_client/examples/build/
+          path: |
+            components/esp_websocket_client/examples/build/bootloader/bootloader.bin
+            components/esp_websocket_client/examples/build/partition_table/partition-table.bin
+            components/esp_websocket_client/examples/build/*.bin
+            components/esp_websocket_client/examples/build/*.elf
+            components/esp_websocket_client/examples/build/flasher_args.json
+            components/esp_websocket_client/examples/build/config/sdkconfig.h
+            components/esp_websocket_client/examples/build/config/sdkconfig.json
           if-no-files-found: error
 
   run-target:
@@ -151,6 +163,8 @@ jobs:
     runs-on:
       - self-hosted
       - ESP32-ETHERNET-KIT
+    # Skip running on forks since it won't have access to secrets
+    if: github.repository == 'espressif/esp-protocols'
     container:
       image: python:3.7-buster
       options: --privileged # Privileged mode has access to serial ports
@@ -161,6 +175,8 @@ jobs:
           name: examples_app_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }}
           path: components/esp_websocket_client/examples/build/
       - name: Install Python packages
+        env:
+          PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple"
         run: |
           pip install --only-binary cryptography --extra-index-url https://dl.espressif.com/pypi/ -r $GITHUB_WORKSPACE/components/esp_websocket_client/examples/requirements.txt
       - name: Download Example Test to target
@@ -169,9 +185,52 @@ jobs:
         working-directory: components/esp_websocket_client/examples
         run: |
           cp sdkconfig.ci sdkconfig.defaults
-          pytest --log-cli-level DEBUG --junit-xml=./test_app_results_${{ matrix.idf_target }}_${{ matrix.idf_ver }}.xml --target=${{ matrix.idf_target }}
+          pytest --log-cli-level DEBUG --junit-xml=./examples_results_${{ matrix.idf_target }}_${{ matrix.idf_ver }}.xml --target=${{ matrix.idf_target }}
       - uses: actions/upload-artifact@v2
         if: always()
         with:
           name: examples_results_${{ matrix.idf_target }}_${{ matrix.idf_ver }}
-          path: examples/*.xml
+          path: components/esp_websocket_client/examples/*.xml
+
+  run-target-mdns:
+    strategy: 
+      matrix: 
+        idf_ver: ["latest"] 
+        idf_target: ["esp32"]
+        config: ["eth_custom_netif", "eth_def", "eth_no_ipv6", "eth_socket"]
+    name: Run mDNS Example Test on target
+    needs: build_mdns
+    runs-on:
+      - self-hosted
+      - ESP32-ETHERNET-KIT
+    # Skip running on forks since it won't have access to secrets
+    if: github.repository == 'espressif/esp-protocols'
+    steps:
+      - name: Clear repository
+        run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE
+      - uses: actions/checkout@v3
+      - uses: actions/download-artifact@v2
+        with:
+          name: examples_app_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }}_${{ matrix.config }}
+          path: components/mdns/examples/build_${{ matrix.config }}
+      - name: Install Python packages
+        env:
+          PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple"
+        run: |
+            sudo apt-get install -y dnsutils
+      - name: Download Example Test to target ${{ matrix.config }}
+        run: |
+            python -m esptool --chip ${{ matrix.idf_target }} write_flash 0x0 components/mdns/examples/build_${{ matrix.config }}/flash_image.bin
+      - name: Run Example Test on target ${{ matrix.config }}
+        working-directory: components/mdns/examples
+        run: |
+            rm -rf build
+            mv build_${{ matrix.config }} build
+            cat sdkconfig.ci.${{ matrix.config }} >> sdkconfig.defaults  
+            python -m pytest --log-cli-level DEBUG --junit-xml=./examples_results_${{ matrix.idf_target }}_${{ matrix.idf_ver }}_${{ matrix.config }}.xml --target=${{ matrix.idf_target }}
+            rm -rf build sdkconfig.defaults
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: examples_results_${{ matrix.idf_target }}_${{ matrix.idf_ver }}_${{ matrix.config }}
+          path: components/mdns/examples/*.xml

components/mdns/examples/pytest.ini

@@ -0,0 +1,24 @@
+[pytest]
+# only the files with prefix `pytest_` would be recognized as pytest test scripts.
+python_files = pytest_*.py
+
+addopts =
+  -s
+  --embedded-services esp,idf
+  -W ignore::_pytest.warning_types.PytestExperimentalApiWarning
+  --tb short
+
+# ignore DeprecationWarning
+filterwarnings =
+  ignore:Call to deprecated create function (.*)\(\):DeprecationWarning
+
+# log related
+log_cli = True
+log_cli_level = INFO
+log_cli_format = %(asctime)s %(levelname)s %(message)s
+log_cli_date_format = %Y-%m-%d %H:%M:%S
+
+log_file = test.log
+log_file_level = INFO
+log_file_format = %(asctime)s %(levelname)s %(message)s
+log_file_date_format = %Y-%m-%d %H:%M:%S

components/mdns/examples/mdns_example_test.py renamed to components/mdns/examples/pytest_mdns.py

@@ -5,19 +5,19 @@
 import struct
 import subprocess
 import time
+import pytest
 from threading import Event, Thread
 
 import dpkt
 import dpkt.dns
-import ttfw_idf
-from tiny_test_fw import DUT
-from tiny_test_fw.Utility import console_log
+from pytest_embedded import Dut
+import subprocess
 
 
 def get_dns_query_for_esp(esp_host):
     dns = dpkt.dns.DNS(b'\x00\x00\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x01')
     dns.qd[0].name = esp_host + u'.local'
-    console_log('Created query for esp host: {} '.format(dns.__repr__()))
+    print('Created query for esp host: {} '.format(dns.__repr__()))
     return dns.pack()
 
 
@@ -31,7 +31,7 @@ def get_dns_answer_to_mdns(tester_host):
     arr.name = tester_host
     arr.ip = socket.inet_aton('127.0.0.1')
     dns.an.append(arr)
-    console_log('Created answer to mdns query: {} '.format(dns.__repr__()))
+    print('Created answer to mdns query: {} '.format(dns.__repr__()))
     return dns.pack()
 
 
@@ -80,87 +80,58 @@ def mdns_server(esp_host, events):
             dns = dpkt.dns.DNS(data)
             if len(dns.qd) > 0 and dns.qd[0].type == dpkt.dns.DNS_A:
                 if dns.qd[0].name == TESTER_NAME:
-                    console_log('Received query: {} '.format(dns.__repr__()))
+                    print('Received query: {} '.format(dns.__repr__()))
                     sock.sendto(get_dns_answer_to_mdns(TESTER_NAME), (MCAST_GRP, UDP_PORT))
                 elif dns.qd[0].name == TESTER_NAME_LWIP:
-                    console_log('Received query: {} '.format(dns.__repr__()))
+                    print('Received query: {} '.format(dns.__repr__()))
                     sock.sendto(get_dns_answer_to_mdns_lwip(TESTER_NAME_LWIP, dns.id), addr)
             if len(dns.an) > 0 and dns.an[0].type == dpkt.dns.DNS_A:
-                console_log('Received answer from {}'.format(dns.an[0].name))
+                print('Received answer from {}'.format(dns.an[0].name))
                 if dns.an[0].name == esp_host + u'.local':
-                    console_log('Received answer to esp32-mdns query: {}'.format(dns.__repr__()))
+                    print('Received answer to esp32-mdns query: {}'.format(dns.__repr__()))
                     events['esp_answered'].set()
                 if dns.an[0].name == esp_host + u'-delegated.local':
-                    console_log('Received answer to esp32-mdns-delegate query: {}'.format(dns.__repr__()))
+                    print('Received answer to esp32-mdns-delegate query: {}'.format(dns.__repr__()))
                     events['esp_delegated_answered'].set()
         except socket.timeout:
             break
         except dpkt.UnpackError:
             continue
 
 
-def test_examples_protocol_mdns(env, config):
+def test_examples_protocol_mdns(dut):
     """
     steps: |
       1. obtain IP address + init mdns example
       2. get the dut host name (and IP address)
       3. check the mdns name is accessible
       4. check DUT output if mdns advertized host is resolved
     """
-    dut1 = env.get_dut('mdns-test', 'examples/protocols/mdns', dut_class=ttfw_idf.ESP32DUT, app_config_name=config)
-    # check and log bin size
-    binary_file = os.path.join(dut1.app.binary_path, 'mdns_test.bin')
-    bin_size = os.path.getsize(binary_file)
-    ttfw_idf.log_performance('mdns-test_bin_size', '{}KB'.format(bin_size // 1024))
-    # 1. start mdns application
-    dut1.start_app()
-    # 2. get the dut host name (and IP address)
-    specific_host = dut1.expect(re.compile(r'mdns hostname set to: \[([^\]]+)\]'), timeout=30)[0]
+
+    specific_host = dut.expect(re.compile(b'mdns hostname set to: \[(.*?)\]')).group(1).decode()
 
     mdns_server_events = {'stop': Event(), 'esp_answered': Event(), 'esp_delegated_answered': Event()}
     mdns_responder = Thread(target=mdns_server, args=(str(specific_host), mdns_server_events))
+    ip_address = dut.expect(re.compile(b'IPv4 address:([a-zA-Z0-9]*).*')).group(1).decode()
+    print('Connected to AP with IP: {}'.format(ip_address))
     try:
-        ip_address = dut1.expect(re.compile(r' eth ip: ([^,]+),'), timeout=30)[0]
-        console_log('Connected to AP with IP: {}'.format(ip_address))
-    except DUT.ExpectTimeout:
-        raise ValueError('ENV_TEST_FAILURE: Cannot connect to AP')
-    try:
-        # 3. check the mdns name is accessible
+        # 3. check the mdns name is accessible.
         mdns_responder.start()
         if not mdns_server_events['esp_answered'].wait(timeout=30):
             raise ValueError('Test has failed: did not receive mdns answer within timeout')
         if not mdns_server_events['esp_delegated_answered'].wait(timeout=30):
             raise ValueError('Test has failed: did not receive mdns answer for delegated host within timeout')
         # 4. check DUT output if mdns advertized host is resolved
-        dut1.expect(re.compile(r'mdns-test: Query A: tinytester.local resolved to: 127.0.0.1'), timeout=30)
-        dut1.expect(re.compile(r'mdns-test: gethostbyname: tinytester-lwip.local resolved to: 127.0.0.1'), timeout=30)
-        dut1.expect(re.compile(r'mdns-test: getaddrinfo: tinytester-lwip.local resolved to: 127.0.0.1'), timeout=30)
+        dut.expect(re.compile(b'mdns-test: Query A: tinytester.local resolved to: 127.0.0.1'), timeout=30)
+        dut.expect(re.compile(b'mdns-test: gethostbyname: tinytester-lwip.local resolved to: 127.0.0.1'), timeout=30)
+        dut.expect(re.compile(b'mdns-test: getaddrinfo: tinytester-lwip.local resolved to: 127.0.0.1'), timeout=30)
         # 5. check the DUT answers to `dig` command
         dig_output = subprocess.check_output(['dig', '+short', '-p', '5353', '@224.0.0.251',
                                               '{}.local'.format(specific_host)])
-        console_log('Resolving {} using "dig" succeeded with:\n{}'.format(specific_host, dig_output))
+        print('Resolving {} using "dig" succeeded with:\n{}'.format(specific_host, dig_output))
         if not ip_address.encode('utf-8') in dig_output:
             raise ValueError('Test has failed: Incorrectly resolved DUT hostname using dig'
                              "Output should've contained DUT's IP address:{}".format(ip_address))
     finally:
         mdns_server_events['stop'].set()
         mdns_responder.join()
-
-
-@ttfw_idf.idf_example_test(env_tag='Example_EthKitV1')
-def test_examples_protocol_mdns_default(env, _):
-    test_examples_protocol_mdns(env, 'eth_def')
-
-
-@ttfw_idf.idf_example_test(env_tag='Example_EthKitV1')
-def test_examples_protocol_mdns_socket(env, _):
-    test_examples_protocol_mdns(env, 'eth_socket')
-
-
-@ttfw_idf.idf_example_test(env_tag='Example_EthKitV1')
-def test_examples_protocol_mdns_custom_netif(env, _):
-    test_examples_protocol_mdns(env, 'eth_custom_netif')
-
-
-if __name__ == '__main__':
-    test_examples_protocol_mdns_default()