test

#!/bin/bash
#
#  Copyright (c) 2018, The OpenThread Authors.
#  All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are met:
#  1. Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
#  2. Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#  3. Neither the name of the copyright holder nor the
#     names of its contributors may be used to endorse or promote products
#     derived from this software without specific prior written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
#  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
#  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
#  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
#  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
#  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
#  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
#  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
#  POSSIBILITY OF SUCH DAMAGE.
#
#    Description:
#      This file runs various tests of OpenThread.
#

set -euo pipefail

readonly OT_BUILDDIR="$(pwd)/build"
readonly OT_SRCDIR="$(pwd)"

readonly COLOR_PASS='\033[0;32m'
readonly COLOR_FAIL='\033[0;31m'
readonly COLOR_NONE='\033[0m'

readonly NODE_MODE="${NODE_MODE:-standalone}"
readonly NODE_TYPE="${NODE_TYPE:-sim}"
readonly OT_NATIVE_IP="${OT_NATIVE_IP:-0}"
readonly THREAD_VERSION="${THREAD_VERSION:-1.1}"
readonly VERBOSE="${VERBOSE:-0}"

build_simulation()
{
    local version="$1"
    local options=("-DOT_THREAD_VERSION=${version}" "-DBUILD_TESTING=ON" "-DOT_REFERENCE_DEVICE=ON")

    if [[ ${version} == "1.2" ]]; then
        options+=("-DOT_DUA=ON")
        options+=("-DOT_MLR=ON")
    fi

    if [[ ${VIRTUAL_TIME} == 1 ]]; then
        options+=("-DOT_SIMULATION_VIRTUAL_TIME=ON")

        if [[ ${NODE_MODE} == "rcp" ]]; then
            options+=("-DOT_SIMULATION_VIRTUAL_TIME_UART=ON")
        fi

    fi

    if [[ ${version} == "1.2" ]]; then
        options+=("-DOT_CSL_RECEIVER=ON")
    fi

    if [[ ${ot_extra_options[*]+x} ]]; then
        options+=("${ot_extra_options[@]}")
    fi

    OT_CMAKE_BUILD_DIR="${OT_BUILDDIR}/cmake/openthread-simulation-${version}" "${OT_SRCDIR}"/script/cmake-build simulation "${options[@]}"

    if [[ ${version} == "1.2" ]]; then

        options+=("-DOT_BACKBONE_ROUTER=ON")

        OT_CMAKE_BUILD_DIR="${OT_BUILDDIR}/cmake/openthread-simulation-${version}-bbr" "${OT_SRCDIR}"/script/cmake-build simulation "${options[@]}"

    fi
}

build_posix()
{
    local version="$1"
    local options=("-DOT_THREAD_VERSION=${version}" "-DBUILD_TESTING=ON")

    if [[ ${version} == "1.2" ]]; then
        options+=("-DOT_DUA=ON")
        options+=("-DOT_MLR=ON")
    fi

    if [[ ${VIRTUAL_TIME} == 1 ]]; then
        options+=("-DOT_POSIX_VIRTUAL_TIME=ON")
    fi

    if [[ ${OT_NATIVE_IP} == 1 ]]; then
        options+=("-DOT_PLATFORM_UDP=ON" "-DOT_PLATFORM_NETIF=ON")
    fi

    if [[ ${ot_extra_options[*]+x} ]]; then
        options+=("${ot_extra_options[@]}")
    fi

    OT_CMAKE_BUILD_DIR="${OT_BUILDDIR}/cmake/openthread-posix-${version}" "${OT_SRCDIR}"/script/cmake-build posix "${options[@]}"

    if [[ ${version} == "1.2" ]]; then

        options+=("-DOT_BACKBONE_ROUTER=ON")

        OT_CMAKE_BUILD_DIR="${OT_BUILDDIR}/cmake/openthread-posix-${version}-bbr" "${OT_SRCDIR}"/script/cmake-build posix "${options[@]}"
    fi
}

do_build()
{
    build_simulation "${THREAD_VERSION}"

    if [[ ${NODE_MODE} == "rcp" ]]; then
        build_posix "${THREAD_VERSION}"
    fi

    # Extra 1.1 build for 1.2 tests
    if [[ ${THREAD_VERSION} == "1.2" ]]; then
        build_simulation 1.1

        if [[ ${NODE_MODE} == "rcp" ]]; then
            build_posix 1.1
        fi
    fi
}

do_clean()
{
    rm -rfv "${OT_BUILDDIR}" || sudo rm -rfv "${OT_BUILDDIR}"
}

do_unit()
{
    local builddir="${OT_BUILDDIR}/cmake/openthread-simulation-${THREAD_VERSION}"
    if [[ ! -d ${builddir} ]]; then
        echo "Cannot find build directory!"
        exit 1
    fi

    cd "${builddir}"
    ninja test
}

do_cert()
{
    export top_builddir="${OT_BUILDDIR}/cmake/openthread-simulation-${THREAD_VERSION}"

    if [[ ${THREAD_VERSION} == "1.2" ]]; then
        export top_builddir_1_1="${OT_BUILDDIR}/cmake/openthread-simulation-1.1"
        export top_builddir_1_2_bbr="${OT_BUILDDIR}/cmake/openthread-simulation-1.2-bbr"
    fi

    [[ ! -d tmp ]] || rm -rvf tmp
    PYTHONUNBUFFERED=1 "$1"
}

do_cert_suite()
{
    export top_builddir="${OT_BUILDDIR}/cmake/openthread-simulation-${THREAD_VERSION}"

    if [[ ${THREAD_VERSION} == "1.2" ]]; then
        export top_builddir_1_1="${OT_BUILDDIR}/cmake/openthread-simulation-1.1"
        export top_builddir_1_2_bbr="${OT_BUILDDIR}/cmake/openthread-simulation-1.2-bbr"
    fi

    local pass_count=0
    local fail_count=0

    [[ ! -f fail.log ]] || rm fail.log

    for test_case in "$@"; do
        rm -rf tmp || sudo rm -rf tmp
        if "${test_case}" &>test.log; then
            echo -e "${COLOR_PASS}PASS${COLOR_NONE} ${test_case}"
            pass_count=$((pass_count + 1))
        else
            echo -e "${COLOR_FAIL}FAIL${COLOR_NONE} ${test_case}"
            fail_count=$((fail_count + 1))
            {
                echo "=============================="
                echo "!!! FAIL:${test_case}"
                echo "=============================="
                cat test.log
            } >>fail.log
        fi
    done

    echo "=================================="
    echo "         Test Summary"
    echo "=================================="
    echo "# TOTAL: $((pass_count + fail_count))"
    echo "# PASS: ${pass_count}"
    echo "# FAIL: ${fail_count}"

    if [[ ${fail_count} -gt 0 ]]; then
        tr -dc '[:print:]\r\n\t' <fail.log
        exit 1
    else
        exit 0
    fi
}

do_expect()
{
    local ot_command
    local rcp_command=
    local test_patterns

    if [[ ${NODE_MODE} == rcp ]]; then
        ot_command="${OT_CLI_PATH}"
        rcp_command="${RADIO_DEVICE}"
        if [[ ${OT_NATIVE_IP} == 1 ]]; then
            test_patterns=(-name 'tun-*.exp')
        else
            test_patterns=(-name 'posix-*.exp' -o -name 'cli-*.exp')
        fi
    else
        ot_command="${OT_BUILDDIR}/cmake/openthread-simulation-${THREAD_VERSION}/examples/apps/cli/ot-cli-ftd"
        test_patterns=(-name 'cli-*.exp' -o -name 'simulation-*.exp')
    fi

    local log_file="tmp/log_expect"
    while read -r script; do
        sudo rm -rf tmp
        mkdir tmp
        {
            if [[ ${OT_NATIVE_IP} == 1 ]]; then
                OT_COMMAND="${ot_command}" RCP_COMMAND="${rcp_command}" sudo -E expect -df "${script}" 2>"${log_file}"
            else
                OT_COMMAND="${ot_command}" RCP_COMMAND="${rcp_command}" expect -df "${script}" 2>"${log_file}"
            fi
        } || {
            local exit_code=$?
            cat "${log_file}" >&2
            echo -e "${COLOR_FAIL}FAIL${COLOR_NONE} ${script}"
            exit "${exit_code}"
        }
        if [[ ${VERBOSE} == 1 ]]; then
            cat "${log_file}" >&2
        fi
        echo -e "${COLOR_PASS}PASS${COLOR_NONE} ${script}"
    done < <(
        if [[ $# != 0 ]]; then
            for script in "$@"; do echo ${script}; done
        else
            find tests/scripts/expect -type f -executable \( "${test_patterns[@]}" \)
        fi
    )

    exit 0
}

print_usage()
{
    echo "USAGE: [ENVIRONMENTS] $0 COMMANDS

ENVIRONMENTS:
    NODE_TYPE       'sim' for CLI, 'ncp-sim' for NCP. The default is 'sim'.
    NODE_MODE       'rcp' for RCP mode, otherwise for standalone mode. The default is standalone mode.
    VERBOSE         1 to build or test verbosely. The default is 0.
    VIRTUAL_TIME    1 for virtual time, otherwise real time. The default is 1.
    THREAD_VERSION  1.1 for Thread 1.1 stack, 1.2 for Thread 1.2 stack. The default is 1.1.

COMMANDS:
    clean           Clean built files to prepare for new build.
    build           Build project for running tests. This can be used to rebuild the project for changes.
    cert            Run a single thread-cert test. ENVIRONMENTS should be the same as those given to build or update.
    cert_suite      Run a batch of thread-cert tests and summarize the test results. Only echo logs for failing tests.
    unit            Run all the unit tests. This should be called after simulation is built.
    expect          Run expect tests.
    help            Print this help.

EXAMPLES:
    # Test CLI with default settings
    $0 clean build cert tests/scripts/thread-cert/Cert_5_1_01_RouterAttach.py
    $0 cert tests/scripts/thread-cert/Cert_5_1_02_ChildAddressTimeout.py

    # Test NCP with default settings
    NODE_TYPE=ncp-sim $0 clean build cert tests/scripts/thread-cert/Cert_5_1_01_RouterAttach.py
    NODE_TYPE=ncp-sim $0 cert tests/scripts/thread-cert/Cert_5_1_02_ChildAddressTimeout.py

    # Test CLI with radio only
    NODE_MODE=rcp $0 clean build cert tests/scripts/thread-cert/Cert_5_1_01_RouterAttach.py
    NODE_MODE=rcp $0 cert tests/scripts/thread-cert/Cert_5_1_02_ChildAddressTimeout.py

    # Test CLI with real time
    VIRTUAL_TIME=0 $0 clean build cert tests/scripts/thread-cert/Cert_5_1_01_RouterAttach.py
    VIRTUAL_TIME=0 $0 cert tests/scripts/thread-cert/Cert_5_1_02_ChildAddressTimeout.py

    # Test Thread 1.2 with real time
    THREAD_VERSION=1.2 VIRTUAL_TIME=0 $0 clean build cert tests/scripts/thread-cert/v1_2_router_5_1_1.py
    THREAD_VERSION=1.2 VIRTUAL_TIME=0 $0 clean build cert_suite tests/scripts/thread-cert/v1_2_*

    # Run a single expect test
    $0 clean build expect tests/scripts/expect/cli-log-level.exp

    # Run all expect tests
    $0 clean build expect
    "

    exit "$1"
}

do_package()
{
    local builddir
    local options=("-DCMAKE_BUILD_TYPE=Release" "-DOT_LOG_LEVEL=INFO")

    if [[ ${ot_extra_options[*]+x} ]]; then
        options+=("${ot_extra_options[@]}")
    fi

    builddir="${OT_BUILDDIR}/cmake/openthread-sim"
    OT_CMAKE_NINJA_TARGET="package" OT_CMAKE_BUILD_DIR="${builddir}" "${OT_SRCDIR}"/script/cmake-build simulation "${options[@]}"
    ls "${builddir}"/openthread-simulation-*.deb

    builddir="${OT_BUILDDIR}/cmake/openthread-host"
    OT_CMAKE_NINJA_TARGET="package" OT_CMAKE_BUILD_DIR="${builddir}" "${OT_SRCDIR}"/script/cmake-build posix "${options[@]}"
    ls "${builddir}"/openthread-standalone-*.deb

    builddir="${OT_BUILDDIR}/cmake/openthread-daemon"
    OT_CMAKE_NINJA_TARGET="package" OT_CMAKE_BUILD_DIR="${builddir}" "${OT_SRCDIR}"/script/cmake-build posix -DOT_DAEMON=ON -DOT_PLATFORM_NETIF=ON -DOT_PLATFORM_UDP=ON "${options[@]}"
    ls "${builddir}"/openthread-daemon-*.deb
}

do_tar()
{
    local target_name="openthread-$1-$2"
    local build_dir="${OT_BUILDDIR}/cmake"
    tar -cf "${target_name}.tar" -C "${build_dir}" "${target_name}"
    mv "${target_name}.tar" "${build_dir}/"
    rm -rf "${build_dir:?}/${target_name}"
}

do_untar()
{
    local target_name="openthread-$1-$2"
    local build_dir="${OT_BUILDDIR}/cmake"
    tar -xf "${build_dir}/${target_name}.tar" -C "${build_dir}"
    rm "${build_dir:?}/${target_name}.tar"
}

envsetup()
{
    if [[ ${NODE_MODE} == 'rcp' ]]; then
        export RADIO_DEVICE="${OT_BUILDDIR}/cmake/openthread-simulation-${THREAD_VERSION}/examples/apps/ncp/ot-rcp"
        export OT_CLI_PATH="${OT_BUILDDIR}/cmake/openthread-posix-${THREAD_VERSION}/src/posix/ot-cli"
        export OT_NCP_PATH="${OT_BUILDDIR}/cmake/openthread-posix-${THREAD_VERSION}/src/posix/ot-ncp"

        if [[ ${THREAD_VERSION} == "1.2" ]]; then
            export RADIO_DEVICE_1_1="${OT_BUILDDIR}/cmake/openthread-simulation-1.1/examples/apps/ncp/ot-rcp"
            export OT_CLI_PATH_1_1="${OT_BUILDDIR}/cmake/openthread-posix-1.1/src/posix/ot-cli"
            export OT_NCP_PATH_1_1="${OT_BUILDDIR}/cmake/openthread-posix-1.1/src/posix/ot-ncp"
            export OT_CLI_PATH_1_2_BBR="${OT_BUILDDIR}/cmake/openthread-posix-1.2-bbr/src/posix/ot-cli"
            export OT_NCP_PATH_1_2_BBR="${OT_BUILDDIR}/cmake/openthread-posix-1.2-bbr/src/posix/ot-ncp"
        fi
    fi

    if [[ ! ${VIRTUAL_TIME+x} ]]; then
        # All expect tests only works in real time mode.
        VIRTUAL_TIME=1
        for arg in "$@"; do
            if [[ $arg == expect ]]; then
                VIRTUAL_TIME=0
                break
            fi
        done
    fi

    readonly VIRTUAL_TIME
    export NODE_TYPE VIRTUAL_TIME

    # CMake always works in verbose mode if VERBOSE exists in environments.
    if [[ ${VERBOSE} == 1 ]]; then
        export VERBOSE
    else
        export -n VERBOSE
    fi

    if [[ ${OT_OPTIONS+x} ]]; then
        read -r -a ot_extra_options <<<"${OT_OPTIONS}"
    else
        ot_extra_options=()
    fi
}

main()
{
    envsetup "$@"

    if [[ -z $1 ]]; then
        print_usage 1
    fi

    [[ ${NODE_MODE} == "rcp" ]] && echo "Using rcp mode" || echo "Using standalone mode"
    [[ ${NODE_TYPE} == "ncp-sim" ]] && echo "Using NCP node" || echo "Using CLI node"
    [[ ${VIRTUAL_TIME} == 1 ]] && echo "Using virtual time" || echo "Using real time"
    [[ ${THREAD_VERSION} == "1.2" ]] && echo "Using Thread 1.2 stack" || echo "Using Thread 1.1 stack"

    while [[ $# != 0 ]]; do
        case "$1" in
            clean)
                do_clean
                ;;
            build)
                do_build
                ;;
            cert)
                shift
                do_cert "$1"
                ;;
            cert_suite)
                shift
                do_cert_suite "$@"
                ;;
            unit)
                do_unit
                ;;
            help)
                print_usage
                ;;
            package)
                do_package
                ;;
            expect)
                shift
                do_expect "$@"
                ;;
            tar)
                shift
                do_tar "$@"
                ;;
            untar)
                shift
                do_untar "$@"
                ;;
        esac
        shift
    done
}

main "$@"