android/xts: add support for vts module level static sharding

automated/android/noninteractive-tradefed/setup.sh

@@ -15,7 +15,7 @@ if [ -n "${ANDROID_VERSION}" ] && echo "${ANDROID_VERSION}" | grep -q  "aosp-mas
 #   JDK="openjdk-8-jdk-headless"
 fi
 
-PKG_DEPS="coreutils usbutils curl wget zip xz-utils python-lxml python-setuptools python-pexpect aapt lib32z1-dev libc6-dev-i386 lib32gcc1 libc6:i386 libstdc++6:i386 libgcc1:i386 zlib1g:i386 libncurses5:i386 python-dev python-protobuf protobuf-compiler python-virtualenv python-pip python-pexpect psmisc"
+PKG_DEPS="coreutils usbutils curl wget zip xz-utils python3-lxml python-lxml python-setuptools python-pexpect aapt lib32z1-dev libc6-dev-i386 lib32gcc1 libc6:i386 libstdc++6:i386 libgcc1:i386 zlib1g:i386 libncurses5:i386 python-dev python-protobuf protobuf-compiler python-virtualenv python-pip python-pexpect psmisc"
 
 dist_name
 case "${dist}" in

automated/android/noninteractive-tradefed/tradefed.sh

@@ -19,13 +19,15 @@ FAILURES_PRINTED="0"
 AP_SSID=""
 # WIFI AP KEY
 AP_KEY=""
+SHARD_NUMBER=1
+SHARD_INDEX=1
 
 usage() {
-    echo "Usage: $0 [-o timeout] [-n serialno] [-c cts_url] [-t test_params] [-p test_path] [-r <aggregated|atomic>] [-f failures_printed] [-a <ap_ssid>] [-k <ap_key>]" 1>&2
+    echo "Usage: $0 [-o timeout] [-n serialno] [-c cts_url] [-t test_params] [-p test_path] [-r <aggregated|atomic>] [-f failures_printed] [-a <ap_ssid>] [-k <ap_key>] [-s <shard_number>] [-i <shard_index>]" 1>&2
     exit 1
 }
 
-while getopts ':o:n:c:t:p:r:f:a:k:' opt; do
+while getopts ':o:n:c:t:p:r:f:a:k:s:i:' opt; do
     case "${opt}" in
         o) TIMEOUT="${OPTARG}" ;;
         n) export ANDROID_SERIAL="${OPTARG}" ;;
@@ -36,6 +38,8 @@ while getopts ':o:n:c:t:p:r:f:a:k:' opt; do
         f) FAILURES_PRINTED="${OPTARG}" ;;
         a) AP_SSID="${OPTARG}" ;;
         k) AP_KEY="${OPTARG}" ;;
+        s) SHARD_NUMBER="${OPTARG}" ;;
+        i) SHARD_INDEX="${OPTARG}" ;;
         *) usage ;;
     esac
 done
@@ -67,6 +71,11 @@ file_name=$(basename "${TEST_URL}")
 unzip -q "${file_name}"
 rm -f "${file_name}"
 
+# Split test module by shard_number and only keep shard #shard_index.
+if [ "${SHARD_NUMBER}" -gt 1 ]; then
+    python3 ./xts_module_sharding.py -p "${TEST_PATH}" -t "${TEST_PARAMS}" -n "${SHARD_NUMBER}" -i "${SHARD_INDEX}" -v
+fi
+
 if [ -d "${TEST_PATH}/results" ]; then
     mv "${TEST_PATH}/results" "${TEST_PATH}/results_$(date +%Y%m%d%H%M%S)"
 fi

automated/android/noninteractive-tradefed/tradefed.yaml

@@ -43,6 +43,22 @@ params:
     # to avoid shutting down by high temperature when run the cts vts test
     SET_GOVERNOR_POWERSAVE: "false"
 
+    # XTS test plan is organized by test modules. Big test plan can be split
+    # into a set of small test plans already. For test modules that are time
+    # consuming and cause stability issue, the below variables can be defined
+    # to shard a big module into small chunks so that they can be run with
+    # multiple single node LAVA jobs in parallel.
+    # Known supported modules:
+    # - vts_kernel_net_tests
+    # - vts_linux_kselftest_*
+    # - vts_ltp_test_*
+    # Number of shards that will be done, defaults to 1 which is the same as
+    # no sharding.
+    SHARD_NUMBER: 1
+    # Which shard to run, defaults to 1 which is the same as no sharding and
+    # run it as XTS upstream decides.
+    SHARD_INDEX: 1
+
 run:
     steps:
         - cd ./automated/android/noninteractive-tradefed
@@ -57,7 +73,7 @@ run:
         - chown testuser:testuser .
         - if echo "${TEST_REBOOT_EXPECTED}" |grep -i "true" ; then ./monitor_fastboot.sh & fi
         - ./monitor_adb.sh &
-        - sudo -u testuser ./tradefed.sh  -o "${TIMEOUT}" -c "${TEST_URL}" -t "${TEST_PARAMS}" -p "${TEST_PATH}" -r "${RESULTS_FORMAT}" -n "${ANDROID_SERIAL}" -f "${FAILURES_PRINTED}" -a "${AP_SSID}" -k "${AP_KEY}" || true
+        - sudo -u testuser ./tradefed.sh  -o "${TIMEOUT}" -c "${TEST_URL}" -t "${TEST_PARAMS}" -p "${TEST_PATH}" -r "${RESULTS_FORMAT}" -n "${ANDROID_SERIAL}" -f "${FAILURES_PRINTED}" -a "${AP_SSID}" -k "${AP_KEY}" -s "${SHARD_NUMBER}" -i "${SHARD_INDEX}"|| true
         # Upload test log and result files to artifactorial.
         - cp -r ./${TEST_PATH}/results ./output/ || true
         - cp -r ./${TEST_PATH}/logs ./output/ || true

automated/android/noninteractive-tradefed/xts_module_sharding.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+
+import argparse
+import logging
+import os
+import sys
+from lxml import etree
+
+
+def get_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="XTS module static sharding parser")
+    parser.add_argument(
+        "-p",
+        "--test-path",
+        required=True,
+        # TODO: check the possibility to split cts plan/config.
+        choices=["android-vts"],
+        help="Path to tradefed package top directory.",
+    )
+    parser.add_argument(
+        "-t", "--test-params", required=True, help="Tradefed shell test parameters."
+    )
+    parser.add_argument(
+        "-n",
+        "--shard-number",
+        required=True,
+        help="The number of shards of a test module.",
+    )
+    parser.add_argument(
+        "-i",
+        "--shard-index",
+        required=True,
+        help="The index of the test module shard to run.",
+    )
+    parser.add_argument(
+        "-v", "--verbose", action="store_true", default=False, help="Be verbose."
+    )
+
+    return parser
+
+
+def shards(lst: list, n: int) -> list:
+    "Yield successive n-sized list shards from a given list."
+    for i in range(0, len(lst), n):
+        yield lst[i : i + n]
+
+
+def update_config(config: str, shard_number: int, shard_index: int) -> None:
+    """
+    Update module config to keep the tests_number//shard_number sized
+    shard #shard_index only.
+    """
+    tree = etree.parse(config)
+    root = tree.getroot()
+
+    # vts test modules like ltp and kselftest modules contain
+    # name="test-command-line" attribute for each sub-test. It can be
+    # used to identify tests. Other options like below ones should be
+    # kept for every shard.
+    # <option name="skip-binary-check" value="true"/>
+    # <option name="per-binary-timeout" value="1080000"/>
+    options = root.xpath('/configuration/test/option[@name="test-command-line"]')
+    sub_test_number = len(options)
+    if sub_test_number == 0:
+        logging.warning(
+            'Test options with name="test-command-line" attribute not found!'
+        )
+        logging.warning("Test sharding skipped.")
+        sys.exit(0)
+    logging.info(f"Original test size: {sub_test_number}")
+    if sub_test_number < shard_number:
+        logging.warning(f"Test number {sub_test_number} is smaller than shard number.")
+        logging.warning("Test sharding skipped.")
+        sys.exit(0)
+    n = sub_test_number // shard_number
+    logging.info(f"Test shard size: {n}")
+
+    count = 1
+    logging.info(f"Updating module config to keep {n}-sized shard #{shard_index} only")
+    for options_shard in shards(options, n):
+        if count != shard_index:
+            for option in options_shard:
+                option.getparent().remove(option)
+        count += 1
+    updated_size = len(
+        root.xpath('/configuration/test/option[@name="test-command-line"]')
+    )
+    logging.info(f"Updated test size: {updated_size}")
+
+    tree.write(config, pretty_print=True, xml_declaration=True, encoding="utf-8")
+    logging.info(f"Updated {config}")
+
+
+def main():
+    args = get_parser().parse_args()
+    logging.basicConfig(
+        level=logging.DEBUG if args.verbose else logging.INFO,
+        format="%(asctime)s: %(levelname)s: %(funcName)s: %(message)s",
+    )
+    logging.debug(f"Arguments: {args}")
+
+    # Find the module config file.
+    # Both --module and --include-filter args can be used to specify
+    # test module. Usage examples:
+    # vts: run vts-kernel [--module module]
+    # cts: run cts [--include-filter module]
+    supported_args = [
+        "--module",
+        "--include-filter",
+    ]
+    params = args.test_params.split()
+    count = 0
+    for supported_arg in supported_args:
+        count += params.count(supported_arg)
+    if count > 1:
+        logging.warning(
+            f"module sharding supports only one module but {count} modules found."
+        )
+        logging.info("Module sharding skipped.")
+        # exit with 0 to let tests to run.
+        sys.exit(0)
+    for param in params:
+        if param in supported_args:
+            param_index = params.index(param)
+            module_name = params[param_index + 1]
+    module_config = os.path.join(
+        args.test_path, "testcases", module_name, module_name + ".config"
+    )
+    module_config = os.path.realpath(module_config)
+    if os.path.exists(module_config):
+        logging.info(f"{module_config} found.")
+    else:
+        logging.warning(f"{module_config} not found!")
+        logging.warning("Module sharding skipped.")
+        sys.exit(0)
+
+    # Shard number and index sanity checks.
+    shard_number = int(args.shard_number)
+    shard_index = int(args.shard_index)
+    if shard_index > shard_number:
+        logging.error("Shard index should be less than or equal to shard number.")
+        sys.exit(1)
+    if shard_number == 1:
+        sys.exit(0)
+    if shard_number > 1:
+        logging.info(f"Splitting {module_name} into {shard_number} shards ...")
+
+    update_config(module_config, shard_number, shard_index)
+
+
+if __name__ == "__main__":
+    main()