ota-test-secureboot

#!/usr/bin/env bash

# TODO: Add provision to store logcat logs for critical instances.
# TODO: Sideload test-cases are not covered.
# TODO: Consider creating configuration file for parameters.
# TODO: Interrupt at different OTA completion %age not in random time intervals.
# TODO: Use mechanical arm for VOL-UP key press and remove this workaround.
# TODO: In Official build test-cases, VOL-Up key intervention is required and USB connection loses.

# Commented this, as this is breaking on negative test case execution and 'interrupt_OTA_update' tests when DUT is not detected.
# Bail on any errors
#set -e

# Allow job control
set -m

#set -x
#trap read debug

# Print date and time after each command
export PROMPT_COMMAND=date

# User Configurations
PRODUCT_NAME=$2
BUILD_VARIANT=$3
USERNAME=$4
PASSWORD=$5
BUILD_TARGET=$PRODUCT_NAME-$BUILD_VARIANT
if [ "$6" == "IRDA" ]; then
    TARGET_FILE_SUFFIX=COHOL00
    OFFICIAL_REL_DST_B=$((OFFICIAL_REL_SRC_A+10))
    OFFICIAL_REL_URL=https://jfstor001.jf.intel.com/artifactory/irda-jf/irda-l/releases/$TARGET_FILE_SUFFIX
else
    TARGET_FILE_SUFFIX=GMINLMASTER
    OFFICIAL_REL_DST_B=$((OFFICIAL_REL_SRC_A+1))
    OFFICIAL_REL_URL=https://mcg-depot.intel.com/artifactory/simple/otc-jf/releases/gmin-l/
fi
OFFICIAL_REL_SRC_A=$7
# ISSUE: ANDROID_SERIAL doesn't behave well when PFT UI is open on same machine.
if [ ! "$8" ]; then ANDROID_DEVICEID=$(adb devices | grep -v 'List' | awk '{print ($1)}'); else ANDROID_SERIAL=$8; ANDROID_DEVICEID=$8; fi


case $1 in
    -q)
        export OTA_TESTS=1
        ;;
    -s)
        export OTA_TESTS=2
        STRESS_TEST_COUNT=1;
        ;;
    -r)
        export OTA_TESTS=2
        STRESS_TEST_COUNT=100;
        ;;
    -o)
        export OTA_TESTS=4
        ;;
    -a)
        export OTA_TESTS=7
        STRESS_TEST_COUNT=100;
        ;;
    *)
        echo "Usage:"
        echo
        echo "  ota-test-secureboot [-q -s -r -o -a] PRODUCT_NAME BUILD_VARIANT IDSID PASSWORD GMINL/IRDA OFFICIAL_REL_SRC_A ANDROID_DEVICEID"
        echo
        echo "    Valid options [-o -q -r -s -a] :"
        echo "      -q: Execute QUICK test cases, requires first two parameters only"
        echo "      -s: Execute STANDARD test cases, requires first two parameters only"
        echo "      -r: Execute STANDARD test cases with stress test , requires first two parameters only"
        echo "      -o: Execute test cases on OFFICIAL BUILD"
        echo "      -a: Execute ALL (QUICK + STANDARD + STRESS + OFFICIAL BUILD) test cases"
        echo
        echo "    Parameters"
        echo "      PRODUCT_NAME: Product name like byt_mrd7 / ecs_e7 / fxn_anchor8 / coho"
        echo "      BUILD_VARIANT: userdebug / user / eng"
        echo "      IDSID: Intel's IDSID"
        echo "      PASSWORD: Password associated with IDSID"
        echo "      GMINL/IRDA : Supply GMINL or IRDA for selecting official build. "
        echo "      OFFICIAL_REL_SRC_A: Official Release # '107' to be used as source, its consecutive release will be used as Destination in GMINL and +10 in IRDA"
        echo "      ANDROID_DEVICEID [Optional parameter]: Supply Android DUT Device ID in case multiple devices are connected to Host machine "
        echo
        [[ $PS1 ]]&&return||exit;
        ;;
esac


##### Functions

function confirmStateSleepNReboot {
    WaitForDUTStateTOCnt=1
    if [ "$1" == "fastboot" ]; then COMMAND=fastboot; else COMMAND=adb; fi
    while true
    do
        if ($COMMAND -s $ANDROID_DEVICEID devices | grep $1) || [ $WaitForDUTStateTOCnt -eq 200 ]
        then
            echo $(date) 'Entered '$1' Mode in '`expr $WaitForDUTStateTOCnt \\* 3`' seconds'; break
        else
            echo $(date) 'Waiting to enter into '$1' Mode' $((WaitForDUTStateTOCnt++)); sleep 3
        fi
    done
    sleep $2
    if [ "$3" == "reboot" ]; then $COMMAND -s $ANDROID_DEVICEID reboot; fi
}


function provisionDevice {
    echo $(date) 'provisionDevice('$1')'
    adb -s $ANDROID_DEVICEID reboot dnx
    confirmStateSleepNReboot fastboot 2 noReboot
    echo $(date) 'Flash Source software A'
    cflasher -f $1 -x flash.xml
    echo $(date) 'Wait for UI to reach lock screen'
    adb -s $ANDROID_DEVICEID wait-for-device
    # If 'Boot has been completed' gives timing trouble, use 'HWC_POWER_MODE_OFF' instead
    adb -s $ANDROID_DEVICEID logcat -v time | grep -e 'Boot has been completed' --max-count=1
}

function verify_SW {
    echo $(date) 'verify_SW('$1')'
    adb -s $ANDROID_DEVICEID wait-for-device
    echo $(date) 'Reboot into fastboot mode'
    adb -s $ANDROID_DEVICEID reboot fastboot
    confirmStateSleepNReboot fastboot 2 noReboot
    # ISSUE: If VOL-UP key is not pressed and timeout happens. DUT boots into normal mode than fastboot mode.
    echo $(date) 'Confirm whether update in DUT is done'
    ./device/intel/build/verify_from_target_files $1
    echo $(date) 'Boot into normal mode'
    fastboot -s $ANDROID_DEVICEID reboot
    adb -s $ANDROID_DEVICEID wait-for-device
    adb -s $ANDROID_DEVICEID logcat -v time | grep -e 'Boot has been completed' --max-count=1
    batteryLevel=$(adb -s $ANDROID_DEVICEID shell dumpsys battery | grep level | awk '{print ($2)}' | tr -d '\r')
    if [ $batteryLevel -le 10 ]; then echo Low Battery = $batteryLevel, Charging for 30 minutes; sleep 1800; else echo Battery is $batteryLevel%; fi
}

function interrupt_OTA_update {
    echo $(date) 'Waiting for DUT to enter recovery mode'
    echo $(date) 'Interrupt/Power-Off DUT at different completion %age during OTA update.'
    #confirmStateSleepNReboot recovery 5 reboot
    confirmStateSleepNReboot recovery 10 reboot
    #confirmStateSleepNReboot recovery 15 reboot
    #confirmStateSleepNReboot recovery 20 reboot
    #confirmStateSleepNReboot recovery 25 reboot
    #confirmStateSleepNReboot recovery 30 reboot
    #confirmStateSleepNReboot recovery 35 reboot
    #confirmStateSleepNReboot recovery 40 reboot
    echo $(date) 'Let OTA update finish which can even take few minutes and boot to UI'
    adb -s $ANDROID_DEVICEID wait-for-device
    adb -s $ANDROID_DEVICEID logcat -v time | grep -e 'Boot has been completed' --max-count=1
}

##### MAIN CODE begins

echo $(date) 'Make sure BIOS settings are correct'
echo $(date) 'This script will take few hours to execute !!!'
echo $(date) 'Setup environment and confirm DUT provisioning before tests execution'

echo $(date) 'Skip VOL-UP key user inputs using this work-around.'
cd hardware/intel/kernelflinger/
if git log | grep -e 'I7993d46f75aa11b7efec032daf3272f4c4011de9' --max-count=1
then
    echo $(date) 'Commit Exists'
else
    git fetch git://android.intel.com/a/bsp/hardware/intel/efi/kernelflinger refs/changes/75/297975/1 && git cherry-pick FETCH_HEAD
fi
cd -

echo $(date) 'Setting Build Environment'
. build/envsetup.sh
lunch $BUILD_TARGET

if [ "$OTA_TESTS" -eq 7 ]; then
    echo $(date) "ALL OTA Secure boot TESTS : QUICK + STANDARD + STRESS + OFFICIAL BUILD"
fi

if [ "$OTA_TESTS" -eq 1 ] || [ "$OTA_TESTS" -eq 7 ]; then
    echo $(date) "Test_Q: QUICK OTA Secure boot TESTS"

    rm -rf ota_q
    echo $(date) 'Test_Q: Create quick test packages : ota-test-prepare -q'
    ./device/intel/build/ota-test-prepare -q
    echo $(date) 'Test_Q: Backup quick test packages to another folder to save from overwriting'
    cp -rfv ota ota_q

    echo $(date) 'Test_Q: Test update from A->B and DUT will boot with B'
    provisionDevice ota/flashfiles-A-testkey.zip
    adb -s $ANDROID_DEVICEID shell getprop > ota_q/getprop_A.log
    verify_SW ota/tfp-A-testkey.zip
    ./device/intel/build/flash-update ota/ota-A-B-testkey.zip $ANDROID_DEVICEID
    interrupt_OTA_update
    adb -s $ANDROID_DEVICEID shell getprop > ota_q/getprop_B.log
    verify_SW ota/tfp-B-testkey.zip

    echo $(date) 'Test_Q: Test update from B->A'
    ./device/intel/build/flash-update ota/ota-B-A-testkey.zip $ANDROID_DEVICEID
    interrupt_OTA_update
    adb -s $ANDROID_DEVICEID shell getprop > ota_q/getprop_A1.log
    verify_SW ota/tfp-A-testkey.zip
fi

if [ "$OTA_TESTS" -eq 2 ] || [ "$OTA_TESTS" -eq 7 ]; then
    echo $(date) "Test_S: STANDARD OTA Secure boot TESTS"

    rm -rf ota_s

    # Provision DUT before starting tests to ensure build/DUT is good.
    make -j12
    make flashfiles -j12
    provisionDevice $OUT/$PRODUCT_NAME-flashfiles-eng.$USER.zip
    adb -s $ANDROID_DEVICEID shell getprop > ota_s/getprop_A.log

    echo $(date) 'Test_S: Create standard test packages : ota-test-prepare -s'
    ./device/intel/build/ota-test-prepare -s
    echo $(date) 'Test_S: Backup test packages to another folder to save from overwriting'
    cp -rfv ota ota_s

: <<'COMMENT_LOCAL-TEST_UPDATE-KEY'
    echo $(date) 'Test_S: Provision with production keys'
    fastboot -s $ANDROID_DEVICEID oem unlock
    sleep 10
    fastboot -s $ANDROID_DEVICEID flash efirun hardware/intel/efi_prebuilts/efitools/linux-x86_64/production-test/LockDownPT.efi
    sleep 10
    fastboot -s $ANDROID_DEVICEID reboot-bootloader
    sleep 10
    fastboot -s $ANDROID_DEVICEID oem verified
    sleep 10
    fastboot -s $ANDROID_DEVICEID reboot-bootloader
COMMENT_LOCAL-TEST_UPDATE-KEY

    provisionDevice ota/flashfiles-A.zip
    adb -s $ANDROID_DEVICEID shell getprop > ota_s/getprop_A0.log
    echo $(date) 'Test_S: verify provisioned SW with wrong input, negative testcase'
    verify_SW ota/tfp-B-testkey.zip
    echo $(date) 'Test_S: verify provisioned SW with correct input'
    # DOUBT: Which key we should use for verification.
    verify_SW ota/tfp-A.zip
    echo $(date) 'Test_S: verify provisioned SW with correct input and interruption'
    # ISSUE: DUT becomes unresponsive after executing this test-case.
: <<'COMMENT_FAILURE_TESTCASE_VERIFY_INTERRUPT'
    adb reboot fastboot
    confirmStateSleepNReboot fastboot 2 noReboot
    ./device/intel/build/verify_from_target_files ota/tfp-A.zip &
    sleep 5
    pkill fastboot
COMMENT_FAILURE_TESTCASE_VERIFY_INTERRUPT

    echo $(date) 'Test_S: Full image update from A->B'
    echo $(date) 'Test_S: Negative test case, send same SW which is provisioned.'
    # ISSUE: ./device/intel/build/flash-update fails if multiple devices are connected with Host. And there is no provision for providing DUT DeviceID.
    ./device/intel/build/flash-update ota/ota-A.zip $ANDROID_DEVICEID
    interrupt_OTA_update
    # ISSUE: It tries to update the provisioned SW.
    adb -s $ANDROID_DEVICEID shell getprop > ota_s/getprop_A0.1.log
    verify_SW ota/tfp-A.zip
    echo $(date) 'Test_S: Provide correct inputs for full image update.'
    ./device/intel/build/flash-update ota/ota-B.zip $ANDROID_DEVICEID
    interrupt_OTA_update
    adb -s $ANDROID_DEVICEID shell getprop > ota_s/getprop_B1.log
    verify_SW ota/tfp-B.zip
    # ISSUE: Cannot update back to Software A. Remove comments and below provisioning line after this issue is resolved.
: <<'COMMENT_FAILURE_TESTCASE_FULL_OTA_ANTIROLLBACK'
    echo $(date) 'Test_S: Full image update from B->A'
    ./device/intel/build/flash-update ota/ota-A.zip $ANDROID_DEVICEID
    interrupt_OTA_update
COMMENT_FAILURE_TESTCASE_FULL_OTA_ANTIROLLBACK
    provisionDevice ota/flashfiles-A.zip
    adb -s $ANDROID_DEVICEID shell getprop > ota_s/getprop_A1.log
    verify_SW ota/tfp-A.zip

    LoopCounter=1
    while [ $LoopCounter -le $STRESS_TEST_COUNT ]
    do
        echo $(date) 'Test_R: OTA Update Count # ' $((LoopCounter++))
        echo $(date) 'Test_R: Update from A->B'
        ./device/intel/build/flash-update ota/ota-A-B.zip $ANDROID_DEVICEID
        interrupt_OTA_update
        adb -s $ANDROID_DEVICEID shell getprop > ota_s/getprop_B$LoopCounter.log
        verify_SW ota/tfp-B.zip

        echo $(date) 'Test_R: Update from B->A'
        echo $(date) 'Test_R: Send wrong update package, it should fail. Negative testcase'
        ./device/intel/build/flash-update ota/ota-A-B.zip $ANDROID_DEVICEID
        echo $(date) 'Test_R: Wait for device to fail update and reboot into normal mode'
        sleep 15
        adb -s $ANDROID_DEVICEID wait-for-device
        adb -s $ANDROID_DEVICEID logcat -v time | grep -e 'Boot has been completed' --max-count=1
        verify_SW ota/tfp-B.zip
        echo $(date) 'Test_R: Send correct update package'
        ./device/intel/build/flash-update ota/ota-B-A.zip $ANDROID_DEVICEID
        interrupt_OTA_update
        adb -s $ANDROID_DEVICEID shell getprop > ota_s/getprop_A$LoopCounter.log
        verify_SW ota/tfp-A.zip
    done
    # ISSUE: adb reboot dnx stops working after exiting the loop.
fi

if [ "$OTA_TESTS" -eq 4 ] || [ "$OTA_TESTS" -eq 7 ]; then
    echo $(date) "OFFICIAL BUILD OTA Secure boot TESTS"

    echo $(date) 'Test_O: Download Official Releases'
    mkdir -p downloadRelease
    cd downloadRelease
    wget --no-proxy --user=$USERNAME --password=$PASSWORD $OFFICIAL_REL_URL$OFFICIAL_REL_SRC_A/$BUILD_TARGET/out/dist/$PRODUCT_NAME-target_files-$TARGET_FILE_SUFFIX$OFFICIAL_REL_SRC_A.zip
    wget --no-proxy --user=$USERNAME --password=$PASSWORD $OFFICIAL_REL_URL$OFFICIAL_REL_DST_B/$BUILD_TARGET/out/dist/$PRODUCT_NAME-target_files-$TARGET_FILE_SUFFIX$OFFICIAL_REL_DST_B.zip
    croot

    rm -rf ota_o
    echo $(date) 'Test_O: Create test packages from Official Release'
    echo $(date) 'Test_O: Create TFP source package'
    ./device/intel/build/ota-test-prepare -s -t downloadRelease/$PRODUCT_NAME-target_files-GMINLMASTER$OFFICIAL_REL_SRC_A.zip A
    echo $(date) 'Test_O: Create TFP destination package'
    ./device/intel/build/ota-test-prepare -s -t downloadRelease/$PRODUCT_NAME-target_files-GMINLMASTER$OFFICIAL_REL_DST_B.zip B
    echo $(date) 'Test_O: Create ota package from A->B'
    ./build/tools/releasetools/ota_from_target_files --verbose -i ota/tfp-A.zip ota/tfp-B.zip ota/ota-A-B.zip
    echo $(date) 'Test_O: Create ota package from B->A'
    ./build/tools/releasetools/ota_from_target_files --verbose -i ota/tfp-B.zip ota/tfp-A.zip ota/ota-B-A.zip
    echo $(date) 'Test_O: Backup official test packages to another folder to save from overwriting'
    rm -rf downloadRelease/
    cp -rfv ota ota_o

    echo $(date) 'Test_O: Full image update from A->B'
    provisionDevice ota/flashfiles-A.zip
    adb -s $ANDROID_DEVICEID shell getprop > ota_o/getprop_A.log
    ./device/intel/build/flash-update ota/ota-B.zip $ANDROID_DEVICEID
    # ISSUE: Enters into recovery screen with options enabled. Signature verification failed.
    # ISSUE: Enters into recovery screen with options enabled. While OTA update in recovery mode, UI was blank.
    # ISSUE: If VOL-UP is not pressed immediately before DUT goes to sleep, OTA process doesn't start on Power-On again.
    confirmStateSleepNReboot recovery 0 noReboot
    adb -s $ANDROID_DEVICEID wait-for-device
    adb -s $ANDROID_DEVICEID logcat -v time | grep -e 'Boot has been completed' --max-count=1
    # TODO: Confirm the verification key input zip file name
    #verify_SW ota/ota-B.zip
    adb -s $ANDROID_DEVICEID shell getprop > ota_o/getprop_B.log
    echo $(date) 'Test_O: Full image update from B->A'
    ./device/intel/build/flash-update ota/ota-A.zip $ANDROID_DEVICEID
    # ISSUE: Cannot update back to Software A.
    #interrupt_OTA_update
    confirmStateSleepNReboot recovery 0 noReboot
    adb -s $ANDROID_DEVICEID wait-for-device
    adb -s $ANDROID_DEVICEID logcat -v time | grep -e 'Boot has been completed' --max-count=1
    #verify_SW ota/tfp-A.zip
    adb -s $ANDROID_DEVICEID shell getprop > ota_o/getprop_A1.log

    echo $(date) 'Test_O: Update from A->B'
    # TODO: Remove below lines as SW A already rolled-back so no need to reprovision.
    provisionDevice ota/flashfiles-A.zip
    adb -s $ANDROID_DEVICEID shell getprop > ota_o/getprop_A2.log
    ./device/intel/build/flash-update ota/ota-A-B.zip $ANDROID_DEVICEID
    # Issue: Interrupting OTA is not working.
    #interrupt_OTA_update
    adb -s $ANDROID_DEVICEID shell getprop > ota_o/getprop_B1.log
    verify_SW ota/tfp-B.zip

    echo $(date) 'Test_O: Update from B->A'
    ./device/intel/build/flash-update ota/ota-B-A.zip $ANDROID_DEVICEID
    #interrupt_OTA_update
    adb -s $ANDROID_DEVICEID shell getprop > ota_o/getprop_A2.log
    verify_SW ota/tfp-A.zip
fi

rm -rf ota/

echo $(date) 'Please manually delete ota_q/ota_s/ota_r folders, if not required.'
echo $(date) 'OTA Secure test cases finished execution'

##### MAIN CODE ends