Skip to content
This repository was archived by the owner on Aug 5, 2022. It is now read-only.

CI Jenkinsfile,tester-exec: streamline, fix issue of lost test results #102

Merged
merged 8 commits into from
Apr 10, 2017
Merged

CI Jenkinsfile,tester-exec: streamline, fix issue of lost test results #102

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
e058dca
CI Jenkinsfile: reduce intermediate testinfo variable use
okartau Apr 6, 2017
ded023f
CI Jenkinsfile: tester archives files from top level
okartau Apr 6, 2017
b8161e0
CI Jenkinsfile: separate xUnit publish step as function
okartau Apr 6, 2017
2996143
CI Jenkinsfile: shift block left, eliminate extra unneeded indent
okartau Apr 6, 2017
f223203
CI Jenkinsfile: add lock section around step_xunit
okartau Apr 6, 2017
87b7147
CI tester-exec: remove symlinks after testing
okartau Apr 7, 2017
2a3c71a
CI testing: don't combine results into tarball
okartau Apr 10, 2017
15346e8
CI tester-exec.sh: reduce use of variables
okartau Apr 10, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
120 changes: 64 additions & 56 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,63 +122,47 @@ try {
def test_runs = [:]
for(int i = 0; i < test_devices.size(); i++) {
def test_device = test_devices[i]
// only if built for machine that this tester wants
if ( target_machine == mapping["${test_device}"] ) {
// testinfo_data may contain multiple lines stating different images
String[] separated_testinfo = testinfo_data["${target_machine}"].split("\n")
for (int m = 0; m < separated_testinfo.length; m++) {
def one_image_testinfo = separated_testinfo[m]
echo "Image #${m} to be tested on test_${test_device} info: ${separated_testinfo[m]}"
test_runs["test_${m}_${test_device}"] = {
node('refkit-tester') {
deleteDir() // clean workspace
echo "Testing test_${test_device} with image_info: ${one_image_testinfo}"
writeFile file: 'tester-exec.sh', text: tester_script
// append newline so that tester-exec.sh can parse it using "read"
one_image_testinfo += "\n"
// write testinfo file on this tester for this image, one line per tester
writeFile file: "testinfo.csv", text: one_image_testinfo
String[] one_testinfo_elems = one_image_testinfo.split(",")
def img = one_testinfo_elems[0]
try {
withEnv(["CI_BUILD_ID=${ci_build_id}",
"MACHINE=${mapping["${test_device}"]}",
"TEST_DEVICE=${test_device}" ]) {
sh 'chmod a+x tester-exec.sh && ./tester-exec.sh'
}
} catch (Exception e) {
throw e
} finally {
// read tests summary prepared by tester-exec.sh
// Here one tester adds it's summary piece to the global buffer.
global_sum_log += readFile "results-summary-${test_device}.${img}.log"
archiveArtifacts allowEmptyArchive: true,
artifacts: '**/*.log, **/*.xml, **/aft-results*.tar.bz2'
// only if built for machine that this tester wants
if ( target_machine == mapping["${test_device}"] ) {
// testinfo_data may contain multiple lines stating different images
String[] separated_testinfo = testinfo_data["${target_machine}"].split("\n")
for (int m = 0; m < separated_testinfo.length; m++) {
def one_image_testinfo = separated_testinfo[m]
echo "Image #${m} to be tested on test_${test_device} info: ${separated_testinfo[m]}"
test_runs["test_${m}_${test_device}"] = {
node('refkit-tester') {
deleteDir() // clean workspace
echo "Testing test_${test_device} with image_info: ${one_image_testinfo}"
writeFile file: 'tester-exec.sh', text: tester_script
// append newline so that tester-exec.sh can parse it using "read"
one_image_testinfo += "\n"
// create testinfo.csv on this tester describing one image
writeFile file: "testinfo.csv", text: one_image_testinfo
def img = one_image_testinfo.split(",")[0]
try {
withEnv(["CI_BUILD_ID=${ci_build_id}",
"MACHINE=${mapping["${test_device}"]}",
"TEST_DEVICE=${test_device}" ]) {
sh 'chmod a+x tester-exec.sh && ./tester-exec.sh'
}
step([$class: 'XUnitPublisher',
testTimeMargin: '3000',
thresholdMode: 1,
thresholds: [
[$class: 'FailedThreshold',
failureNewThreshold: '0',
failureThreshold: '0',
unstableNewThreshold: '99999',
unstableThreshold: '99999'],
[$class: 'SkippedThreshold',
failureNewThreshold: '99999',
failureThreshold: '99999',
unstableNewThreshold: '99999',
unstableThreshold: '99999']],
tools: [[$class: 'JUnitType',
deleteOutputFiles: true,
failIfNotNew: true,
pattern: 'TEST-*.xml',
skipNoTestFiles: false,
stopProcessingIfError: true]]])
} // node
} // test_runs =
} // for m
} // if target_machine == mapping
} catch (Exception e) {
throw e
} finally {
// read tests summary prepared by tester-exec.sh
// Here one tester adds it's summary piece to the global buffer.
global_sum_log += readFile "results-summary-${test_device}.${img}.log"
archiveArtifacts allowEmptyArchive: true,
artifacts: '*.log, *.xml'
}
// without locking we may lose tester result set(s)
// if testers run xunit step in nearly same time
lock(resource: "step-xunit") {
step_xunit()
}
} // node
} // test_runs =
} // for
} // if target_machine == mapping
} // for i
stage('Parallel test run') {
set_gh_status_pending(is_pr, 'Testing')
Expand Down Expand Up @@ -300,3 +284,27 @@ def set_gh_status_pending(is_pr, _msg) {
setGitHubPullRequestStatus state: 'PENDING', context: "${env.JOB_NAME}", message: "${_msg}"
}
}

def step_xunit() {
step([$class: 'XUnitPublisher',
testTimeMargin: '3000',
thresholdMode: 1,
thresholds: [
[$class: 'FailedThreshold',
failureNewThreshold: '0',
failureThreshold: '0',
unstableNewThreshold: '99999',
unstableThreshold: '99999'],
[$class: 'SkippedThreshold',
failureNewThreshold: '99999',
failureThreshold: '99999',
unstableNewThreshold: '99999',
unstableThreshold: '99999']],
tools: [
[$class: 'JUnitType',
deleteOutputFiles: true,
failIfNotNew: true,
pattern: 'TEST-*.xml',
skipNoTestFiles: false,
stopProcessingIfError: true]]])
}
24 changes: 9 additions & 15 deletions docker/tester-exec.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ testimg() {
_IMG_NAME=$1
TEST_SUITE_FILE=$2
TEST_CASES_FILE=$3
_IMG_NAME_MACHINE=${_IMG_NAME}-${MACHINE}

# Get test suite
wget ${_WGET_OPTS} ${TEST_SUITE_FOLDER_URL}/${_IMG_NAME}/${TEST_SUITE_FILE}
Expand All @@ -34,21 +33,16 @@ testimg() {
cp $HOME/.config.ini.wlan ${_WLANCONF}
chmod 644 ${_WLANCONF}

FN_BASE=${_IMG_NAME_MACHINE}-${CI_BUILD_ID}
FILENAME=${FN_BASE}.wic
FILENAME_BMAP=${FILENAME}.bmap
FILENAME_XZ=${FILENAME}.xz
FILENAME_ZIP=${FILENAME}.zip

FILENAME=${_IMG_NAME}-${MACHINE}-${CI_BUILD_ID}.wic
set +e
wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME_BMAP}
wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME_XZ} -O - | unxz - > ${FILENAME}
wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME}.bmap
wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME}.xz -O - | unxz - > ${FILENAME}
if [ ! -s ${FILENAME} ]; then
wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME_ZIP}
if [ -s ${FILENAME_ZIP} ]; then
unzip ${FILENAME_ZIP}
wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME}.zip
if [ -s ${FILENAME}.zip ]; then
unzip ${FILENAME}.zip
else
echo "ERROR: No file ${FILENAME_XZ} or ${FILENAME_ZIP} found, can not continue."
echo "ERROR: No file ${FILENAME}.xz or ${FILENAME}.zip found, can not continue."
exit 1
fi
fi
Expand Down Expand Up @@ -76,6 +70,8 @@ testimg() {
daft ${DEVICE} ${FILENAME} --record
AFT_EXIT_CODE=$?

# delete symlinks, these point outside of local set and are useless
find . -type l -print -delete
# modify names inside TEST-*.xml files to contain device and img_name
# as these get shown in same xUnit results table in Jenkins
sed -e "s/name=\"oeqa/name=\"${DEVICE}.${_IMG_NAME}.oeqa/g" -i TEST-*.xml
Expand Down Expand Up @@ -107,8 +103,6 @@ testimg() {
echo " Run rate:${run_rate}% Pass rate of total:${pass_rate_of_total}% Pass rate of exec:${pass_rate_of_exec}%" >> $sumfile
fi
echo "-------------------------------------------------------------------" >> $sumfile
# combine artifacts into single file for easier download
tar c --ignore-failed-read results* *.xml *.log | bzip2 -c9 > aft-results_${DEVICE}_${_IMG_NAME}_${TEST_SUITE_FILE}.tar.bz2
set -e

return ${AFT_EXIT_CODE}
Expand Down