From e058dcaa6e8099e653c7cd634003c383bbdf040d Mon Sep 17 00:00:00 2001 From: Olev Kartau Date: Thu, 6 Apr 2017 10:32:29 +0300 Subject: [PATCH 1/8] CI Jenkinsfile: reduce intermediate testinfo variable use Split and use first elem can be done on one line. Comment made shorted and more concise. Signed-off-by: Olev Kartau --- Jenkinsfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 47dce5d19f..9841377b16 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -136,10 +136,9 @@ try { writeFile file: 'tester-exec.sh', text: tester_script // append newline so that tester-exec.sh can parse it using "read" one_image_testinfo += "\n" - // write testinfo file on this tester for this image, one line per tester + // create testinfo.csv on this tester describing one image writeFile file: "testinfo.csv", text: one_image_testinfo - String[] one_testinfo_elems = one_image_testinfo.split(",") - def img = one_testinfo_elems[0] + def img = one_image_testinfo.split(",")[0] try { withEnv(["CI_BUILD_ID=${ci_build_id}", "MACHINE=${mapping["${test_device}"]}", From ded023f86e4ecd04acbac757ec189155bf5697be Mon Sep 17 00:00:00 2001 From: Olev Kartau Date: Thu, 6 Apr 2017 10:34:53 +0300 Subject: [PATCH 2/8] CI Jenkinsfile: tester archives files from top level Tester archived artifacts using "any path" wildcard **/ which has been dragged along from some older life of this script. Currently all result files are in top level directory. Signed-off-by: Olev Kartau --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9841377b16..f3670abf52 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -152,7 +152,7 @@ try { // Here one tester adds it's summary piece to the global buffer. global_sum_log += readFile "results-summary-${test_device}.${img}.log" archiveArtifacts allowEmptyArchive: true, - artifacts: '**/*.log, **/*.xml, **/aft-results*.tar.bz2' + artifacts: '*.log, *.xml, aft-results*.tar.bz2' } step([$class: 'XUnitPublisher', testTimeMargin: '3000', From b8161e0d1dd1c0b38061c6ecebe61d144baf0546 Mon Sep 17 00:00:00 2001 From: Olev Kartau Date: Thu, 6 Apr 2017 10:43:47 +0300 Subject: [PATCH 3/8] CI Jenkinsfile: separate xUnit publish step as function This improves readability and shortens lines, also makes it easier to write repeated tester blocks. Signed-off-by: Olev Kartau --- Jenkinsfile | 45 +++++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f3670abf52..edf06e703f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -154,26 +154,7 @@ try { archiveArtifacts allowEmptyArchive: true, artifacts: '*.log, *.xml, aft-results*.tar.bz2' } - step([$class: 'XUnitPublisher', - testTimeMargin: '3000', - thresholdMode: 1, - thresholds: [ - [$class: 'FailedThreshold', - failureNewThreshold: '0', - failureThreshold: '0', - unstableNewThreshold: '99999', - unstableThreshold: '99999'], - [$class: 'SkippedThreshold', - failureNewThreshold: '99999', - failureThreshold: '99999', - unstableNewThreshold: '99999', - unstableThreshold: '99999']], - tools: [[$class: 'JUnitType', - deleteOutputFiles: true, - failIfNotNew: true, - pattern: 'TEST-*.xml', - skipNoTestFiles: false, - stopProcessingIfError: true]]]) + step_xunit() } // node } // test_runs = } // for m @@ -299,3 +280,27 @@ def set_gh_status_pending(is_pr, _msg) { setGitHubPullRequestStatus state: 'PENDING', context: "${env.JOB_NAME}", message: "${_msg}" } } + +def step_xunit() { + step([$class: 'XUnitPublisher', + testTimeMargin: '3000', + thresholdMode: 1, + thresholds: [ + [$class: 'FailedThreshold', + failureNewThreshold: '0', + failureThreshold: '0', + unstableNewThreshold: '99999', + unstableThreshold: '99999'], + [$class: 'SkippedThreshold', + failureNewThreshold: '99999', + failureThreshold: '99999', + unstableNewThreshold: '99999', + unstableThreshold: '99999']], + tools: [ + [$class: 'JUnitType', + deleteOutputFiles: true, + failIfNotNew: true, + pattern: 'TEST-*.xml', + skipNoTestFiles: false, + stopProcessingIfError: true]]]) +} From 2996143e49f5317650efd07f615ca03f9a4ed194 Mon Sep 17 00:00:00 2001 From: Olev Kartau Date: Thu, 6 Apr 2017 10:48:33 +0300 Subject: [PATCH 4/8] CI Jenkinsfile: shift block left, eliminate extra unneeded indent No functional change Signed-off-by: Olev Kartau --- Jenkinsfile | 72 ++++++++++++++++++++++++++--------------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index edf06e703f..0d3bd4c4eb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -122,43 +122,43 @@ try { def test_runs = [:] for(int i = 0; i < test_devices.size(); i++) { def test_device = test_devices[i] - // only if built for machine that this tester wants - if ( target_machine == mapping["${test_device}"] ) { - // testinfo_data may contain multiple lines stating different images - String[] separated_testinfo = testinfo_data["${target_machine}"].split("\n") - for (int m = 0; m < separated_testinfo.length; m++) { - def one_image_testinfo = separated_testinfo[m] - echo "Image #${m} to be tested on test_${test_device} info: ${separated_testinfo[m]}" - test_runs["test_${m}_${test_device}"] = { - node('refkit-tester') { - deleteDir() // clean workspace - echo "Testing test_${test_device} with image_info: ${one_image_testinfo}" - writeFile file: 'tester-exec.sh', text: tester_script - // append newline so that tester-exec.sh can parse it using "read" - one_image_testinfo += "\n" - // create testinfo.csv on this tester describing one image - writeFile file: "testinfo.csv", text: one_image_testinfo - def img = one_image_testinfo.split(",")[0] - try { - withEnv(["CI_BUILD_ID=${ci_build_id}", - "MACHINE=${mapping["${test_device}"]}", - "TEST_DEVICE=${test_device}" ]) { - sh 'chmod a+x tester-exec.sh && ./tester-exec.sh' - } - } catch (Exception e) { - throw e - } finally { - // read tests summary prepared by tester-exec.sh - // Here one tester adds it's summary piece to the global buffer. - global_sum_log += readFile "results-summary-${test_device}.${img}.log" - archiveArtifacts allowEmptyArchive: true, - artifacts: '*.log, *.xml, aft-results*.tar.bz2' + // only if built for machine that this tester wants + if ( target_machine == mapping["${test_device}"] ) { + // testinfo_data may contain multiple lines stating different images + String[] separated_testinfo = testinfo_data["${target_machine}"].split("\n") + for (int m = 0; m < separated_testinfo.length; m++) { + def one_image_testinfo = separated_testinfo[m] + echo "Image #${m} to be tested on test_${test_device} info: ${separated_testinfo[m]}" + test_runs["test_${m}_${test_device}"] = { + node('refkit-tester') { + deleteDir() // clean workspace + echo "Testing test_${test_device} with image_info: ${one_image_testinfo}" + writeFile file: 'tester-exec.sh', text: tester_script + // append newline so that tester-exec.sh can parse it using "read" + one_image_testinfo += "\n" + // create testinfo.csv on this tester describing one image + writeFile file: "testinfo.csv", text: one_image_testinfo + def img = one_image_testinfo.split(",")[0] + try { + withEnv(["CI_BUILD_ID=${ci_build_id}", + "MACHINE=${mapping["${test_device}"]}", + "TEST_DEVICE=${test_device}" ]) { + sh 'chmod a+x tester-exec.sh && ./tester-exec.sh' } - step_xunit() - } // node - } // test_runs = - } // for m - } // if target_machine == mapping + } catch (Exception e) { + throw e + } finally { + // read tests summary prepared by tester-exec.sh + // Here one tester adds it's summary piece to the global buffer. + global_sum_log += readFile "results-summary-${test_device}.${img}.log" + archiveArtifacts allowEmptyArchive: true, + artifacts: '*.log, *.xml, aft-results*.tar.bz2' + } + step_xunit() + } // node + } // test_runs = + } // for + } // if target_machine == mapping } // for i stage('Parallel test run') { set_gh_status_pending(is_pr, 'Testing') From f2232032b2e7a7ebbdfa62852bb03bb55e0f0b2e Mon Sep 17 00:00:00 2001 From: Olev Kartau Date: Thu, 6 Apr 2017 13:47:23 +0300 Subject: [PATCH 5/8] CI Jenkinsfile: add lock section around step_xunit It was discovered we lost one or more tester result file sets from xUnit processing every now and then. Log files analysis shows that when multiple testers attempted xUnit step in nearly same time, it caused loss of data. We can avoid same-time run by locking this section. Signed-off-by: Olev Kartau --- Jenkinsfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0d3bd4c4eb..8430ef052d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -154,7 +154,11 @@ try { archiveArtifacts allowEmptyArchive: true, artifacts: '*.log, *.xml, aft-results*.tar.bz2' } - step_xunit() + // without locking we may lose tester result set(s) + // if testers run xunit step in nearly same time + lock(resource: "step-xunit") { + step_xunit() + } } // node } // test_runs = } // for From 87b7147319e99faebe7e872dc5485b1f8d02a949 Mon Sep 17 00:00:00 2001 From: Olev Kartau Date: Fri, 7 Apr 2017 13:39:22 +0300 Subject: [PATCH 6/8] CI tester-exec: remove symlinks after testing Few symlinks exist after testing which point out of local tree, created by tested device. These become broken when exported to CI master, and will show as garbage. Signed-off-by: Olev Kartau --- docker/tester-exec.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/tester-exec.sh b/docker/tester-exec.sh index 0bd3d5b5d4..1881fcfcf1 100755 --- a/docker/tester-exec.sh +++ b/docker/tester-exec.sh @@ -76,6 +76,8 @@ testimg() { daft ${DEVICE} ${FILENAME} --record AFT_EXIT_CODE=$? + # delete symlinks, these point outside of local set and are useless + find . -type l -print -delete # modify names inside TEST-*.xml files to contain device and img_name # as these get shown in same xUnit results table in Jenkins sed -e "s/name=\"oeqa/name=\"${DEVICE}.${_IMG_NAME}.oeqa/g" -i TEST-*.xml From 2a3c71add939a5bcf6f71c444e17f91ac9e8dc87 Mon Sep 17 00:00:00 2001 From: Olev Kartau Date: Mon, 10 Apr 2017 09:16:49 +0300 Subject: [PATCH 7/8] CI testing: don't combine results into tarball This tarball has no use. Signed-off-by: Olev Kartau --- Jenkinsfile | 2 +- docker/tester-exec.sh | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8430ef052d..39ae48d792 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -152,7 +152,7 @@ try { // Here one tester adds it's summary piece to the global buffer. global_sum_log += readFile "results-summary-${test_device}.${img}.log" archiveArtifacts allowEmptyArchive: true, - artifacts: '*.log, *.xml, aft-results*.tar.bz2' + artifacts: '*.log, *.xml' } // without locking we may lose tester result set(s) // if testers run xunit step in nearly same time diff --git a/docker/tester-exec.sh b/docker/tester-exec.sh index 1881fcfcf1..2ede81bbbe 100755 --- a/docker/tester-exec.sh +++ b/docker/tester-exec.sh @@ -109,8 +109,6 @@ testimg() { echo " Run rate:${run_rate}% Pass rate of total:${pass_rate_of_total}% Pass rate of exec:${pass_rate_of_exec}%" >> $sumfile fi echo "-------------------------------------------------------------------" >> $sumfile - # combine artifacts into single file for easier download - tar c --ignore-failed-read results* *.xml *.log | bzip2 -c9 > aft-results_${DEVICE}_${_IMG_NAME}_${TEST_SUITE_FILE}.tar.bz2 set -e return ${AFT_EXIT_CODE} From 15346e8f64a3aaa1814809e02d89f818d7313aed Mon Sep 17 00:00:00 2001 From: Olev Kartau Date: Mon, 10 Apr 2017 10:23:11 +0300 Subject: [PATCH 8/8] CI tester-exec.sh: reduce use of variables If a variable is used only once, or just decodes part of name by adding it as part of variable name, it is not really helpful. Reduce amount of lines by not defining such variables. Signed-off-by: Olev Kartau --- docker/tester-exec.sh | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/docker/tester-exec.sh b/docker/tester-exec.sh index 2ede81bbbe..79ebcb9caa 100755 --- a/docker/tester-exec.sh +++ b/docker/tester-exec.sh @@ -21,7 +21,6 @@ testimg() { _IMG_NAME=$1 TEST_SUITE_FILE=$2 TEST_CASES_FILE=$3 - _IMG_NAME_MACHINE=${_IMG_NAME}-${MACHINE} # Get test suite wget ${_WGET_OPTS} ${TEST_SUITE_FOLDER_URL}/${_IMG_NAME}/${TEST_SUITE_FILE} @@ -34,21 +33,16 @@ testimg() { cp $HOME/.config.ini.wlan ${_WLANCONF} chmod 644 ${_WLANCONF} - FN_BASE=${_IMG_NAME_MACHINE}-${CI_BUILD_ID} - FILENAME=${FN_BASE}.wic - FILENAME_BMAP=${FILENAME}.bmap - FILENAME_XZ=${FILENAME}.xz - FILENAME_ZIP=${FILENAME}.zip - + FILENAME=${_IMG_NAME}-${MACHINE}-${CI_BUILD_ID}.wic set +e - wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME_BMAP} - wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME_XZ} -O - | unxz - > ${FILENAME} + wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME}.bmap + wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME}.xz -O - | unxz - > ${FILENAME} if [ ! -s ${FILENAME} ]; then - wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME_ZIP} - if [ -s ${FILENAME_ZIP} ]; then - unzip ${FILENAME_ZIP} + wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME}.zip + if [ -s ${FILENAME}.zip ]; then + unzip ${FILENAME}.zip else - echo "ERROR: No file ${FILENAME_XZ} or ${FILENAME_ZIP} found, can not continue." + echo "ERROR: No file ${FILENAME}.xz or ${FILENAME}.zip found, can not continue." exit 1 fi fi