diff --git a/Jenkinsfile b/Jenkinsfile index 434f3be33..bb6e55776 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -85,6 +85,7 @@ Void distro_version_test(String branch, String distro, String expected) { "instead of string starting with '${expected}'") } } + return } /* groovylint-disable-next-line CompileStatic */ @@ -594,7 +595,7 @@ pipeline { 'full_regression,foobar,@stages.tag@'], [tags: [[tag: 'Test-tag', value: 'datamover foobar']], tag_template: 'datamover,@stages.tag@ foobar,@stages.tag@'], - /* this one doesn't quite work due to the @commits.value@ substituion + /* this one doesn't quite work due to the @commits.value@ substitution not accounting for the skip-list [tags: [[tag: 'Test-tag', value: 'datamover'], [tag: 'Features', value: 'foobar'], diff --git a/vars/Date.groovy b/vars/Date.groovy new file mode 100644 index 000000000..999a03e3c --- /dev/null +++ b/vars/Date.groovy @@ -0,0 +1,39 @@ +// vars/Date.groovy + +/** + * Date.groovy + * + * Jenkins sandbox-compatible replacement for java.util.Date + * Uses System.currentTimeMillis() which is allowed in the sandbox + */ + +/** + * Create a new Date object using current time in milliseconds + * Returns the current time in milliseconds since epoch + * + * @return long Current time in milliseconds + */ +long call() { + return System.currentTimeMillis() +} + +/** + * Create a Date object from a specific time in milliseconds + * + * @param timeInMillis Time in milliseconds since epoch + * @return long Time in milliseconds + */ +long call(long timeInMillis) { + return timeInMillis +} + +/** + * Get time in milliseconds from a Date value + * For compatibility with code expecting .getTime() + * + * @param dateValue The date value (already in milliseconds) + * @return long Time in milliseconds + */ +long getTime(long dateValue) { + return dateValue +} diff --git a/vars/buildRpm.groovy b/vars/buildRpm.groovy index fbf8e0d2b..b2710f5ce 100755 --- a/vars/buildRpm.groovy +++ b/vars/buildRpm.groovy @@ -44,7 +44,7 @@ */ Map call(Map config = [:]) { - Date startDate = new Date() + long startDate = System.currentTimeMillis() String context = config.get('context', 'build/' + env.STAGE_NAME) String description = config.get('description', env.STAGE_NAME) String build_script = config.get('build_script', 'ci/rpm/build.sh') diff --git a/vars/cloverReportPublish.groovy b/vars/cloverReportPublish.groovy index b3a23e323..9170d0d8c 100755 --- a/vars/cloverReportPublish.groovy +++ b/vars/cloverReportPublish.groovy @@ -5,7 +5,7 @@ /** * clover Report Publish step method * - * Consolidiate clover data for publising. + * Consolidate clover data for publishing. * * @param config Map of parameters passed * @@ -31,11 +31,11 @@ Map call(Map config = [:]) { // If we don't have a BULLSEYE environment variable set - // there are no Bullsye reports to process. + // there are no Bullseye reports to process. if (!env.BULLSEYE) { return ['result': 'SUCCESS', 'cloverreportpublish_time': 0] } - Date startDate = new Date() + long startDate = System.currentTimeMillis() Map stage_info = parseStageInfo(config) String coverage_website = config.get('coverage_website', @@ -61,7 +61,11 @@ Map call(Map config = [:]) { target_stash += '-' + stage_info['build_type'] } - unstash config.get('stash', "${target_stash}-build-vars") + try { + unstash config.get('stash', "${target_stash}-build-vars") + } catch (hudson.AbortException ex) { + println("Unstash failed: ${ex}") + } int stash_cnt = 0 stashes.each { name -> diff --git a/vars/daosPackagesVersion.groovy b/vars/daosPackagesVersion.groovy index 3232ec25d..a94304e15 100644 --- a/vars/daosPackagesVersion.groovy +++ b/vars/daosPackagesVersion.groovy @@ -96,9 +96,8 @@ String call(String distro, String next_version) { String version_file = normalized_distro try { unstash normalized_distro + '-rpm-version' - /* groovylint-disable-next-line CatchException */ - } catch (Exception e1) { - // backward compatibilty + } catch (hudson.AbortException e1) { + // backward compatibility try { // ugly backwards compatibility hack due to hardware distro // being el8 now @@ -107,9 +106,8 @@ String call(String distro, String next_version) { } unstash _distro + '-rpm-version' version_file = _distro - /* groovylint-disable-next-line CatchException */ - } catch (Exception e2) { - print('Ingoring missing but deprecated ' + _distro + '-rpm-version' + ' stash') + } catch (hudson.AbortException e2) { + print('Ignoring missing but deprecated ' + _distro + '-rpm-version' + ' stash') return '' } } diff --git a/vars/durationSeconds.groovy b/vars/durationSeconds.groovy index f3e3ae402..6134fe70b 100755 --- a/vars/durationSeconds.groovy +++ b/vars/durationSeconds.groovy @@ -1,17 +1,36 @@ +/* groovylint-disable DuplicateNumberLiteral, UnnecessaryGetter */ // vars/durationSeconds.groovy /** * durationSeconds step method * + * @param startTime, long value with start time in milliseconds. + * @param endTime, long value with end time in milliseconds, default current time. + * returns: Duration time in seconds. + */ + +int call(long startTime, long endTime=0) { + long actualEndTime + if (endTime == 0) { + actualEndTime = System.currentTimeMillis() + } else { + actualEndTime = endTime + } + int delta = (actualEndTime - startTime) / 1000 + return delta +} + + /** + * durationSeconds step method (Date overload) + * * @param startDate, Date object with start time. - * @param endDate, Date object with end time, default current date time. + * @param endDate, Date object with end time, default current time. * returns: Duration time in seconds. */ int call(Date startDate, Date endDate=null) { - if (endDate == null) { - endDate = new Date() - } - int delta = (endDate.getTime() - startDate.getTime())/1000 + long startTime = startDate.getTime() + long endTime = endDate ? endDate.getTime() : System.currentTimeMillis() + int delta = (endTime - startTime) / 1000 return delta } diff --git a/vars/functionalTest.groovy b/vars/functionalTest.groovy index aa7b949d3..2d9520200 100755 --- a/vars/functionalTest.groovy +++ b/vars/functionalTest.groovy @@ -26,7 +26,7 @@ * config['description'] Description to report for SCM status. * Default env.STAGE_NAME. * - * config['failure_artifacts'] Failure aritfifacts to return. + * config['failure_artifacts'] Failure artifacts to return. * Default env.STAGE_NAME. * * config['ignore_failure'] Ignore test failures. Default false. @@ -41,10 +41,10 @@ * * config['node_count'] Count of nodes that will actually be used * the test. Default will be based on the - * enviroment variables for the stage. + * environment variables for the stage. * * config['stashes'] List of stashes to use. Default will be - * baed on the environment variables for the + * based on the environment variables for the * stage. * * config['target'] Target distribution, such as 'centos7', @@ -62,7 +62,7 @@ */ Map call(Map config = [:]) { - Date startDate = new Date() + long startDate = System.currentTimeMillis() String nodelist = config.get('NODELIST', env.NODELIST) String context = config.get('context', 'test/' + env.STAGE_NAME) String description = config.get('description', env.STAGE_NAME) @@ -114,40 +114,6 @@ Map call(Map config = [:]) { run_test_config['context'] = context run_test_config['description'] = description - String script = 'if ! pip3 install' - script += ''' --upgrade --upgrade-strategy only-if-needed launchable; then - set +e - echo "Failed to install launchable" - id - pip3 list --user || true - find ~/.local/lib -type d - hostname - pip3 --version - pip3 index versions launchable - pip3 install --user launchable== - exit 1 - fi - pip3 list --user || true - ''' - sh label: 'Install Launchable', - script: script - - try { - withCredentials([string(credentialsId: 'launchable-test', variable: 'LAUNCHABLE_TOKEN')]) { - sh label: 'Send build data', - /* groovylint-disable-next-line GStringExpressionWithinString */ - script: '''export PATH=$PATH:$HOME/.local/bin - launchable record build --name ${BUILD_TAG//%2F/-} --source src=. - launchable subset --time 60m --build ${BUILD_TAG//%2F/-} ''' + - '--get-tests-from-previous-sessions --rest=rest.txt raw > subset.txt' - } - /* groovylint-disable-next-line CatchException */ - } catch (Exception error) { - println( - "Ignoring failure to record " + env.STAGE_NAME + " tests with launchable: " + - error.getMessage()) - } - Map runtestData = [:] if (config.get('test_function', 'runTestFunctional') == 'runTestFunctionalV2') { diff --git a/vars/functionalTestPostV2.groovy b/vars/functionalTestPostV2.groovy index 1b395517e..090f3d409 100755 --- a/vars/functionalTestPostV2.groovy +++ b/vars/functionalTestPostV2.groovy @@ -44,8 +44,13 @@ void call(Map config = [:]) { // Need to unstash the script result from runTest String results_map = 'results_map_' + sanitizedStageName() - unstash name: results_map - Map results = readYaml file: results_map + Map results = [:] + try { + unstash name: results_map + results = readYaml file: results_map + } catch (hudson.AbortException e) { + println("Failed to unstash ${results_map}: ${e.message}") + } String prev_result = currentBuild.result junit(testResults: junit_results) @@ -88,39 +93,7 @@ void call(Map config = [:]) { config.get('artifacts', env.STAGE_NAME + '/**') archiveArtifacts(artifacts: artifacts) - // Analyze test failures - String jobName = env.JOB_NAME.replace('/', '_') - jobName += '_' + env.BUILD_NUMBER - String fileName = env.DAOS_STACK_JOB_STATUS_DIR + '/' + jobName - - if (fileExists('ci/functional/launchable_analysis')) { - sh(label: 'Analyze failed tests vs. Launchable subset', - script: 'ci/functional/launchable_analysis "' + fileName + '"') - } - - String script = 'pip3 install' - script += ' --user --upgrade launchable~=1.0' - - sh(label: 'Install Launchable', - script: script) - - try { - withCredentials([string(credentialsId: 'launchable-test', variable: 'LAUNCHABLE_TOKEN')]) { - sh(label: 'Submit test results to Launchable', - /* groovylint-disable-next-line GStringExpressionWithinString */ - script: 'if ls -l "' + env.STAGE_NAME + '''"/*/*/xunit1_results.xml 2>/dev/null; then - export PATH=$PATH:$HOME/.local/bin - launchable record tests --build ${BUILD_TAG//%2F/-} pytest ''' + - '"' + env.STAGE_NAME + '''"/*/*/xunit1_results.xml - fi''') - } - /* groovylint-disable-next-line CatchException */ - } catch (Exception error) { - println( - "Ignoring failure to record " + env.STAGE_NAME + " tests with launchable: " + - error.getMessage()) - } if (!ignore_failure && results['result'] == 'FAILURE') { - unstable "Failure detected with test harness or hardware." + unstable 'Failure detected with test harness or hardware.' } } diff --git a/vars/hwDistroTarget.groovy b/vars/hwDistroTarget.groovy index 9317015dd..60ab32384 100644 --- a/vars/hwDistroTarget.groovy +++ b/vars/hwDistroTarget.groovy @@ -1,3 +1,4 @@ +/* groovylint-disable DuplicateNumberLiteral */ // vars/hwDistroTarget.groovy /** @@ -11,11 +12,11 @@ */ String call(String size) { - (name, version) = hwDistroTarget2(size) - return name + version + List result = hwDistroTarget2(size) + return result[0] + result[1] } String call() { - (name, version) = hwDistroTarget2() - return name + version + List result = hwDistroTarget2() + return result[0] + result[1] } diff --git a/vars/hwDistroTarget2.groovy b/vars/hwDistroTarget2.groovy index a7908e678..8990c424c 100644 --- a/vars/hwDistroTarget2.groovy +++ b/vars/hwDistroTarget2.groovy @@ -11,6 +11,10 @@ * Method to return the distro target and version (as a list) for a given stage */ +String targetRegex() { + return '([a-z]+)(.*)' +} + // I'd love to use a more explicit // String, String hw_distro(String size) here but it chokes Jenkins (at // least) @@ -28,7 +32,7 @@ List call(String size) { } distro = cachedCommitPragma('Func-hw-test-' + size + '-distro', cachedCommitPragma('Func-hw-test-distro', distro)) - return (distro =~ /([a-z]+)(.*)/)[0][1..2] + return (distro =~ targetRegex())[0][1..2] } List call() { @@ -36,5 +40,5 @@ List call() { return hwDistroTarget2(env.STAGE_NAME[env.STAGE_NAME.lastIndexOf(' ') + 1..-1].toLowerCase()) } - return (parseStageInfo()['target'] =~ /([a-z]+)(.*)/)[0][1..2] + return (parseStageInfo()['target'] =~ targetRegex())[0][1..2] } diff --git a/vars/junitSimpleReport.groovy b/vars/junitSimpleReport.groovy index ddbdbaf18..8a6416414 100755 --- a/vars/junitSimpleReport.groovy +++ b/vars/junitSimpleReport.groovy @@ -25,7 +25,7 @@ void call(Map config = [:]) { int zero = 0 int one = 1 - simple = 'simple' + String simple = 'simple' String jclass = config.get('class', simple) String jname = config.get('name', simple) String jsuite = config.get('suite', sanitizedStageName()) diff --git a/vars/notifyBrokenBranch.groovy b/vars/notifyBrokenBranch.groovy index 7a55dfc4a..72bc39fcd 100644 --- a/vars/notifyBrokenBranch.groovy +++ b/vars/notifyBrokenBranch.groovy @@ -1,3 +1,4 @@ +/* groovylint-disable DuplicateStringLiteral */ // vars/notifyBrokenBranch.groovy /* groovylint-disable VariableName */ @@ -8,16 +9,14 @@ * * config['branches'] List of branches to notify for. Default "master" * config['onPR'] Send e-mail when called from a PR. Default false - * */ -def call(Map config = [:]) { - +void call(Map config = [:]) { String branches if (config['branches']) { branches = config['branches'].split() } else { - branches = ["master"] + branches = ['master'] } // Needed this as a work around that env['GIT_BRANCH'] is blacklisted @@ -40,7 +39,7 @@ def call(Map config = [:]) { subject: 'Build broken on ' + git_branch, onPR: config['onPR'] - String branch = git_branch.toUpperCase().replaceAll("-", "_") + String branch = git_branch.toUpperCase().replaceAll('-', '_') // This will need to be implemented in trusted-pipe-line lib eventually // as checking if environment variables exist is blacklisted in the // groovy sandbox. @@ -48,7 +47,7 @@ def call(Map config = [:]) { // def watchers = env["DAOS_STACK_${branch}_WATCHER"] if (branch == 'MASTER') { String watchers = env.DAOS_STACK_MASTER_WATCHER - if (watchers != "null") { + if (watchers != 'null') { emailextDaos body: git_branch + ' is broken.\n\n' + 'See ' + env.BUILD_URL + ' for more details.', to: watchers, diff --git a/vars/provisionNodes.groovy b/vars/provisionNodes.groovy index eb213d483..5ec50533c 100644 --- a/vars/provisionNodes.groovy +++ b/vars/provisionNodes.groovy @@ -48,8 +48,7 @@ */ /* groovylint-disable-next-line MethodSize */ Map call(Map config = [:]) { - /* groovylint-disable-next-line NoJavaUtilDate */ - Date startDate = new Date() + long startDate = System.currentTimeMillis() String nodeString = config['NODELIST'] List node_list = config['NODELIST'].split(',') int node_max_cnt = node_list.size() @@ -126,7 +125,6 @@ Map call(Map config = [:]) { if (!fileExists('ci/provisioning/log_cleanup.sh') || !fileExists('ci/provisioning/post_provision_config.sh')) { - return provisionNodesV1(config) } diff --git a/vars/runScriptWithStashes.groovy b/vars/runScriptWithStashes.groovy index 097b0275c..f2f7b41a0 100644 --- a/vars/runScriptWithStashes.groovy +++ b/vars/runScriptWithStashes.groovy @@ -1,3 +1,4 @@ +/* groovylint-disable DuplicateNumberLiteral */ // vars/runScriptWithStashes.groovy /** @@ -10,67 +11,66 @@ * script shell script to run * label label to use when running the script */ - Map call(Map kwargs = [:]) { - Date startDate = new Date() +Map call(Map kwargs = [:]) { + long startDate = System.currentTimeMillis() List stashes = kwargs.get('stashes', []) String script = kwargs.get('script', '') String label = kwargs.get('label', "Run ${script} with stashes") - Integer stash_count = 0 + Integer stashCount = 0 stashes.each { stash -> try { unstash stash - println("Successfully unstashed ${stash}.") - stash_count++ - /* groovylint-disable-next-line CatchException */ - } catch (Exception ex) { + println("Successfully un-stashed ${stash}.") + stashCount++ + } catch (hudson.AbortException ex) { println("Ignoring failure to unstash ${stash}. Perhaps the stage was skipped?") } } - Integer return_code = 255 + Integer returnCode = 255 if (!script) { // Nothing to do if no stash exist println('No script provided, skipping execution') - return_code = 0 - } else if (stash_count < 1) { + returnCode = 0 + } else if (stashCount < 1) { // Nothing to do if no stash exist println('No code coverage stashes found, skipping merged code coverage report') - return_code = 0 + returnCode = 0 } else { // Run the script try { sh(script: script, label: label) - return_code = 0 + returnCode = 0 } catch (hudson.AbortException e) { // groovylint-disable UnnecessaryGetter // groovylint-disable-next-line NoDef, VariableTypeRequired - def rc_val = (e.getMessage() =~ /\d+$/) - if (rc_val) { - return_code = rc_val[0] as Integer + def returnCodeValue = (e.getMessage() =~ /\d+$/) + if (returnCodeValue) { + returnCode = returnCodeValue[0] as Integer } } } // Generate a result for the stage - Date endDate = new Date() + long endDate = System.currentTimeMillis() Integer runTime = durationSeconds(startDate, endDate) String status = 'SUCCESS' - if (return_code != 0) { + if (returnCode != 0) { status = 'FAILURE' } - Map results = ['result_code': return_code, + Map results = ['result_code': returnCode, 'result': status, 'start_date': startDate, 'end_date': endDate, 'runtest_time': runTime] - String results_map = 'results_map_' + sanitizedStageName() - writeYaml file: results_map, + String resultsMap = 'resultsMap_' + sanitizedStageName() + writeYaml file: resultsMap, data: results, overwrite: true - stash name: results_map, - includes: results_map + stash name: resultsMap, + includes: resultsMap return results } diff --git a/vars/runTest.groovy b/vars/runTest.groovy index 91b590b65..71dcfad05 100644 --- a/vars/runTest.groovy +++ b/vars/runTest.groovy @@ -14,7 +14,7 @@ Map call(Map config = [:]) { * config['stashes'] Stashes from the build to unstash * config['failure_artifacts'] Artifacts to link to when test fails, if any * config['ignore_failure'] Whether a FAILURE result should post a failed step - * config['notify_result'] Flag to notify SCM for the resultstatus, + * config['notify_result'] Flag to notify SCM for the result status, * default true, Use false if the notification * will be in post processing. * @@ -48,9 +48,7 @@ Map call(Map config = [:]) { // github expectations at the same time to also include any Matrix // environment variables. - // Must use Date() in pipeline-lib - // groovylint-disable-next-line NoJavaUtilDate - Date startDate = new Date() + long startDate = System.currentTimeMillis() String context = config.get('context', 'test/' + env.STAGE_NAME) String description = config.get('description', env.STAGE_NAME) String flow_name = config.get('flow_name', env.STAGE_NAME) @@ -137,7 +135,7 @@ Map call(Map config = [:]) { } } - Date endDate = new Date() + long endDate = System.currentTimeMillis() int runTime = durationSeconds(startDate, endDate) // We need to pass the rc to the post step. diff --git a/vars/runTestFunctional.groovy b/vars/runTestFunctional.groovy index a3ff7e846..6eae4cd71 100644 --- a/vars/runTestFunctional.groovy +++ b/vars/runTestFunctional.groovy @@ -4,7 +4,6 @@ * runTestFunctional.groovy * * runTestFunctional pipeline step - * */ Map call(Map config = [:]) { diff --git a/vars/runTestFunctionalV1.groovy b/vars/runTestFunctionalV1.groovy index bc6158d48..a8b1c0872 100644 --- a/vars/runTestFunctionalV1.groovy +++ b/vars/runTestFunctionalV1.groovy @@ -1,12 +1,13 @@ +/* groovylint-disable DuplicateStringLiteral, GStringExpressionWithinString, LineLength, VariableName */ // vars/runTestFunctionalV1.groovy /** * runTestFunctionalV1.groovy * * runTestFunctionalV1 pipeline step - * */ +/* groovylint-disable-next-line MethodSize */ Map call(Map config = [:]) { /** * runTestFunctionalV1 step method @@ -39,7 +40,7 @@ Map call(Map config = [:]) { */ String test_rpms = 'false' - if (config['test_rpms'] == "true") { + if (config['test_rpms'] == 'true') { test_rpms = 'true' } String functional_test_script = '''test_tag=$(git show -s --format=%%B | sed -ne "/^Test-tag%s:/s/^.*: *//p") @@ -107,10 +108,10 @@ Map call(Map config = [:]) { config['test_tag'], config['node_count'], config['ftest_arg']) - config['junit_files'] = "install/lib/daos/TESTING/ftest/avocado/job-results/job-*/*.xml install/lib/daos/TESTING/ftest/*_results.xml" + config['junit_files'] = 'install/lib/daos/TESTING/ftest/avocado/job-results/job-*/*.xml install/lib/daos/TESTING/ftest/*_results.xml' config['failure_artifacts'] = 'Functional' - if (test_rpms == 'true' && config['stashes']){ + if (test_rpms == 'true' && config['stashes']) { // we don't need (and might not even have) stashes if testing // from RPMs config.remove('stashes') @@ -123,5 +124,4 @@ Map call(Map config = [:]) { config.remove('test_rpms') return runTest(config) - } diff --git a/vars/runTestFunctionalV2.groovy b/vars/runTestFunctionalV2.groovy index 3acfb43bf..985f21b46 100644 --- a/vars/runTestFunctionalV2.groovy +++ b/vars/runTestFunctionalV2.groovy @@ -66,7 +66,7 @@ Map call(Map config = [:]) { config['failure_artifacts'] = env.STAGE_NAME } - if (test_rpms && config['stashes']){ + if (test_rpms && config['stashes']) { // we don't need (and might not even have) stashes if testing // from RPMs config.remove('stashes') @@ -88,14 +88,14 @@ Map call(Map config = [:]) { // Restore the ignore failure setting config['ignore_failure'] = ignore_failure - String covfile = 'test.cov' + String coverageFile = 'test.cov' if (!fileExists('test.cov')) { - covfile += '_not_done' - fileOperations([fileCreateOperation(fileName: covfile, + coverageFile += '_not_done' + fileOperations([fileCreateOperation(fileName: coverageFile, fileContent: '')]) } String name = 'func' + stage_info['pragma_suffix'] + '-cov' stash name: config.get('coverage_stash', name), - includes: covfile + includes: coverageFile return runData } diff --git a/vars/scmNotify.groovy b/vars/scmNotify.groovy index cbb948aa6..19c3fe604 100755 --- a/vars/scmNotify.groovy +++ b/vars/scmNotify.groovy @@ -1,3 +1,4 @@ +/* groovylint-disable CouldBeElvis, DuplicateStringLiteral */ // vars/scmNotify.groovy /* This provides a way of notifying the SCM such as GitLab/GitHub with @@ -16,22 +17,20 @@ * * See the githubNotify pipeline step for the parameters to pass. */ - -def call(Map config = [:]) { - - def errtxt = 'Jenkins not configured to notify SCM repository of builds.' - if (env.DAOS_JENKINS_NOTIFY_STATUS == null) { - println errtxt - return - } - try { - if (! config['credentialsId']) { - config['credentialsId'] = scmStatusIdSystem() +void call(Map config = [:]) { + String errorText = 'Jenkins not configured to notify SCM repository of builds.' + if (env.DAOS_JENKINS_NOTIFY_STATUS == null) { + println errorText + return + } + try { + if (!config['credentialsId']) { + config['credentialsId'] = scmStatusIdSystem() + } + } catch (java.lang.NoSuchMethodError e) { + // Did not find a shared scmStatusIdSystem routine. + // Assume DAOS_JENKINS_NOTIFY_STATUS contains a credential id. + config['credentialsId'] = env.DAOS_JENKINS_NOTIFY_STATUS } - } catch (java.lang.NoSuchMethodError e) { - // Did not find a shared scmStatusIdSystem routine. - // Assume DAOS_JENKINS_NOTIFY_STATUS contains a credential id. - config['credentialsId'] = env.DAOS_JENKINS_NOTIFY_STATUS - } - scmNotifyTrusted(config) + scmNotifyTrusted(config) } diff --git a/vars/sconsBuild.groovy b/vars/sconsBuild.groovy index 83f47c02a..6e1d48bd4 100644 --- a/vars/sconsBuild.groovy +++ b/vars/sconsBuild.groovy @@ -1,19 +1,20 @@ -/* groovylint-disable VariableName */ +/* groovylint-disable DuplicateNumberLiteral, DuplicateStringLiteral, VariableName */ +/* groovylint-disable GStringExpressionWithinString */ // vars/sconsBuild.groovy /** * sconsBuild.groovy * * sconsBuild pipeline step - * */ String num_proc() { - return sh(label: "Get number of processors online", - script: "/usr/bin/getconf _NPROCESSORS_ONLN", + return sh(label: 'Get number of processors online', + script: '/usr/bin/getconf _NPROCESSORS_ONLN', returnStdout: true) } +/* groovylint-disable-next-line MethodSize */ Map call(Map config = [:]) { /** * sconsBuild step method @@ -72,17 +73,17 @@ Map call(Map config = [:]) { * config['code_coverage'] Boolean, build with code coverage. Default false. */ - Date startDate = new Date() + long startDate = System.currentTimeMillis() String config_target if (config['target']) { - config_target = config['target'] - config.remove('target') + config_target = config['target'] + config.remove('target') } Map stage_info = parseStageInfo(config) String tee_file = '| tee $WORKSPACE/' + stage_info['log_to_file'] - String failure_artifacts='' + String failure_artifacts = '' if (config['failure_artifacts']) { failure_artifacts = config['failure_artifacts'] } else { @@ -97,10 +98,10 @@ Map call(Map config = [:]) { * the potential tampering before the scm operation. */ if (config['scons_local_replace'] && config_target) { - sh "rm -rf \"\${WORKSPACE}/${config_target}/scons_local\"" + sh "rm -rf \"\${WORKSPACE}/${config_target}/scons_local\"" } - def scm_config = [withSubmodules: true] + Map scm_config = [withSubmodules: true] if (config['scm']) { scm_config = config['scm'] if (config_target && !scm_config['checkoutDir']) { @@ -110,29 +111,29 @@ Map call(Map config = [:]) { checkoutScm(scm_config) if (env.DAOS_JENKINS_NOTIFY_STATUS != null) { scmNotify description: env.STAGE_NAME, - context: "build" + "/" + env.STAGE_NAME, - status: "PENDING" + context: 'build/' + env.STAGE_NAME, + status: 'PENDING' } if (config['scons_local_replace'] && config_target) { - // replace the scons_local directory in config_target - // with a link to the scons_local directory. - sh """rm -rf "\${WORKSPACE}/${config_target}/scons_local" - ln -s "${WORKSPACE}/scons_local" \ - "\${WORKSPACE}/${config_target}/scons_local" - """ + // replace the scons_local directory in config_target + // with a link to the scons_local directory. + sh """rm -rf "\${WORKSPACE}/${config_target}/scons_local" + ln -s "${WORKSPACE}/scons_local" \ + "\${WORKSPACE}/${config_target}/scons_local" + """ } if (stage_info['compiler'] == 'covc') { - httpRequest url: env.JENKINS_URL + - 'job/daos-stack/job/tools/job/master' + - '/lastSuccessfulBuild/artifact/' + - 'bullseyecoverage-linux.tar', - httpMode: 'GET', - outputFile: 'bullseye.tar' + httpRequest url: env.JENKINS_URL + + 'job/daos-stack/job/tools/job/master' + + '/lastSuccessfulBuild/artifact/' + + 'bullseyecoverage-linux.tar', + httpMode: 'GET', + outputFile: 'bullseye.tar' } - def set_cwd = '' + String set_cwd = '' if (config['directory']) { set_cwd = "cd ${config['directory']}\n" } else if (config_target) { @@ -197,11 +198,11 @@ Map call(Map config = [:]) { scons_args += ' --code-coverage' } //scons -c is not perfect so get out the big hammer - String clean_cmd = "" + String clean_cmd = '' if (config['skip_clean']) { clean_cmd += "echo 'skipping scons -c'\n" } else { - String clean_files = "_build.external" + String clean_files = '_build.external' if (config['clean']) { clean_files = config['clean'] } @@ -223,29 +224,29 @@ Map call(Map config = [:]) { String prebuild = '' if (config['prebuild']) { - prebuild = config['prebuild'] + '\n' + prebuild = config['prebuild'] + '\n' } String prefix_1 = '' if (config['TARGET_PREFIX']) { - scons_args += " TARGET_PREFIX=${config['TARGET_PREFIX']}" - if (config['target_work']) { - String target_dirs = '' - if (config['target_dirs']) { - target_dirs = config['target_dirs'] - } else if (config_target) { - target_dirs = config_target + scons_args += " TARGET_PREFIX=${config['TARGET_PREFIX']}" + if (config['target_work']) { + String target_dirs = '' + if (config['target_dirs']) { + target_dirs = config['target_dirs'] + } else if (config_target) { + target_dirs = config_target + } + String target_work = config['target_work'] + prefix_1 = + """for new_dir in ${target_dirs}; do + mkdir -p "\${WORKSPACE}/${target_work}/\${new_dir}" + rm -f "${config['TARGET_PREFIX']}/\${new_dir}" + ln -s "\${WORKSPACE}/${target_work}/\${new_dir}" \ + "${config['TARGET_PREFIX']}/\${new_dir}" + done + """ } - String target_work = config['target_work'] - prefix_1 = - """for new_dir in ${target_dirs}; do - mkdir -p "\${WORKSPACE}/${target_work}/\${new_dir}" - rm -f "${config['TARGET_PREFIX']}/\${new_dir}" - ln -s "\${WORKSPACE}/${target_work}/\${new_dir}" \ - "${config['TARGET_PREFIX']}/\${new_dir}" - done - """ - } } if (config['scons_args']) { scons_args += ' ' + config['scons_args'] @@ -259,13 +260,16 @@ Map call(Map config = [:]) { Map cov_config = [:] cov_config['project'] = config['coverity'] cov_config['tool_path'] = './cov_analysis' - if (coverityToolDownload(cov_config) < 0) - return + if (coverityToolDownload(cov_config) < 0) { + int runTime = durationSeconds(startDate) + runData['sconsbuild_time'] = runTime + return runData + } script += "PATH+=:${WORKSPACE}/cov_analysis/bin\n" - scons_exe = "cov-build --dir cov-int " + scons_exe + scons_exe = 'cov-build --dir cov-int ' + scons_exe } - // the config cache is unreliable so always force a reconfig + // the config cache is unreliable so always force a reconfiguration // with "--config=force" script += '''if ! ''' + scons_exe + ''' --config=force $SCONS_ARGS''' + tee_file + '''; then @@ -282,7 +286,7 @@ Map call(Map config = [:]) { failure_artifacts + '"' + ''' exit \$rc fi''' - String full_script = "#!/bin/bash\nset -ex\n" + + String full_script = '#!/bin/bash\nset -ex\n' + set_cwd + prebuild + prefix_1 + script int rc = 0 rc = sh(script: full_script, label: env.STAGE_NAME, returnStatus: true) @@ -296,11 +300,11 @@ Map call(Map config = [:]) { runData['result'] = 'SUCCESS' } if (env.DAOS_JENKINS_NOTIFY_STATUS != null) { - stepResult name: env.STAGE_NAME, context: "build", + stepResult name: env.STAGE_NAME, context: 'build', result: runData['result'] } if (!config['returnStatus'] && (rc != 0)) { - error "sconsBuild failed for ${full_script}" + error "sconsBuild failed for ${full_script}" } if (config['stash_files']) { String target_stash = stage_info['target'] + '-' + diff --git a/vars/testRpm.groovy b/vars/testRpm.groovy index 5294abf58..4984f30da 100755 --- a/vars/testRpm.groovy +++ b/vars/testRpm.groovy @@ -23,7 +23,7 @@ * config['description'] Description to report for SCM status. * Default env.STAGE_NAME. * - * config['failure_artifacts'] Failure aritfifacts to return. + * config['failure_artifacts'] Failure artifacts to return. * Default env.STAGE_NAME. * * config['ignore_failure'] Ignore test failures. Default false. @@ -44,29 +44,27 @@ * * config['test_script'] Script to build RPMs. * Default 'ci/rpm/test_daos.sh'. - * */ Map call(Map config = [:]) { + Map runData = provisionNodes NODELIST: config.get('NODELIST', env.NODELIST), + node_count: 1, + profile: config.get('profile', 'daos_ci'), + distro: parseStageInfo(config)['ci_target'], + inst_repos: config.get('inst_repos', ''), + inst_rpms: config.get('inst_rpms', '') - Map runData = provisionNodes NODELIST: config.get('NODELIST', env.NODELIST), - node_count: 1, - profile: config.get('profile', 'daos_ci'), - distro: parseStageInfo(config)['ci_target'], - inst_repos: config.get('inst_repos', ''), - inst_rpms: config.get('inst_rpms', '') - - - Map runtestData = runTest script: 'export DAOS_PKG_VERSION="' + - config['daos_pkg_version'] + '"\n' + - config.get('test_script', 'ci/rpm/test_daos.sh'), + Map runtestData = runTest script: 'export DAOS_PKG_VERSION="' + + config['daos_pkg_version'] + '"\n' + + config.get('test_script', 'ci/rpm/test_daos.sh'), junit_files: config.get('junit_files', null), failure_artifacts: config.get('failure_artifacts', env.STAGE_NAME), ignore_failure: config.get('ignore_failure', false), description: config.get('description', env.STAGE_NAME), context: config.get('context', 'test/' + env.STAGE_NAME) - runtestData.each{ resultKey, data -> runData[resultKey] = data } - runData['rpmtest_time'] = durationSeconds(new Date()) + runtestData.each { resultKey, data -> runData[resultKey] = data } + long endDate = System.currentTimeMillis() + runData['rpmtest_time'] = durationSeconds(endDate) - return runData + return runData } diff --git a/vars/unitTest.groovy b/vars/unitTest.groovy index a0b42b595..0f6c2d8a3 100755 --- a/vars/unitTest.groovy +++ b/vars/unitTest.groovy @@ -122,10 +122,9 @@ Map afterTest(Map config, Map testRunInfo) { return result } +/* groovylint-disable-next-line MethodSize */ Map call(Map config = [:]) { - // Must use Date() in pipeline-lib - // groovylint-disable-next-line NoJavaUtilDate - Date startDate = new Date() + long startDate = System.currentTimeMillis() String nodelist = config.get('NODELIST', env.NODELIST) String test_script = config.get('test_script', 'ci/unit/test_main.sh') Map stage_info = parseStageInfo(config) @@ -223,7 +222,7 @@ Map call(Map config = [:]) { stash name: results_map, includes: results_map - // Stash any optional test coverage rreports for the stage + // Stash any optional test coverage reports for the stage String code_coverage = 'code_coverage_' + sanitizedStageName() stash name: code_coverage, includes: '**/code_coverage.json', diff --git a/vars/unitTestPost.groovy b/vars/unitTestPost.groovy index 4c619f74d..aa081e008 100755 --- a/vars/unitTestPost.groovy +++ b/vars/unitTestPost.groovy @@ -1,3 +1,4 @@ +/* groovylint-disable DuplicateNumberLiteral */ // vars/unitTestPost.groovy /** @@ -24,6 +25,7 @@ */ // groovylint-disable DuplicateStringLiteral, VariableName +/* groovylint-disable-next-line MethodSize */ void call(Map config = [:]) { Map stage_info = parseStageInfo(config) String cbcResult = currentBuild.currentResult @@ -32,15 +34,25 @@ void call(Map config = [:]) { String valgrind_pattern = stage_info.get('valgrind_pattern', 'unit-test-*memcheck.xml') if (config['valgrind_stash']) { - stash name: config['valgrind_stash'], includes: valgrind_pattern + try { + stash name: config['valgrind_stash'], includes: valgrind_pattern + } catch (hudson.AbortException e) { + println('Failed to stash Valgrind files with pattern ' + + "${valgrind_pattern}: ${e.message}") + } } String context = config.get('context', 'test/' + env.STAGE_NAME) String description = config.get('description', env.STAGE_NAME) String flow_name = config.get('flow_name', env.STAGE_NAME) // Need to unstash the script result from runTest String results_map = 'results_map_' + sanitizedStageName() - unstash name: results_map - Map results = readYaml file: results_map + Map results = [:] + try { + unstash name: results_map + results = readYaml file: results_map + } catch (hudson.AbortException e) { + println("Failed to unstash ${results_map}: ${e.message}") + } List artifact_list = config.get('artifacts', ['run_test.sh/*']) @@ -89,7 +101,7 @@ void call(Map config = [:]) { if (stage_info['build_type']) { target_stash += '-' + stage_info['build_type'] } - // Coverage instrumented tests and Vagrind are probably mutually exclusive + // Coverage instrumented tests and Valgrind are probably mutually exclusive if (stage_info['compiler'] == 'covc') { return } diff --git a/vars/valgrindReportPublish.groovy b/vars/valgrindReportPublish.groovy index b887a1de9..86052a6b6 100755 --- a/vars/valgrindReportPublish.groovy +++ b/vars/valgrindReportPublish.groovy @@ -4,25 +4,31 @@ /** * Valgrind Report Publish step method * - * Consolidiate Valgrind data for publising. + * Consolidate Valgrind data for publishing. * * @param config Map of parameters passed * * config['ignore_failure'] Ignore test failures. Default false. * - * config['valgrind_pattern'] Pattern for Valgind files. + * config['valgrind_pattern'] Pattern for Valgrind files. * Default: '*.memcheck.xml' * * config['valgrind_stashes'] list of stashes for valgrind results. */ void call(Map config = [:]) { + if (env.DAOS_VALGRIND_DISABLED != null) { + // Valgrind plugin is not being maintained and is generating log noise + // This is to allow testing with it disabled or missing. + println 'Valgrind publishing disabled via DAOS_VALGRIND_DISABLED' + return + } List stashes = [] if (config['valgrind_stashes']) { stashes = config['valgrind_stashes'] } else { // Older code publishes the valgrind in the same stage as ran the - // vagrind test. + // valgrind test. // That does not work if you have multiple valgrind stages running. // Need to have only one Valgrind publish stage println 'No valgrind_stashes passed! Running older code!' @@ -34,10 +40,9 @@ void call(Map config = [:]) { stashes.each { stash -> try { unstash stash - println("Success unstashing ${stash}.") + println("Success un-stashing ${stash}.") stash_cnt++ - /* groovylint-disable-next-line CatchException */ - } catch (Exception ex) { + } catch (hudson.AbortException ex) { println("Ignoring failure to unstash ${stash}. Perhaps the stage was skipped?") } }