From 10b3b86fac02ce04befcd9f8e8ec296ae8dbd3ad Mon Sep 17 00:00:00 2001 From: Martin Jansa Date: Thu, 23 Jan 2014 14:33:33 +0100 Subject: [PATCH] build.sh: Upgrade to 3.3.6 :Release Notes: :Detailed Notes: 3.2.5..3.3.6: b7bc094 build.sh: Archive tarball with sources when available (v3.3.6) 6bcbf1a build.sh: Move generate_bom to separate function (v3.3.5) a0a7b23 build.sh: Remove support for unpacking GF_ToolChain (v3.3.4) 14afdbc build.sh: Use separate job_type for integrate jobs (v3.3.2) bce3579 Revert "build.sh: Create tarball with licenses" (v3.3.1) 14e4dc8 build-sh: Remove MACHINE from WEBOS_DISTRO_BUILD_ID (v3.3.0) c6bd467 build.sh: Swap build-type and site in WEBOS_DISTRO_BUILD_ID 9329cec build.sh: Fix incremental builds to checkout the correct branch bbed618 build.sh: Show QA issues in console output ffcb5ca build.sh: Create tarball with licenses 16dcff2 build.sh: Publish tarball with buildstats and show top 20 tasks 295b56f build.sh: Use dot instead of colon as site and job-type separator :Testing Performed: Build tested. :QA Notes: No change in image. :Issues Addressed: [GF-60602] Distribute source code to comply with the open source obligation [ES-10] Generate package differences file and include with build results [ES-375] Remove MACHINE from in WEBOS_DISTRO_BUILD_ID [ES-374] Swap and in WEBOS_DISTRO_BUILD_ID [ES-378] build.sh ignores branches specified in weboslayers.py [GF-52560] Fix or hide all known QA issues to get clear qa.log from verification builds [ES-316] Add deploy/licenses tree to artifacts retained by the build [ES-271] Measure where time is going in our various OE builds [GF-6531] Don't use colons in WEBOS_DISTRO_BUILD_ID Open-webOS-DCO-1.0-Signed-off-by: Martin Jansa Change-Id: If47bfff99a323acd10cd1af2ce1c48227fc33bb2 Reviewed-on: https://g2g.palm.com/4469 Reviewed-by: DCO Verification Reviewed-by: Martin Jansa Tested-by: Martin Jansa Reviewed-by: Susan Montooth --- scripts/build.sh | 130 +++++++++++++++++++++++++++++++---------------- 1 file changed, 87 insertions(+), 43 deletions(-) diff --git a/scripts/build.sh b/scripts/build.sh index 51ba910..f224f89 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2013 LG Electronics, Inc. +# Copyright (c) 2013-2014 LG Electronics, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,7 +18,7 @@ #set -x # Some constants -SCRIPT_VERSION="3.2.5" +SCRIPT_VERSION="3.3.6" AUTHORITATIVE_OFFICIAL_BUILD_SITE="svl" BUILD_REPO="build-webos" @@ -104,9 +104,10 @@ function check_project { echo "NOTE: Checking out $layer in $GERRIT_REFSPEC" >&2 git checkout FETCH_HEAD else - # for incremental builds we should add "git fetch" here - echo "NOTE: Checking out $layer in origin/master" >&2 - git checkout remotes/origin/master + current_branch=`git branch --list|grep ^*\ |awk '{print $2}'` + echo "NOTE: Run 'git remote update && git reset --hard origin/$current_branch' in $layer" >&2 + echo "NOTE: Current branch - $current_branch" + git remote update && git reset --hard origin/$current_branch fi popd >/dev/null fi @@ -144,6 +145,21 @@ function check_project_vars { echo "$ldesc" } +function generate_bom { + MACHINE=$1 + I=$2 + BBFLAGS=$3 + FILENAME=$4 + + mkdir -p "${ARTIFACTS}/${MACHINE}/${I}" || true + /usr/bin/time -f "$TIME_STR" bitbake ${BBFLAGS} -g ${I} + grep '^"\([^"]*\)" \[label="\([^ ]*\) :\([^\\]*\)\\n\([^"]*\)"\]$' package-depends.dot |\ + grep -v '^"\([^"]*\)" \[label="\([^ (]*([^ ]*)\) :\([^\\]*\)\\n\([^"]*\)"\]$' |\ + sed 's/^"\([^"]*\)" \[label="\([^ ]*\) :\([^\\]*\)\\n\([^"]*\)"\]$/\1;\2;\3;\4/g' |\ + sed "s#${BUILD_TOPDIR}/BUILD/.././##g" |\ + sort > ${ARTIFACTS}/${MACHINE}/${I}/${FILENAME} +} + TEMP=`getopt -o p:m:I:T:M:bshVu: --long buildhistory-path:,manifest-path:,images:,targets:,machines:,bom,signatures,help,version,scp-url: \ -n $(basename $0) -- "$@"` @@ -222,24 +238,37 @@ else # It's not expected that this script would ever be used for Open webOS as is, # but the tests for it have been added as a guide for creating that edition. case ${JOB_NAME} in - build-*-official-*) + *-official-*) job_type="official" ;; - build-*-engineering-*) - job_type="engr" - ;; - clean-engineering-*) # it cannot be verf or engr, because clean builds are managing layer checkouts alone job_type="clean" ;; - # The build-*-integrate-* jobs are the verification builds done right before + *-engineering-*) + job_type="engr" + ;; + + *-verify-*) + job_type="verf" + ;; + + # The *-integrate-* jobs are like the verification builds done right before # the official builds. They have different names so that they can use a # separate, special pool of Jenkins slaves. - build-*-verify-*|build-*-integrate-*) - job_type="verf" + *-integrate-*) + job_type="integ" + ;; + + # The *-multilayer-* builds allow developers to trigger a multi-layer build + # from their desktop, without using the Jenkins parameterized build UI. + # + # The 'mlverf' job type is used so that the build-id makes it obvious that + # a multilayer build was performed (useful when evaluating CCC's). + *-multilayer-*) + job_type="mlverf" ;; # Legacy job names @@ -260,7 +289,7 @@ else *) echo "Unrecognized JOB_NAME: '${JOB_NAME}'" job_type="unrecognized!${JOB_NAME}" ;; - esac + esac # Convert job_types we recognize into abbreviations case $job_type in @@ -281,20 +310,33 @@ else # when this assumption is broken. BUILDHISTORY_BRANCH="master" else - # job_type can not contain any hyphens (except the trailing separator) - job_type="${job_type//-/}-" + # job_type can not contain any hyphens + job_type="${job_type//-/}" + fi + + # Append the separators to site and build-type. + # + # Use intermediate variables so that the remainder of the script need not concern + # itself with the separators, which are purely related to formatting the build id. + idsite="$site" + idtype="$job_type" + + if [ -n "$idsite" ]; then + idsite="${idsite}-" fi - if [ -n "${site}" ]; then - site="${site}:" + if [ -n "$idtype" ]; then + idtype="${idtype}." fi # BUILD_NUMBER should be set by the Jenkins executor if [ -z "${BUILD_NUMBER}" ] ; then - echo "JENKINS_URL set but BUILD_NUMBER isn't" + echo "JENKINS_URL is set, but BUILD_NUMBER isn't" exit 1 fi - export WEBOS_DISTRO_BUILD_ID=${site}${job_type}${BUILD_NUMBER} + + # Format WEBOS_DISTRO_BUILD_ID as .- + export WEBOS_DISTRO_BUILD_ID=${idtype}${idsite}${BUILD_NUMBER} fi # Generate BOM files with metadata checked out by mcf (pinned versions) @@ -305,7 +347,7 @@ if [ -n "${CREATE_BOM}" -a -n "${MACHINES}" ]; then TIMESTAMP_OLD=$TIMESTAMP printf "TIME: build.sh before first bom: $TIMESTAMP, +$TIMEDIFF, +$TIMEDIFF_START\n" - if [ "$job_type" = "verf-" -o "$job_type" = "engr-" -o "$job_type" = "clean-" ] ; then + if [ "$job_type" = "verf" -o "$job_type" = "mlverf" -o "$job_type" = "integ" -o "$job_type" = "engr" -o "$job_type" = "clean" ] ; then # don't use -before suffix for official builds, because they don't need -after and .diff because # there is no logic for using different revisions than weboslayers.py BOM_FILE_SUFFIX="-before" @@ -318,13 +360,7 @@ if [ -n "${CREATE_BOM}" -a -n "${MACHINES}" ]; then cd BUILD-${M}; . bitbake.rc for I in ${IMAGES} ${TARGETS}; do - mkdir -p "${ARTIFACTS}/${M}/${I}" || true - /usr/bin/time -f "$TIME_STR" bitbake ${BBFLAGS} -g ${I} - grep '^"\([^"]*\)" \[label="\([^ ]*\) :\([^\\]*\)\\n\([^"]*\)"\]$' package-depends.dot |\ - grep -v '^"\([^"]*\)" \[label="\([^ (]*([^ ]*)\) :\([^\\]*\)\\n\([^"]*\)"\]$' |\ - sed 's/^"\([^"]*\)" \[label="\([^ ]*\) :\([^\\]*\)\\n\([^"]*\)"\]$/\1;\2;\3;\4/g' |\ - sed "s#$BUILD_TOPDIR/BUILD-$M/.././##g" |\ - sort > ${ARTIFACTS}/${M}/${I}/bom${BOM_FILE_SUFFIX}.txt + generate_bom "${MACHINE}" "${I}" "${BBFLAGS}" "bom${BOM_FILE_SUFFIX}.txt" done cd .. done @@ -336,8 +372,7 @@ TIMEDIFF_START=`expr $TIMESTAMP - $TIMESTAMP_START` TIMESTAMP_OLD=$TIMESTAMP printf "TIME: build.sh before verf/engr/clean logic: $TIMESTAMP, +$TIMEDIFF, +$TIMEDIFF_START\n" -# Be aware there is '-' already appended from job_type="${job_type//-/}-" above -if [ "$job_type" = "verf-" -o "$job_type" = "engr-" ] ; then +if [ "$job_type" = "verf" -o "$job_type" = "mlverf" -o "$job_type" = "integ" -o "$job_type" = "engr" ] ; then if [ "$GERRIT_PROJECT" != "${BUILD_REPO}" ] ; then set -e # checkout issues are critical for verification and engineering builds for project in "${BUILD_LAYERS[@]}" ; do @@ -349,8 +384,7 @@ if [ "$job_type" = "verf-" -o "$job_type" = "engr-" ] ; then BBFLAGS="${BBFLAGS} -k" fi -# Be aware there is '-' already appended from job_type="${job_type//-/}-" above -if [ "$job_type" = "clean-" ] ; then +if [ "$job_type" = "clean" ] ; then set -e # checkout issues are critical for clean build desc="[DESC]" for project in "${BUILD_LAYERS[@]}" ; do @@ -363,7 +397,7 @@ fi # Generate BOM files again, this time with metadata possibly different for engineering and verification builds if [ -n "${CREATE_BOM}" -a -n "${MACHINES}" ]; then - if [ "$job_type" = "verf-" -o "$job_type" = "engr-" -o "$job_type" = "clean-" ] ; then + if [ "$job_type" = "verf" -o "$job_type" = "mlverf" -o "$job_type" = "integ" -o "$job_type" = "engr" -o "$job_type" = "clean" ] ; then TIMESTAMP=`date +%s` TIMEDIFF=`expr $TIMESTAMP - $TIMESTAMP_OLD` TIMEDIFF_START=`expr $TIMESTAMP - $TIMESTAMP_START` @@ -378,14 +412,10 @@ if [ -n "${CREATE_BOM}" -a -n "${MACHINES}" ]; then cd BUILD-${M}; . bitbake.rc for I in ${IMAGES} ${TARGETS}; do - mkdir -p "${ARTIFACTS}/${M}/${I}" || true - /usr/bin/time -f "$TIME_STR" bitbake ${BBFLAGS} -g ${I} - grep '^"\([^"]*\)" \[label="\([^ ]*\) :\([^\\]*\)\\n\([^"]*\)"\]$' package-depends.dot |\ - grep -v '^"\([^"]*\)" \[label="\([^ (]*([^ ]*)\) :\([^\\]*\)\\n\([^"]*\)"\]$' |\ - sed 's/^"\([^"]*\)" \[label="\([^ ]*\) :\([^\\]*\)\\n\([^"]*\)"\]$/\1;\2;\3;\4/g' |\ - sed "s#$BUILD_TOPDIR/BUILD-$M/.././##g" |\ - sort > ${ARTIFACTS}/${M}/${I}/bom-after.txt - diff ${ARTIFACTS}/${M}/${I}/bom-before.txt ${ARTIFACTS}/${M}/${I}/bom-after.txt > ${ARTIFACTS}/${M}/${I}/bom-diff.txt + generate_bom "${MACHINE}" "${I}" "${BBFLAGS}" "bom-after.txt" + diff ${ARTIFACTS}/${MACHINE}/${I}/bom-before.txt \ + ${ARTIFACTS}/${MACHINE}/${I}/bom-after.txt \ + > ${ARTIFACTS}/${MACHINE}/${I}/bom-diff.txt done cd .. done @@ -409,7 +439,7 @@ if [ -n "${SIGNATURES}" -a -n "${MACHINES}" ]; then mkdir -p "${ARTIFACTS}/${M}" || true # normally this is executed for all MACHINEs togethere, but we're using MACHINE-specific BSP layers ../oe-core/scripts/sstate-diff-machines.sh --tmpdir=. --targets="${IMAGES} ${TARGETS}" --machines="${M}" - tar cjvf ${ARTIFACTS}/${M}/sstate-diff.tar.bz2 sstate-diff --remove-files + tar cjf ${ARTIFACTS}/${M}/sstate-diff.tar.bz2 sstate-diff --remove-files cd .. done fi @@ -458,8 +488,17 @@ if [ -n "${MACHINES}" ]; then mkdir -p "${ARTIFACTS}/${M}" || true # copy webosvbox if we've built vmdk image - cp qa.log ${ARTIFACTS}/${M} || true + if [ -e qa.log ]; then + cp qa.log ${ARTIFACTS}/${M} || true + # show them in console log so they are easier to spot (without downloading qa.log from artifacts + echo "WARN: Following QA issues were found:" + cat qa.log + else + echo "NOTE: No QA issues were found." + fi cp WEBOS_BOM_data.pkl ${ARTIFACTS}/${M} || true + grep "Elapsed time" buildstats/*/*/*/* | sed 's/^.*\/\(.*\): Elapsed time: \(.*\)$/\2 \1/g' | sort -n | tail -n 20 | tee -a ${ARTIFACTS}/${M}/top20buildstats.txt + tar cjf ${ARTIFACTS}/${M}/buildstats.tar.bz2 buildstats for I in ${IMAGES}; do mkdir -p "${ARTIFACTS}/${M}/${I}" || true # we store only tar.gz, vmdk.zip and .epk images @@ -520,6 +559,11 @@ if [ -n "${MACHINES}" ]; then fi fi done + if [ -d deploy/sources ] ; then + # exclude diff.gz files, because with old archiver they contain whole source (nothing creates .orig directory) + # see http://lists.openembedded.org/pipermail/openembedded-core/2013-December/087729.html + tar czf ${ARTIFACTS}/${M}/sources.tar.gz deploy/sources --exclude \*.diff.gz + fi cd .. done else