Permalink
Browse files

build.sh: Upgrade to 3.3.6

:Release Notes:

:Detailed Notes:
3.2.5..3.3.6:
b7bc094 build.sh: Archive tarball with sources when available (v3.3.6)
6bcbf1a build.sh: Move generate_bom to separate function (v3.3.5)
a0a7b23 build.sh: Remove support for unpacking GF_ToolChain (v3.3.4)
14afdbc build.sh: Use separate job_type for integrate jobs (v3.3.2)
bce3579 Revert "build.sh: Create tarball with licenses" (v3.3.1)
14e4dc8 build-sh: Remove MACHINE from WEBOS_DISTRO_BUILD_ID (v3.3.0)
c6bd467 build.sh: Swap build-type and site in WEBOS_DISTRO_BUILD_ID
9329cec build.sh: Fix incremental builds to checkout the correct branch
bbed618 build.sh: Show QA issues in console output
ffcb5ca build.sh: Create tarball with licenses
16dcff2 build.sh: Publish tarball with buildstats and show top 20 tasks
295b56f build.sh: Use dot instead of colon as site and job-type separator

:Testing Performed:
Build tested.

:QA Notes:
No change in image.

:Issues Addressed:
[GF-60602] Distribute source code to comply with the open source
           obligation
[ES-10] Generate package differences file and include with build results
[ES-375] Remove MACHINE from <build-type> in WEBOS_DISTRO_BUILD_ID
[ES-374] Swap <build-type> and <site> in WEBOS_DISTRO_BUILD_ID
[ES-378] build.sh ignores branches specified in weboslayers.py
[GF-52560] Fix or hide all known QA issues to get clear qa.log from
           verification builds
[ES-316] Add deploy/licenses tree to artifacts retained by the build
[ES-271] Measure where time is going in our various OE builds
[GF-6531] Don't use colons in WEBOS_DISTRO_BUILD_ID

Open-webOS-DCO-1.0-Signed-off-by: Martin Jansa <martin.jansa@lge.com>

Change-Id: If47bfff99a323acd10cd1af2ce1c48227fc33bb2
Reviewed-on: https://g2g.palm.com/4469
Reviewed-by: DCO Verification
Reviewed-by: Martin Jansa <Martin.Jansa@lge.com>
Tested-by: Martin Jansa <Martin.Jansa@lge.com>
Reviewed-by: Susan Montooth <susan.montooth@lge.com>
  • Loading branch information...
1 parent 9bd148f commit 10b3b86fac02ce04befcd9f8e8ec296ae8dbd3ad Martin Jansa committed with susan-montooth Jan 23, 2014
Showing with 87 additions and 43 deletions.
  1. +87 −43 scripts/build.sh
View
@@ -1,6 +1,6 @@
#!/bin/bash
-# Copyright (c) 2013 LG Electronics, Inc.
+# Copyright (c) 2013-2014 LG Electronics, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -18,7 +18,7 @@
#set -x
# Some constants
-SCRIPT_VERSION="3.2.5"
+SCRIPT_VERSION="3.3.6"
AUTHORITATIVE_OFFICIAL_BUILD_SITE="svl"
BUILD_REPO="build-webos"
@@ -104,9 +104,10 @@ function check_project {
echo "NOTE: Checking out $layer in $GERRIT_REFSPEC" >&2
git checkout FETCH_HEAD
else
- # for incremental builds we should add "git fetch" here
- echo "NOTE: Checking out $layer in origin/master" >&2
- git checkout remotes/origin/master
+ current_branch=`git branch --list|grep ^*\ |awk '{print $2}'`
+ echo "NOTE: Run 'git remote update && git reset --hard origin/$current_branch' in $layer" >&2
+ echo "NOTE: Current branch - $current_branch"
+ git remote update && git reset --hard origin/$current_branch
fi
popd >/dev/null
fi
@@ -144,6 +145,21 @@ function check_project_vars {
echo "$ldesc"
}
+function generate_bom {
+ MACHINE=$1
+ I=$2
+ BBFLAGS=$3
+ FILENAME=$4
+
+ mkdir -p "${ARTIFACTS}/${MACHINE}/${I}" || true
+ /usr/bin/time -f "$TIME_STR" bitbake ${BBFLAGS} -g ${I}
+ grep '^"\([^"]*\)" \[label="\([^ ]*\) :\([^\\]*\)\\n\([^"]*\)"\]$' package-depends.dot |\
+ grep -v '^"\([^"]*\)" \[label="\([^ (]*([^ ]*)\) :\([^\\]*\)\\n\([^"]*\)"\]$' |\
+ sed 's/^"\([^"]*\)" \[label="\([^ ]*\) :\([^\\]*\)\\n\([^"]*\)"\]$/\1;\2;\3;\4/g' |\
+ sed "s#${BUILD_TOPDIR}/BUILD/.././##g" |\
+ sort > ${ARTIFACTS}/${MACHINE}/${I}/${FILENAME}
+}
+
TEMP=`getopt -o p:m:I:T:M:bshVu: --long buildhistory-path:,manifest-path:,images:,targets:,machines:,bom,signatures,help,version,scp-url: \
-n $(basename $0) -- "$@"`
@@ -222,24 +238,37 @@ else
# It's not expected that this script would ever be used for Open webOS as is,
# but the tests for it have been added as a guide for creating that edition.
case ${JOB_NAME} in
- build-*-official-*)
+ *-official-*)
job_type="official"
;;
- build-*-engineering-*)
- job_type="engr"
- ;;
-
clean-engineering-*)
# it cannot be verf or engr, because clean builds are managing layer checkouts alone
job_type="clean"
;;
- # The build-*-integrate-* jobs are the verification builds done right before
+ *-engineering-*)
+ job_type="engr"
+ ;;
+
+ *-verify-*)
+ job_type="verf"
+ ;;
+
+ # The *-integrate-* jobs are like the verification builds done right before
# the official builds. They have different names so that they can use a
# separate, special pool of Jenkins slaves.
- build-*-verify-*|build-*-integrate-*)
- job_type="verf"
+ *-integrate-*)
+ job_type="integ"
+ ;;
+
+ # The *-multilayer-* builds allow developers to trigger a multi-layer build
+ # from their desktop, without using the Jenkins parameterized build UI.
+ #
+ # The 'mlverf' job type is used so that the build-id makes it obvious that
+ # a multilayer build was performed (useful when evaluating CCC's).
+ *-multilayer-*)
+ job_type="mlverf"
;;
# Legacy job names
@@ -260,7 +289,7 @@ else
*) echo "Unrecognized JOB_NAME: '${JOB_NAME}'"
job_type="unrecognized!${JOB_NAME}"
;;
- esac
+ esac
# Convert job_types we recognize into abbreviations
case $job_type in
@@ -281,20 +310,33 @@ else
# when this assumption is broken.
BUILDHISTORY_BRANCH="master"
else
- # job_type can not contain any hyphens (except the trailing separator)
- job_type="${job_type//-/}-"
+ # job_type can not contain any hyphens
+ job_type="${job_type//-/}"
+ fi
+
+ # Append the separators to site and build-type.
+ #
+ # Use intermediate variables so that the remainder of the script need not concern
+ # itself with the separators, which are purely related to formatting the build id.
+ idsite="$site"
+ idtype="$job_type"
+
+ if [ -n "$idsite" ]; then
+ idsite="${idsite}-"
fi
- if [ -n "${site}" ]; then
- site="${site}:"
+ if [ -n "$idtype" ]; then
+ idtype="${idtype}."
fi
# BUILD_NUMBER should be set by the Jenkins executor
if [ -z "${BUILD_NUMBER}" ] ; then
- echo "JENKINS_URL set but BUILD_NUMBER isn't"
+ echo "JENKINS_URL is set, but BUILD_NUMBER isn't"
exit 1
fi
- export WEBOS_DISTRO_BUILD_ID=${site}${job_type}${BUILD_NUMBER}
+
+ # Format WEBOS_DISTRO_BUILD_ID as <build-type>.<site>-<build number>
+ export WEBOS_DISTRO_BUILD_ID=${idtype}${idsite}${BUILD_NUMBER}
fi
# Generate BOM files with metadata checked out by mcf (pinned versions)
@@ -305,7 +347,7 @@ if [ -n "${CREATE_BOM}" -a -n "${MACHINES}" ]; then
TIMESTAMP_OLD=$TIMESTAMP
printf "TIME: build.sh before first bom: $TIMESTAMP, +$TIMEDIFF, +$TIMEDIFF_START\n"
- if [ "$job_type" = "verf-" -o "$job_type" = "engr-" -o "$job_type" = "clean-" ] ; then
+ if [ "$job_type" = "verf" -o "$job_type" = "mlverf" -o "$job_type" = "integ" -o "$job_type" = "engr" -o "$job_type" = "clean" ] ; then
# don't use -before suffix for official builds, because they don't need -after and .diff because
# there is no logic for using different revisions than weboslayers.py
BOM_FILE_SUFFIX="-before"
@@ -318,13 +360,7 @@ if [ -n "${CREATE_BOM}" -a -n "${MACHINES}" ]; then
cd BUILD-${M};
. bitbake.rc
for I in ${IMAGES} ${TARGETS}; do
- mkdir -p "${ARTIFACTS}/${M}/${I}" || true
- /usr/bin/time -f "$TIME_STR" bitbake ${BBFLAGS} -g ${I}
- grep '^"\([^"]*\)" \[label="\([^ ]*\) :\([^\\]*\)\\n\([^"]*\)"\]$' package-depends.dot |\
- grep -v '^"\([^"]*\)" \[label="\([^ (]*([^ ]*)\) :\([^\\]*\)\\n\([^"]*\)"\]$' |\
- sed 's/^"\([^"]*\)" \[label="\([^ ]*\) :\([^\\]*\)\\n\([^"]*\)"\]$/\1;\2;\3;\4/g' |\
- sed "s#$BUILD_TOPDIR/BUILD-$M/.././##g" |\
- sort > ${ARTIFACTS}/${M}/${I}/bom${BOM_FILE_SUFFIX}.txt
+ generate_bom "${MACHINE}" "${I}" "${BBFLAGS}" "bom${BOM_FILE_SUFFIX}.txt"
done
cd ..
done
@@ -336,8 +372,7 @@ TIMEDIFF_START=`expr $TIMESTAMP - $TIMESTAMP_START`
TIMESTAMP_OLD=$TIMESTAMP
printf "TIME: build.sh before verf/engr/clean logic: $TIMESTAMP, +$TIMEDIFF, +$TIMEDIFF_START\n"
-# Be aware there is '-' already appended from job_type="${job_type//-/}-" above
-if [ "$job_type" = "verf-" -o "$job_type" = "engr-" ] ; then
+if [ "$job_type" = "verf" -o "$job_type" = "mlverf" -o "$job_type" = "integ" -o "$job_type" = "engr" ] ; then
if [ "$GERRIT_PROJECT" != "${BUILD_REPO}" ] ; then
set -e # checkout issues are critical for verification and engineering builds
for project in "${BUILD_LAYERS[@]}" ; do
@@ -349,8 +384,7 @@ if [ "$job_type" = "verf-" -o "$job_type" = "engr-" ] ; then
BBFLAGS="${BBFLAGS} -k"
fi
-# Be aware there is '-' already appended from job_type="${job_type//-/}-" above
-if [ "$job_type" = "clean-" ] ; then
+if [ "$job_type" = "clean" ] ; then
set -e # checkout issues are critical for clean build
desc="[DESC]"
for project in "${BUILD_LAYERS[@]}" ; do
@@ -363,7 +397,7 @@ fi
# Generate BOM files again, this time with metadata possibly different for engineering and verification builds
if [ -n "${CREATE_BOM}" -a -n "${MACHINES}" ]; then
- if [ "$job_type" = "verf-" -o "$job_type" = "engr-" -o "$job_type" = "clean-" ] ; then
+ if [ "$job_type" = "verf" -o "$job_type" = "mlverf" -o "$job_type" = "integ" -o "$job_type" = "engr" -o "$job_type" = "clean" ] ; then
TIMESTAMP=`date +%s`
TIMEDIFF=`expr $TIMESTAMP - $TIMESTAMP_OLD`
TIMEDIFF_START=`expr $TIMESTAMP - $TIMESTAMP_START`
@@ -378,14 +412,10 @@ if [ -n "${CREATE_BOM}" -a -n "${MACHINES}" ]; then
cd BUILD-${M};
. bitbake.rc
for I in ${IMAGES} ${TARGETS}; do
- mkdir -p "${ARTIFACTS}/${M}/${I}" || true
- /usr/bin/time -f "$TIME_STR" bitbake ${BBFLAGS} -g ${I}
- grep '^"\([^"]*\)" \[label="\([^ ]*\) :\([^\\]*\)\\n\([^"]*\)"\]$' package-depends.dot |\
- grep -v '^"\([^"]*\)" \[label="\([^ (]*([^ ]*)\) :\([^\\]*\)\\n\([^"]*\)"\]$' |\
- sed 's/^"\([^"]*\)" \[label="\([^ ]*\) :\([^\\]*\)\\n\([^"]*\)"\]$/\1;\2;\3;\4/g' |\
- sed "s#$BUILD_TOPDIR/BUILD-$M/.././##g" |\
- sort > ${ARTIFACTS}/${M}/${I}/bom-after.txt
- diff ${ARTIFACTS}/${M}/${I}/bom-before.txt ${ARTIFACTS}/${M}/${I}/bom-after.txt > ${ARTIFACTS}/${M}/${I}/bom-diff.txt
+ generate_bom "${MACHINE}" "${I}" "${BBFLAGS}" "bom-after.txt"
+ diff ${ARTIFACTS}/${MACHINE}/${I}/bom-before.txt \
+ ${ARTIFACTS}/${MACHINE}/${I}/bom-after.txt \
+ > ${ARTIFACTS}/${MACHINE}/${I}/bom-diff.txt
done
cd ..
done
@@ -409,7 +439,7 @@ if [ -n "${SIGNATURES}" -a -n "${MACHINES}" ]; then
mkdir -p "${ARTIFACTS}/${M}" || true
# normally this is executed for all MACHINEs togethere, but we're using MACHINE-specific BSP layers
../oe-core/scripts/sstate-diff-machines.sh --tmpdir=. --targets="${IMAGES} ${TARGETS}" --machines="${M}"
- tar cjvf ${ARTIFACTS}/${M}/sstate-diff.tar.bz2 sstate-diff --remove-files
+ tar cjf ${ARTIFACTS}/${M}/sstate-diff.tar.bz2 sstate-diff --remove-files
cd ..
done
fi
@@ -458,8 +488,17 @@ if [ -n "${MACHINES}" ]; then
mkdir -p "${ARTIFACTS}/${M}" || true
# copy webosvbox if we've built vmdk image
- cp qa.log ${ARTIFACTS}/${M} || true
+ if [ -e qa.log ]; then
+ cp qa.log ${ARTIFACTS}/${M} || true
+ # show them in console log so they are easier to spot (without downloading qa.log from artifacts
+ echo "WARN: Following QA issues were found:"
+ cat qa.log
+ else
+ echo "NOTE: No QA issues were found."
+ fi
cp WEBOS_BOM_data.pkl ${ARTIFACTS}/${M} || true
+ grep "Elapsed time" buildstats/*/*/*/* | sed 's/^.*\/\(.*\): Elapsed time: \(.*\)$/\2 \1/g' | sort -n | tail -n 20 | tee -a ${ARTIFACTS}/${M}/top20buildstats.txt
+ tar cjf ${ARTIFACTS}/${M}/buildstats.tar.bz2 buildstats
for I in ${IMAGES}; do
mkdir -p "${ARTIFACTS}/${M}/${I}" || true
# we store only tar.gz, vmdk.zip and .epk images
@@ -520,6 +559,11 @@ if [ -n "${MACHINES}" ]; then
fi
fi
done
+ if [ -d deploy/sources ] ; then
+ # exclude diff.gz files, because with old archiver they contain whole source (nothing creates .orig directory)
+ # see http://lists.openembedded.org/pipermail/openembedded-core/2013-December/087729.html
+ tar czf ${ARTIFACTS}/${M}/sources.tar.gz deploy/sources --exclude \*.diff.gz
+ fi
cd ..
done
else

0 comments on commit 10b3b86

Please sign in to comment.