Permalink
Browse files

build release tarball: open source it (#1474)

* build release tarball: open source it

* change it from being given a branch to working on the current checkout
* change it to work with create_distro_tarball, which was already opensourced
* make it build openssl 1.0.2 if needed

* doc ubuntu 14 dep
  • Loading branch information...
1 parent 58c175c commit 6a15917500de119f924712001ccd28ab2358f03d @jeffkaufman jeffkaufman committed on GitHub Jan 11, 2017
Showing with 183 additions and 119 deletions.
  1. +96 −0 devel/build_release_tarball.sh
  2. +87 −119 devel/create_distro_tarball.sh
@@ -0,0 +1,96 @@
+#!/bin/bash
+#
+# Note: you might need to type your password a few times, once early, and once
+# at the end.
+#
+# This should be run on a release branch to make sure we can make a tarball and
+# at least build it on our workstations. It will also copy the tarball into
+# ~/release (where the binaries usually go).
+#
+# Like most of our dev tools this assumes Ubuntu 14 LTS. If that isn't what you
+# have, it's probably easiest to run this in a VM.
+#
+# Note that if this fails you may need to tweak the file list inside
+# devel/create_distro_tarball.sh
+
+set -e # exit script if any command returns an error
+set -u # exit the script if any variable is uninitialized
+
+function usage {
+ echo "Usage: devel/build_release_tarball.sh <beta|stable>"
+ exit 1
+}
+
+if [ $# -ne 1 ]; then
+ usage
+fi
+
+if [ ! -d net/instaweb ]; then
+ echo "This script must be run from the root of the mps checkout."
+ exit 1
+fi
+
+source net/instaweb/public/VERSION
+RELEASE="$MAJOR.$MINOR.$BUILD.$PATCH"
+CHANNEL="$1"
+
+deps="libpng12-dev libicu-dev libssl-dev libjpeg-dev realpath build-essential
+ pkg-config gperf unzip libapr1-dev libaprutil1-dev apache2-dev"
+if dpkg-query -Wf '${Status}\n' $deps 2>&1 | \
+ grep -v "install ok installed"; then
+ # Only run apt-get install if one of the deps is not already installed.
+ # See: http://stackoverflow.com/questions/1298066
+ sudo apt-get install $deps
+fi
+
+RELEASE_DIR="$HOME/release/$RELEASE"
+mkdir -p "$RELEASE_DIR"
+REVISION="$(build/lastchange.sh "$PWD" | sed 's/LASTCHANGE=//')"
+TARBALL="$RELEASE_DIR/mod-pagespeed-$CHANNEL-$RELEASE-r$REVISION.tar.bz2"
+devel/create_distro_tarball.sh "$TARBALL"
+
+echo "Tarball should now be at $TARBALL"
+
+# Try to build it
+BUILD_DIR="$(mktemp -d)"
+echo "Doing a test build inside $BUILD_DIR"
+cd "$BUILD_DIR"
+
+if openssl version | grep "^OpenSSL 1[.]0[.][01]\|^OpenSSL 0[.]"; then
+ echo "Your openssl version is too old to build the tarball; we need 1.0.2+"
+ echo "Building 1.0.2 from source..."
+ OPENSSL_VERSION="1.0.2j"
+ wget "https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz"
+ tar -xzvf "openssl-${OPENSSL_VERSION}.tar.gz"
+ cd openssl-"${OPENSSL_VERSION}"
+ ./config --prefix="$BUILD_DIR" shared
+ make
+ make install
+ export SSL_CERT_DIR=/etc/ssl/certs
+ export PKG_CONFIG_PATH="$BUILD_DIR/lib/pkgconfig"
+ export LD_LIBRARY_PATH="$BUILD_DIR/lib"
+ cd "$BUILD_DIR"
+fi
+
+tar xjf "$TARBALL"
+cd modpagespeed*
+./generate.sh -Dsystem_include_path_apr=/usr/include/apr-1.0/ \
+ -Dsystem_include_path_httpd=/usr/include/apache2
+cd src
+make -j6
+out/Debug/mod_pagespeed_test
+# These tests fail because they are golded against a specific version of
+# compression libraries.
+# TODO(sligocki): Could we change the tests to be less fragile or test in a
+# different way in this case?
+BROKEN_TESTS=\
+ImageConverterTest.OptimizePngOrConvertToJpeg:\
+ImageConverterTest.ConvertOpaqueGifToJpeg:\
+JpegOptimizerTest.ValidJpegsLossy:\
+JpegOptimizerTest.ValidJpegLossyAndColorSampling:\
+JpegOptimizerTest.ValidJpegsProgressiveAndLossy
+out/Debug/pagespeed_automatic_test --gtest_filter=-$BROKEN_TESTS
+
+echo "Cleaning up"
+rm -rf "$BUILD_DIR"
+
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
#
# Copyright 2011 Google Inc.
#
@@ -28,14 +28,15 @@
# the .gyp[i] files and doesn't work out of the box. The pruning was also done
# as of branch 33, so further tweaks might be required for this mode in
# 34 or newer.
+#
+# This is expected to be run from build_release_tarball.sh, on the branch you
+# want a tarball for.
set -e # exit script if any command returns an error
set -u # exit the script if any variable is uninitialized
function usage {
- echo -n "create_distro_tarball_debian.sh [ --minimal ] [ --checkout_dir dir ] tarball"
- echo " [ --checkout commit_ish | source_tree ]"
- echo "examples of commit_ish would be 'master', '33', '1.11.33.4' or 04e3237"
+ echo "create_distro_tarball_debian.sh [ --minimal ] tarball"
exit 1
}
@@ -65,163 +66,130 @@ src/build/gyp_chromium -D use_system_libs=1 \$*
SCRIPT_END
}
-if [ $# -lt 2 ]; then
+if [ $# -lt 1 ]; then
usage
exit
fi
-CHECKOUT_DIR=
-if [ "$1" = "--checkout_dir" ]; then
- if [ -z $2 ]; then
- usage
- fi
- CHECKOUT_DIR=$2
- shift 2
-else
- CHECKOUT_DIR=$(mktemp -d)
- echo $CHECKOUT_DIR
-fi
-
MINIMAL=0
if [ "$1" == "--minimal" ]; then
MINIMAL=1
shift 1
fi
-TARBALL=$1
-if [ -z $TARBALL ]; then
+TARBALL="$1"
+if [ -z "$TARBALL" ]; then
usage
fi
-touch $TARBALL
-TARBALL=$(realpath $TARBALL)
+touch "$TARBALL"
+TARBALL="$(realpath $TARBALL)"
+MPS_CHECKOUT="$PWD"
-if [ "$2" == --checkout ]; then
- if [ -z $3 ]; then
- usage
- else
- GIT_TAG=$3
- if [ -z $CHECKOUT_DIR ]; then
- CHECKOUT_DIR=$(mktemp -d)
- fi
- SRC_DIR=$CHECKOUT_DIR
- cd $CHECKOUT_DIR
- git clone https://github.com/pagespeed/mod_pagespeed.git src
- cd src
- git checkout "$GIT_TAG"
- git submodule update --init --recursive
- fi
-else
- SRC_DIR=$2
-fi
+git submodule update --init --recursive
-cd $SRC_DIR
-
-# Pick up our version info, and wrap src inside a modpagespeed-version dir,
-# so unpackers don't get a src/ surprise
-source src/net/instaweb/public/VERSION
-VER_STRING=$MAJOR.$MINOR.$BUILD.$PATCH
-DIR=modpagespeed-$VER_STRING
-rm -rf $DIR
-mkdir $DIR
-echo ln -sf $SRC_DIR/src $DIR/src/
-ln -sf $SRC_DIR/src $DIR/src
+# Pick up our version info, and wrap src inside a modpagespeed-version dir.
+source net/instaweb/public/VERSION
+VER_STRING="$MAJOR.$MINOR.$BUILD.$PATCH"
+TEMP_DIR="$(mktemp -d)"
+WRAPPER_DIR="modpagespeed-$VER_STRING"
+mkdir "$TEMP_DIR/$WRAPPER_DIR"
+DIR="$WRAPPER_DIR/src"
+ln -s "$MPS_CHECKOUT" "$TEMP_DIR/$DIR"
# Also create a little helper script that shows how to run gyp
-config > $DIR/generate.sh
-chmod +x $DIR/generate.sh
+config > "$TEMP_DIR/$WRAPPER_DIR/generate.sh"
+chmod +x "$TEMP_DIR/$WRAPPER_DIR/generate.sh"
# Normally, the build system runs build/lastchange.sh to figure out what
# to put into the last portion of the version number. We are, however, going to
# be getting rid of the .git dirs, so that will not work (nor would it without
# network access). Luckily, we can provide the number via LASTCHANGE.in,
# so we just compute it now, and save it there.
-cd src/
-./build/lastchange.sh $PWD > LASTCHANGE.in
-cd ..
+./build/lastchange.sh "$MPS_CHECKOUT" > LASTCHANGE.in
# Things that depends on minimal or not.
if [ $MINIMAL -eq 0 ]; then
- GTEST=$DIR/src/testing
- GFLAGS=$DIR/src/third_party/gflags
- GIFLIB=$DIR/src/third_party/giflib
- ICU="$DIR/src/third_party/icu/icu.gyp \
- $DIR/src/third_party/icu/source/common/unicode/"
- JSONCPP=$DIR/src/third_party/jsoncpp
- LIBWEBP=$DIR/src/third_party/libwebp
- PROTOBUF=$DIR/src/third_party/protobuf
- RE2=$DIR/src/third_party/re2
+ GTEST=$DIR/testing
+ GFLAGS=$DIR/third_party/gflags
+ GIFLIB=$DIR/third_party/giflib
+ ICU="$DIR/third_party/icu/icu.gyp \
+ $DIR/third_party/icu/source/common/unicode/"
+ JSONCPP=$DIR/third_party/jsoncpp
+ LIBWEBP=$DIR/third_party/libwebp
+ PROTOBUF=$DIR/third_party/protobuf
+ RE2=$DIR/third_party/re2
else
- GTEST="$DIR/src/testing \
- --exclude $DIR/src/testing/gmock \
- --exclude $DIR/src/testing/gtest"
- GFLAGS=$DIR/src/third_party/gflags/gflags.gyp
- GIFLIB=$DIR/src/third_party/giflib/giflib.gyp
- ICU=$DIR/src/third_party/icu/icu.gyp
- JSONCPP=$DIR/src/third_party/jsoncpp/jsoncpp.gyp
- LIBWEBP="$DIR/src/third_party/libwebp/COPYING \
- $DIR/src/third_party/libwebp/examples/gif2webp_util.*"
- PROTOBUF="$DIR/src/third_party/protobuf/*.gyp \
- $DIR/src/third_party/protobuf/*.gypi"
- RE2=$DIR/src/third_party/re2/re2.gyp
+ GTEST="$DIR/testing \
+ --exclude $DIR/testing/gmock \
+ --exclude $DIR/testing/gtest"
+ GFLAGS=$DIR/third_party/gflags/gflags.gyp
+ GIFLIB=$DIR/third_party/giflib/giflib.gyp
+ ICU=$DIR/third_party/icu/icu.gyp
+ JSONCPP=$DIR/third_party/jsoncpp/jsoncpp.gyp
+ LIBWEBP="$DIR/third_party/libwebp/COPYING \
+ $DIR/third_party/libwebp/examples/gif2webp_util.*"
+ PROTOBUF="$DIR/third_party/protobuf/*.gyp \
+ $DIR/third_party/protobuf/*.gypi"
+ RE2=$DIR/third_party/re2/re2.gyp
fi
# It's tarball time!
# Note that this is highly-version specific, and requires tweaks for every
# release as its dependencies change. Always run the version of this
# script that's on the branch you're making a tarball for.
+cd "$TEMP_DIR"
tar cj --dereference --exclude='.git' --exclude='.svn' --exclude='.hg' -f $TARBALL \
--exclude='*.mk' --exclude='*.pyc' \
- --exclude=$DIR/src/net/instaweb/genfiles/*/*.cc \
- $DIR/generate.sh \
- $DIR/src/LASTCHANGE.in \
- $DIR/src/base \
- $DIR/src/build \
- --exclude $DIR/src/build/android/arm-linux-androideabi-gold \
- $DIR/src/install \
- $DIR/src/net/instaweb \
- $DIR/src/pagespeed \
- $DIR/src/strings \
+ --exclude=$DIR/net/instaweb/genfiles/*/*.cc \
+ $WRAPPER_DIR/generate.sh \
+ $DIR/LASTCHANGE.in \
+ $DIR/base \
+ $DIR/build \
+ --exclude $DIR/build/android/arm-linux-androideabi-gold \
+ $DIR/install \
+ $DIR/net/instaweb \
+ $DIR/pagespeed \
+ $DIR/strings \
$GTEST \
- $DIR/src/third_party/apr/apr.gyp \
- $DIR/src/third_party/aprutil/aprutil.gyp \
- $DIR/src/third_party/aprutil/apr_memcache2.h \
- $DIR/src/third_party/aprutil/apr_memcache2.c \
- $DIR/src/third_party/httpd/httpd.gyp \
- $DIR/src/third_party/httpd24/httpd24.gyp \
- $DIR/src/third_party/base64 \
- $DIR/src/third_party/brotli \
- $DIR/src/third_party/chromium/src/base \
+ $DIR/third_party/apr/apr.gyp \
+ $DIR/third_party/aprutil/aprutil.gyp \
+ $DIR/third_party/aprutil/apr_memcache2.h \
+ $DIR/third_party/aprutil/apr_memcache2.c \
+ $DIR/third_party/httpd/httpd.gyp \
+ $DIR/third_party/httpd24/httpd24.gyp \
+ $DIR/third_party/base64 \
+ $DIR/third_party/brotli \
+ $DIR/third_party/chromium/src/base \
--exclude src/third_party/chromium/src/base/third_party/xdg_mime \
--exclude src/third_party/chromium/src/base/third_party/xdg_user_dirs \
- $DIR/src/third_party/chromium/src/build \
- --exclude $DIR/src/third_party/chromium/src/build/android \
- $DIR/src/third_party/chromium/src/googleurl \
- $DIR/src/third_party/chromium/src/net/tools \
- $DIR/src/third_party/closure/ \
- $DIR/src/third_party/closure_library/ \
- $DIR/src/third_party/css_parser \
- $DIR/src/third_party/domain_registry_provider \
+ $DIR/third_party/chromium/src/build \
+ --exclude $DIR/third_party/chromium/src/build/android \
+ $DIR/third_party/chromium/src/googleurl \
+ $DIR/third_party/chromium/src/net/tools \
+ $DIR/third_party/closure/ \
+ $DIR/third_party/closure_library/ \
+ $DIR/third_party/css_parser \
+ $DIR/third_party/domain_registry_provider \
$GFLAGS \
$GIFLIB \
- $DIR/src/third_party/google-sparsehash \
- $DIR/src/third_party/grpc \
- $DIR/src/third_party/hiredis \
+ $DIR/third_party/google-sparsehash \
+ $DIR/third_party/grpc \
+ $DIR/third_party/hiredis \
$ICU \
$JSONCPP \
- $DIR/src/third_party/libjpeg_turbo/libjpeg_turbo.gyp \
- $DIR/src/third_party/libpng/libpng.gyp \
+ $DIR/third_party/libjpeg_turbo/libjpeg_turbo.gyp \
+ $DIR/third_party/libpng/libpng.gyp \
$LIBWEBP \
- $DIR/src/third_party/modp_b64 \
- $DIR/src/third_party/optipng \
+ $DIR/third_party/modp_b64 \
+ $DIR/third_party/optipng \
$PROTOBUF \
- $DIR/src/third_party/rdestl \
+ $DIR/third_party/rdestl \
$RE2 \
- $DIR/src/third_party/redis-crc \
- $DIR/src/third_party/serf \
- $DIR/src/third_party/zlib/zlib.gyp \
- $DIR/src/tools/gyp \
- $DIR/src/url
+ $DIR/third_party/redis-crc \
+ $DIR/third_party/serf \
+ $DIR/third_party/zlib/zlib.gyp \
+ $DIR/tools/gyp \
+ $DIR/url
-if [ x != x$CHECKOUT_DIR ]; then
- echo "You may want to rm -rf $CHECKOUT_DIR to clean up"
-fi
+cd "$MPS_CHECKOUT"
+rm -r "$TEMP_DIR"

0 comments on commit 6a15917

Please sign in to comment.