diff --git a/README.md b/README.md index e0d93b35..92867a7b 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,8 @@ support. * Lightweight stack compared to Apache-ModPHP * Composer support * Various frameworks support out of the box (no configuration) -* Dynamic installing of [supported extensions](support/ext) listed as `ext-` requirements in `composer.json`. +* Dynamic installing of PHP extensions, listed as `ext-` requirements in `composer.json`. +* Interfacing with PECL platform to download and build extensions dynamically ## How to use it @@ -415,6 +416,58 @@ A tail on each unique log file will be run at application startup ], +### PHP Extensions + +Different types of extensions can be added to the PHP runtime, with different +levels of compatibility. All these extensions can be added in the `require` +block of the `composer.json` and this buildpack will try to install them. + +``` +{ + ... + "require": { + "ext-sodium": "*", + "ext-calendar": "*", + } +} +``` + +#### Internal PHP Extensions + +The PHP binary is not built with all modules embedded. To reduce build size and +attack surface, these extensions have to be enabled manually. The supported list +can be found here: https://github.com/Scalingo/php-buildpack/tree/master/support/ext-internal + +#### PECL PHP Extensions + +PECL extensions are registered on the https://pecl.php.net platform. This +buildpack will try to download and build them during deployments. + +Some extensions may require system dependencies absent from the stack your +application is based on. In this case you're encouraged to prepend the [APT +Buildpack](https://doc.scalingo.com/platform/deployment/buildpacks/apt) and +install the missing package according to the extension documentation. + +You may need to add extra compilation flags, in this case define them through +environment variables with the following pattern: + +``` +# replace $extension_name by the right extension name +PHP_PECL_EXTENSION_CONFIGURE_ARGS_$extension_name="--flag-to-add" +``` + +If the extension you are trying to install is a Zend extension, please define + +``` +# replace $extension_name by the right extension name +PHP_PECL_EXTENSION_IS_ZEND_$extension_name=true +``` + +#### Third Party PHP Extensions + +Some third party extensions can also be installed, list can be found here +https://github.com/Scalingo/php-buildpack/tree/master/support/ext + ## Node.js If your app contains a `package.json`, Node.js and its dependencies will be diff --git a/bin/compile b/bin/compile index 1611b113..7de8c5f0 100755 --- a/bin/compile +++ b/bin/compile @@ -8,10 +8,14 @@ shopt -s dotglob basedir="$( cd -P "$( dirname "$0" )" && pwd )" source "$basedir/../conf/buildpack.conf" source $basedir/common.sh +source $basedir/../lib/package source $basedir/../lib/composer source $basedir/../lib/datadog source $basedir/../lib/scout source $basedir/../lib/newrelic +source $basedir/../lib/apt +source $basedir/../lib/pecl +source $basedir/../lib/pecl_oci8 if [ "$PHP_BUILDPACK_NO_NODE" != "true" ] ; then source $basedir/../lib/nodejs @@ -23,54 +27,13 @@ fi BUILD_DIR="$1" CACHE_DIR="$2" +ENV_DIR="$3" cd "$BUILD_DIR" mkdir -p "$CACHE_DIR/package" export APP_ENV=${APP_ENV:-prod} -function fetch_engine_package() { - local engine="$1" - local version="$2" - local location="$3" - - local base_url="$PHP_BASE_URL" - if [ "$engine" = "nginx" ] ; then - base_url="$NGINX_BASE_URL" - fi - - fetch_package "$base_url" "${engine}-${version}" "$location" -} - -function fetch_package() { - local base_url="$1" - local package="$2" - local location="$3" - - mkdir -p "$location" - - local checksum_url="${base_url}/package/${package}.md5" - local package_url="${base_url}/package/${package}.tgz" - local checksum=$(curl --fail --retry 3 --retry-delay 2 --connect-timeout 3 --max-time 30 "$checksum_url" 2> /dev/null) - local cache_checksum="" - - if [ -f "$CACHE_DIR/package/${package}.md5" ]; then - local cache_checksum=$(cat "$CACHE_DIR/package/${package}.md5") - fi - - mkdir -p "$CACHE_DIR/package/$(dirname "$package")" - - if [ "$cache_checksum" != "$checksum" ]; then - curl --fail --retry 3 --retry-delay 2 --connect-timeout 3 --max-time 30 "$package_url" -L -s > "$CACHE_DIR/package/${package}.tgz" - echo "$checksum" > "$CACHE_DIR/package/${package}.md5" - else - echo "Checksums match. Fetching from cache." - fi - - mkdir -p "$location" - tar xzf "$CACHE_DIR/package/${package}.tgz" -C "$location" -} - function detect_framework() { BUILD_DIR=$1 for f in "$basedir/../frameworks/"*; do @@ -416,7 +379,7 @@ if [ "$PHP_BUILDPACK_NO_NODE" != "true" ] ; then install_node_deps "$BUILD_DIR" fi -install_composer_deps "$BUILD_DIR" +install_composer_deps "${BUILD_DIR}" "${CACHE_DIR}" "${ENV_DIR}" # Detect PHP framework # Set FRAMEWORK if not set in environment by user @@ -474,6 +437,7 @@ mv /app/vendor/php vendor/php [ -d "/app/vendor/libtidy" ] && mv /app/vendor/libtidy vendor/libtidy [ -d "/app/vendor/libsodium" ] && mv /app/vendor/libsodium vendor/libsodium [ -d "/app/vendor/libwebp" ] && mv /app/vendor/libwebp vendor/libwebp +[ -d "/app/vendor/oracle-client" ] && mv /app/vendor/oracle-client vendor/oracle-client mkdir -p "bin" "vendor/bin" diff --git a/lib/apt b/lib/apt new file mode 100644 index 00000000..3477290a --- /dev/null +++ b/lib/apt @@ -0,0 +1,28 @@ +function apt_install() { + local apt_deps="${1}" + local build_dir="${2}" + local cache_dir="${3}" + local env_dir="${4}" + + apt_manifest="$(mktemp "${build_dir}/Aptfile.php-XXX")" + echo "${apt_deps}" | tr ' ' '\n' > "${apt_manifest}" + + apt_deps_buildpack_dir=$(mktemp /tmp/apt_buildpack_XXXX) + rm "${apt_deps_buildpack_dir}" + + readonly apt_buildpack_url="https://github.com/Scalingo/apt-buildpack.git" + git clone --quiet --depth=1 "${apt_buildpack_url}" "${apt_deps_buildpack_dir}" >/dev/null 2>&1 + + readonly apt_log_file=$(mktemp /tmp/apt_log_XXXX.log) + APT_FILE_MANIFEST="$(basename ${apt_manifest})" "${apt_deps_buildpack_dir}/bin/compile" "${build_dir}" "${cache_dir}" "${env_dir}" >"${apt_log_file}" 2>&1 + if [ $? -ne 0 ] ; then + tail -n 30 "${apt_log_file}" | indent + echo + warn "Fail to install apt packages $apt_deps" + echo + exit 1 + fi + + # Source new libs for future buildpack (update of LD_LIBRARY_PATH) + echo "${apt_deps_buildpack_dir}/export" +} diff --git a/lib/composer b/lib/composer index 95ba30cc..3a39bde2 100644 --- a/lib/composer +++ b/lib/composer @@ -50,7 +50,7 @@ function install_composer_deps() { chmod a+x "$CACHE_DIR/composer.phar" echo "$checksum" > $CACHE_DIR/composer.phar.md5 else - echo "Checksums match. Fetching from cache." + echo "Checksums match. Fetching Composer from cache." |indent fi cp "$CACHE_DIR/composer.phar" "$target/vendor/composer/bin/" @@ -63,9 +63,25 @@ function install_composer_deps() { if [ -n "$required_extensions" ]; then status "Bundling additional extensions" for ext in $required_extensions; do - echo "$ext" | indent - # TODO: Find a better way to ignore extensions which were not found in S3 - if [ "$ext" = "memcached" ] ; then + local apt_deps="" + local ext_version=$(jq --raw-output ".require | .[\"ext-${ext}\"]" < "${BUILD_DIR}/composer.json") + + if [ "$ext" = "oci8" ] ; then + apt_deps="libaio-dev" + fi + + if [ -n "$apt_deps" ] ; then + echo "Installing dependencies for ${ext}: ${apt_deps}" | indent + install_env_file="$(apt_install "${apt_deps}" "${BUILD_DIR}" "${CACHE_DIR}" "${ENV_DIR}")" + # Source environment to export LD_LIBRARY_PATH + source "${install_env_file}" + rm "${install_env_file}" + fi + + # Install third-party dependencies after ubuntu packages have been added + if [ "$ext" = "oci8" ] ; then + install_oracle_client_extension "${BUILD_DIR}" "${CACHE_DIR}" + elif [ "$ext" = "memcached" ] ; then fetch_package "$PHP_BASE_URL" "libmemcached-${libmemcached_version}" "/app/vendor/libmemcached" | indent elif [ "$ext" = "gmp" ] ; then fetch_package "$PHP_BASE_URL" "gmp-${gmp_version}" "/app/vendor/gmp" | indent @@ -77,12 +93,20 @@ function install_composer_deps() { fi fetch_package "$PHP_BASE_URL" "libsodium-${sodium_version}" "/app/vendor/libsodium" | indent fi - fetch_package "$PHP_BASE_URL" "ext/$(php_api_version)/php-${ext}" "/app/vendor/php" 2>/dev/null || true | indent + + local extension_package_path="ext/$(php_api_version)/php-${ext}" + package_found="$(has_package "${PHP_BASE_URL}" "${extension_package_path}")" + if [ "${package_found}" = "true" ] ; then + echo "Installing PHP extension: ${ext}" | indent + fetch_package "${PHP_BASE_URL}" "${extension_package_path}" "/app/vendor/php" + else + install_pecl_extension "${ext}" "${ext_version}" "${BUILD_DIR}" "$CACHE_DIR" + fi done fi if [ -n "$COMPOSER_GITHUB_TOKEN" ]; then - status "Configuring the github authentication for Composer" + status "Configuring the GitHub authentication for Composer" php "vendor/composer/bin/composer.phar" config -g github-oauth.github.com "$COMPOSER_GITHUB_TOKEN" --no-interaction fi @@ -101,4 +125,3 @@ function install_composer_deps() { cd "$cwd" } | indent } - diff --git a/lib/package b/lib/package new file mode 100644 index 00000000..d439336f --- /dev/null +++ b/lib/package @@ -0,0 +1,55 @@ +function fetch_engine_package() { + local engine="$1" + local version="$2" + local location="$3" + + local base_url="$PHP_BASE_URL" + if [ "$engine" = "nginx" ] ; then + base_url="$NGINX_BASE_URL" + fi + + fetch_package "$base_url" "${engine}-${version}" "$location" +} + +function has_package() { + local base_url="${1}" + local package="${2}" + + local package_url="${base_url}/package/${package}.tgz" + + curl --output /dev/null --silent --head --location --fail "${package_url}" + if [ $? -eq 0 ] ; then + echo "true" + else + echo "false" + fi +} + +function fetch_package() { + local base_url="$1" + local package="$2" + local location="$3" + + mkdir -p "$location" + + local checksum_url="${base_url}/package/${package}.md5" + local package_url="${base_url}/package/${package}.tgz" + local checksum=$(curl --fail --retry 3 --retry-delay 2 --connect-timeout 3 --max-time 30 "$checksum_url" 2> /dev/null) + local cache_checksum="" + + if [ -f "$CACHE_DIR/package/${package}.md5" ]; then + local cache_checksum=$(cat "$CACHE_DIR/package/${package}.md5") + fi + + mkdir -p "$CACHE_DIR/package/$(dirname "$package")" + + if [ "$cache_checksum" != "$checksum" ]; then + curl --fail --retry 3 --retry-delay 2 --connect-timeout 3 --max-time 30 "$package_url" --location --silent > "$CACHE_DIR/package/${package}.tgz" + echo "$checksum" > "$CACHE_DIR/package/${package}.md5" + else + echo "Checksums match. Fetching from cache." | indent + fi + + mkdir -p "$location" + tar xzf "$CACHE_DIR/package/${package}.tgz" --directory "$location" +} diff --git a/lib/pecl b/lib/pecl new file mode 100644 index 00000000..cc06ae73 --- /dev/null +++ b/lib/pecl @@ -0,0 +1,87 @@ +function install_pecl_extension() { + local extension_name="${1}" + local version="${2}" + local cache_dir="${3}" + + if [[ $version = '*' ]]; then + local version=$(curl --silent "https://pecl.php.net/rest/r/$extension_name/latest.txt") + fi + local ext_dir=/app/vendor/php/lib/php/extensions/no-debug-non-zts-$(php_api_version) + + local cache_extension_file="${cache_dir}/${extension_name}-${version}.so" + if [ -f "${cache_extension_file}" ]; then + echo "Installing PECL extension ${extension_name} version ${version} from the cache" | indent + cp "${cache_extension_file}" "${ext_dir}/${extension_name}.so" + enable_pecl_extension ${extension_name} + return + fi + + local build_dir=$(pwd) + local temp_dir=$(mktmpdir "pecl-extension") + + echo "Installing PECL extension ${extension_name} version ${version}" | indent + + pushd "${temp_dir}" > /dev/null + + curl --silent --location "https://pecl.php.net/get/${extension_name}-${version}.tgz" | tar xz + + pushd ${extension_name}-${version} > /dev/null + ( + set +e + readonly phpize_log_file=$(mktemp "/tmp/pecl-phpize-${extension_name}-XXXX.log") + /app/vendor/php/bin/phpize > "${phpize_log_file}" 2>&1 + [ $? -eq 0 ] || install_pecl_error "${extension_name}" "package" "${phpize_log_file}" + + local configure_extension_args_var_name="PHP_PECL_EXTENSION_CONFIGURE_ARGS_$extension_name" + local configure_extension_args=$(printenv $configure_extension_args_var_name) + local flags=$(echo $configure_extension_args | sed "s|\$BUILD_DIR|$build_dir|") + + if [[ $extension_name = 'oci8' ]]; then + flags="--with-oci8=instantclient,${ORACLE_HOME}" + fi + + readonly configure_log_file=$(mktemp "/tmp/pecl-configure-${extension_name}-XXXX.log") + ./configure --with-php-config=/app/vendor/php/bin/php-config $flags > "${configure_log_file}" 2>&1 + [ $? -eq 0 ] || install_pecl_error "${extension_name}" "configure build" "${configure_log_file}" + + readonly make_log_file=$(mktemp "/tmp/pecl-make-${extension_name}-XXXX.log") + make -j 2 > "${make_log_file}" 2>&1 + [ $? -eq 0 ] || install_pecl_error "${extension_name}" "compile" "${make_log_file}" + ) + + cp modules/${extension_name}.so "${ext_dir}/${extension_name}.so" + cp modules/${extension_name}.so "${cache_extension_file}" + enable_pecl_extension ${extension_name} + + popd > /dev/null + popd > /dev/null +} + +function enable_pecl_extension() { + local extension_name="${1}" + local is_zend_extension_var_name="PHP_PECL_EXTENSION_IS_ZEND_$extension_name" + local is_zend_extension=$(printenv $is_zend_extension_var_name) + + if [[ $is_zend_extension = "true" ]] ; then + echo "zend_extension=${extension_name}.so" > "/app/vendor/php/etc/conf.d/${extension_name}.ini" + else + echo "extension=${extension_name}.so" > "/app/vendor/php/etc/conf.d/${extension_name}.ini" + fi +} + +function install_pecl_error() { + local extension_name="${1}" + local action="${2}" + local log_file="${3}" + + echo + tail -n 30 "${log_file}" | indent + echo + # This sleep prevents from having the following lines mixed up with the output + # of the above tail. Mystery of shellscripting. + sleep 0.1 + warn "Fail to ${action} of PHP PECL extension: ${extension_name}" + echo "Read above logs to understand the source of the issue" | indent + echo + exit 1 +} diff --git a/lib/pecl_oci8 b/lib/pecl_oci8 new file mode 100644 index 00000000..35c84981 --- /dev/null +++ b/lib/pecl_oci8 @@ -0,0 +1,26 @@ +function install_oracle_client_extension() { + # ---- + # Install oracle client + local build_dir="${1}" + local cache_dir="${2}" + + local oracle_install_dir="${build_dir}/vendor/oracle-client/" + mkdir -p "${oracle_install_dir}" + + curl --silent https://download.oracle.com/otn_software/linux/instantclient/instantclient-basic-linuxx64.zip --output oracleclient.zip + unzip -q oracleclient.zip -d "${oracle_install_dir}" + rm oracleclient.zip + + export ORACLE_HOME="${oracle_install_dir}$(ls ${oracle_install_dir})" + export LD_LIBRARY_PATH="${ORACLE_HOME}/lib:${ORACLE_HOME}:${LD_LIBRARY_PATH}" + + curl --silent https://download.oracle.com/otn_software/linux/instantclient/instantclient-sdk-linuxx64.zip --output oraclesdk.zip + unzip -q oraclesdk.zip -d "${oracle_install_dir}" + rm oraclesdk.zip + + readonly startup_script="${1}/.profile.d/oracle-client.sh" + # Replacing build_dir by /app, final location of the image data. + runtime_oracle_home="$(echo "${ORACLE_HOME}" | sed -e "s+${build_dir}+/app+")" + echo "export ORACLE_HOME=${runtime_oracle_home}" >> "${startup_script}" + echo "export LD_LIBRARY_PATH=${runtime_oracle_home}/lib:${runtime_oracle_home}:\${LD_LIBRARY_PATH}" >> "${startup_script}" +}