From ecb3ba1aa43ba98e18492c3d46c37364f3984a9a Mon Sep 17 00:00:00 2001 From: Eric Kilmer Date: Tue, 15 Feb 2022 16:40:26 -0500 Subject: [PATCH] Simplify build script THe previous logic was very complex and didn't account for weird situations. Just remove the complexity in favor of telling users to start fresh --- README.md | 22 ++++-------- build_dependencies.sh | 80 +++++++------------------------------------ 2 files changed, 19 insertions(+), 83 deletions(-) diff --git a/README.md b/README.md index 7a134f35..f37f9ca3 100644 --- a/README.md +++ b/README.md @@ -45,11 +45,13 @@ You can pass `--help` to the script to look at all options. Note that vcpkg will use binary caching to store built dependency packages (usually at `~/.cache/vcpkg` or manually set with environment variable `VCPKG_DEFAULT_BINARY_CACHE`) so that upon reinstallation/building (re-running the script) you won't have to rebuild everything from scratch, unless the package itself has been updated, you are using a different vcpkg triplet, or any of the vcpkg scripts have changed (updated vcpkg repo). If you'd like to turn off binary caching (not recommended), then you can either pass `--no-binarycaching` to the build script after the main options listed in `--help` or add `-binarycaching` to the `VCPKG_FEATURE_FLAGS` environment variable. -**ATTENTION**: If you are having issues or want to start with a fresh installation directory, pass the `--clean` option to clear the installation directories and, if specified, the export directory. +**ATTENTION**: If you are having issues it is best to start fresh. Delete all of the created `vcpkg` directory. If you have binary caching on and nothing has changed, then you should be able to reuse your previously built dependencies. -### Export Directories (recommended) +### Export Directories -By default, vcpkg will install all of your dependencies to its own in-repo `installed` directory. Passing `--export-dir ` to the `./build_dependencies.sh` script, you can store the required dependencies in a separate directory. Otherwise, the built dependencies will be stored within the vcpkg repo directory itself. It is preferred to create a new export directory to keep track of different LLVM versions, since they cannot coexist within the same export (read: installation) directory. +By default, vcpkg will install all of your dependencies to its own in-repo `installed` directory. Passing `--export-dir ` to the `./build_dependencies.sh` script, you can store the required dependencies in a separate directory. Otherwise, the built dependencies will be stored within the vcpkg repo directory itself (`vcpkg/installed` relative path if in the root of this repo). If you are not pulling the repo multiple times for each LLVM install, then separate export directories are required to keep track of different LLVM versions, since they cannot coexist within the same export (read: installation) directory. + +NOTE: You cannot install new packages to the export directory. However, you can continue to install new packages without rebuilding to vcpkg's installation directory and re-export them. ```bash ./build_dependencies.sh --export-dir vcpkg-llvm-12-install llvm-12 @@ -57,21 +59,11 @@ By default, vcpkg will install all of your dependencies to its own in-repo `inst will build all of the dependencies listed in `dependencies.txt` _and_ LLVM 12 and install into a local directory named `vcpkg-llvm-10-install`. -#### Installing additional packages - -**NOTE** If you download the pre-built binaries, this does not apply. - -To install more packages to an existing vcpkg installation, just run the script without specifying any extra build configuration arguments, unless you have an export directory, and list the packages you'd like to install or add them to `dependencies.txt`: - -```bash -./build_dependencies.sh --export-dir vcpkg-llvm-12-install fmt -``` - -will install `fmt` into a local directory named `vcpkg-llvm-12-install` with whatever triplet is found within that vcpkg export directory. If multiple triplets are found in the export directory, the script will build `fmt` for each of those triplets. +It is important to remember that because different LLVM installations cannot coexist, you must either use a separate export directory or you should remove `vcpkg/installed` directory before installing another version. ### Updating -Updating dependencies is the same as installing them from source. Just run the script again (without any packages listed) and make sure to point it to your exported directory, if necessary. Do not specify anything regarding the triplet, like `--release` or `--asan`---the triplet(s) are detected automatically. +To ensure you update dependencies, it is best to completely remove the `vcpkg/installed` directory and re-run the build script. If you are more familiar with vcpkg, then you can use vcpkg's own commands to better manage your packages. Read the vcpkg documentation to learn more. ### Debug and Release Builds diff --git a/build_dependencies.sh b/build_dependencies.sh index df855e3c..5ec9878d 100755 --- a/build_dependencies.sh +++ b/build_dependencies.sh @@ -22,10 +22,6 @@ function Help echo " Build with ASAN triplet as detected in this script" echo " --export-dir " echo " Export built dependencies to directory path" - echo " --clean" - echo " Clean the installation directory and/or remove an existing export" - echo " directory if it exists. Use this or specify a new export directory" - echo " if you want to install different LLVM versions" echo " [...]" echo " Extra args to pass to 'vcpkg install'. Like LLVM version," echo " other ports, vcpkg-specific options, etc." @@ -34,7 +30,6 @@ function Help RELEASE="false" ASAN="false" EXPORT_DIR="" -CLEAN="false" VCPKG_ARGS=() while [[ $# -gt 0 ]] ; do key="$1" @@ -57,10 +52,6 @@ while [[ $# -gt 0 ]] ; do echo "[+] Exporting to directory ${EXPORT_DIR}" shift # past argument ;; - --clean) - CLEAN="true" - msg "Cleaning installation and specified export directory" - ;; *) VCPKG_ARGS+=("$1") esac @@ -164,47 +155,15 @@ vcpkg_dir="${repo_dir:?}/vcpkg" if [[ -z ${EXPORT_DIR} ]]; then # Set default export directory variable. Used for printing end message EXPORT_DIR="${vcpkg_dir}" -fi - -# Handle cleaning or additional package installation to our installation root -if [[ ${CLEAN} == "true" ]]; then - rm -rf "${EXPORT_DIR:?}/installed" || true - if [[ "${EXPORT_DIR}" != "${vcpkg_dir}" ]]; then - rm -rf "${EXPORT_DIR}" || true - fi else - # Install additional packages to an existing installation directory - if [ -d "${EXPORT_DIR}" ]; then - extra_vcpkg_args+=("--x-install-root=${EXPORT_DIR}/installed") + if [[ -d "${EXPORT_DIR}" ]]; then + die "Export directory already exists, please delete: '${EXPORT_DIR}'" fi fi +extra_vcpkg_args+=("--triplet=${triplet}" "--host-triplet=${triplet}") -# Check if triplet was not user-specified on CLI -triplets_array=() -if echo "${VCPKG_ARGS[@]}" | grep -w -v -q -- '--triplet=' ; then - # Check if triplet exists in export directory - if [ -d "${EXPORT_DIR}/installed" ]; then - for f in "${EXPORT_DIR}/installed/"* ; do - dirname="${f##*/}" - if [ "$dirname" != "vcpkg" ]; then - triplets_array+=("$dirname") - fi - done - if [ ${#triplets_array[@]} -eq 1 ]; then - triplet="${triplets_array[0]}" - msg "Detected triplet '${triplet}' in export directory" - elif [ ${#triplets_array[@]} -gt 1 ]; then - msg "Detected more than one triplet in export directory: '${triplets_array[*]}'" - fi - fi -fi -# Only have one triplet -if [ ${#triplets_array[@]} -le 1 ]; then - extra_vcpkg_args+=("--triplet=${triplet}") -fi - -extra_cmake_usage_args+=("-DVCPKG_TARGET_TRIPLET=${triplet}") +extra_cmake_usage_args+=("-DVCPKG_TARGET_TRIPLET=${triplet}" "-DVCPKG_HOST_TRIPLET=${triplet}") repo_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" vcpkg_info_file="${repo_dir}/vcpkg_info.txt" @@ -246,24 +205,11 @@ msg " " "${VCPKG_ARGS[@]}" ( set -x - multi_triplet_arg= - i=0 - if [ ${#triplets_array[@]} -gt 1 ]; then - multi_triplet_arg="--triplet=${triplets_array[$i]}" - fi - # Increment regardless for cases where installing into existing export dir - i=$((i+1)) - - # Install packages for all triplets - while : ; do - "${vcpkg_dir}/vcpkg" install "${multi_triplet_arg}" "${extra_vcpkg_args[@]}" '@overlays.txt' '@dependencies.txt' "${VCPKG_ARGS[@]}" + "${vcpkg_dir}/vcpkg" install "${extra_vcpkg_args[@]}" '@overlays.txt' '@dependencies.txt' "${VCPKG_ARGS[@]}" - [ ${i} -lt ${#triplets_array[@]} ] || break - multi_triplet_arg="--triplet=${triplets_array[$i]}" - i=$((i+1)) - done - - "${vcpkg_dir}/vcpkg" upgrade "${extra_vcpkg_args[@]}" '@overlays.txt' --no-dry-run + # This forces everyone to be updated but maybe this is too much trouble to + # actually rebuild everything that has a mismatching hash. + # "${vcpkg_dir}/vcpkg" upgrade "${extra_vcpkg_args[@]}" '@overlays.txt' --no-dry-run find "${vcpkg_dir}"/installed/*/tools/protobuf/ -type f -exec chmod 755 {} + || true find "${EXPORT_DIR}"/installed/*/tools/protobuf/ -type f -exec chmod 755 {} + || true @@ -273,12 +219,10 @@ msg " " "${VCPKG_ARGS[@]}" # Don't export if we've already installed to an existing EXPORT_DIR if [[ ! -d "${EXPORT_DIR}" ]]; then tmp_export_dir=temp-export - "${vcpkg_dir}/vcpkg" export "${extra_vcpkg_args[@]}" '@overlays.txt' --raw --output=${tmp_export_dir} '@dependencies.txt' "${VCPKG_ARGS[@]}" - extra_mv_arg=() - if [[ ${CLEAN} == "true" ]]; then - extra_mv_arg+=("-f") - fi - mv "${extra_mv_arg[@]}" "${vcpkg_dir}/${tmp_export_dir}" "${EXPORT_DIR}" + set -x + "${vcpkg_dir}/vcpkg" export --x-all-installed --raw "--output=${tmp_export_dir}" + mv "${vcpkg_dir}/${tmp_export_dir}" "${EXPORT_DIR}" + set +x fi echo ""