ubuntu-bartender

#!/bin/bash -e

############ The Ubuntu Bartender #############################################

# Sometimes you just want an Ubuntu Old Fashioned [1], but you don't want to
# make it yourself.

# Sometimes you just want someone else to gather the ingredients, make the
# drink for you, and clean up afterwards.

# So sit back, relax, and let the Ubuntu Bartender do the work for you.

# 1: https://github.com/ubuntu-bartenders/ubuntu-old-fashioned

############ Overview #########################################################

# This script will collect all the required bits Ubuntu Old Fashioned
# needs to build Ubuntu images, build those images for you, then clean up
# afterwards.

# A simple bionic build can be accomplished with the incantation:

# ./ubuntu-bartender --livecd-rootfs-branch ubuntu/bionic -- --series bionic

# Arguments specified after the -- are used by Ubuntu Old Fashioned directly.

# The Ubuntu Bartender can also be invoked with series specific helper script
# that can save you from needing to manually specify the correct livecd_rootfs
# branch and series.

# For example, this is identical to the invocation above:

# ./ubuntu-bionic-bartender

############ Dependencies #####################################################

# The Ubuntu Bartender requires the following executables to be in the PATH:

dependencies="petname git"

# Which we verify before moving on:

for dependency in $dependencies
do
  if ! command -v $dependency &>/dev/null
  then
    echo "error: $dependency was not found in PATH" >&2
    exit 255
  fi
done

############ Configuration ####################################################

# The Ubuntu Bartender needs to know the location of a few git repositories:

UBUNTU_OLD_FASHIONED_REPO=${UBUNTU_OLD_FASHIONED_REPO:-https://github.com/ubuntu-bartenders/ubuntu-old-fashioned.git}
LIVECD_ROOTFS_REPO=${LIVECD_ROOTFS_REPO:-https://git.launchpad.net/livecd-rootfs}
HOOK_EXTRAS_REPO=${HOOK_EXTRAS_REPO:-}
HOOK_EXTRAS_DIR=${HOOK_EXTRAS_DIR:-}
HOOK_EXTRAS_RELEASE_NOTES_TOOLS_REPO=${HOOK_EXTRAS_RELEASE_NOTES_TOOLS_REPO:-}
HOOK_EXTRAS_SBOM_TOOLS_REPO=${HOOK_EXTRAS_SBOM_TOOLS_REPO:-}

# And any specific branches for each that should be used:

UBUNTU_OLD_FASHIONED_BRANCH=${UBUNTU_OLD_FASHIONED_BRANCH:-master}
HOOK_EXTRAS_RELEASE_NOTES_TOOLS_BRANCH=${HOOK_EXTRAS_RELEASE_NOTES_TOOLS_BRANCH:-}
HOOK_EXTRAS_SBOM_TOOLS_BRANCH=${HOOK_EXTRAS_SBOM_TOOLS_BRANCH:-}
HOOK_EXTRAS_SBOM_TOOLS_DIR=${HOOK_EXTRAS_SBOM_TOOLS_DIR:-}

# LIVECD_ROOTFS_BRANCH is inferred below from the script name
HOOK_EXTRAS_BRANCH=${HOOK_EXTRAS_BRANCH:-}

# The Ubuntu Bartender utilizes a build-provider to create Ubuntu Images.
# We use multipass by default:

BUILD_PROVIDER=${BUILD_PROVIDER:-multipass}

# Should we tear down the build provider and clean up after ourselves
# when the build is completed?

SHOULD_CLEANUP=${SHOULD_CLEANUP:-YES}

# Where to build the ingredients.tar.gz file
TEMP_DIR_LOC=$HOME

# Parse the name of the script to see if we can infer some variables

maybe_series=$(basename $0 | cut -d'-' -f2)
if [ "$maybe_series" != "bartender" ]
then
  series_flag="--series $maybe_series"
  LIVECD_ROOTFS_BRANCH=${LIVECD_ROOTFS_BRANCH:-ubuntu/$maybe_series}
else
  series_flag=""
  LIVECD_ROOTFS_BRANCH=${LIVECD_ROOTFS_BRANCH:-ubuntu/master}
fi


############ Provider Configuration ##########################################

# Providers also have some config
# This sets default values, and allows for inclusion in CLI and file
# Please preface each provider default with the provider name

# AWS SPECIFIC DEFAULTS
AWS_AMI_NAME_FILTER="ubuntu/images-testing/hvm-ssd/ubuntu-bionic-daily-amd64-server-*"
AWS_PROFILE="default"
AWS_INSTANCE_TYPE="m5.large"
# Official Canonical OwnerId
AWS_AMI_OWNER=099720109477
AWS_KEYPAIR_NAME=$USER
AWS_ARM_BUILD=false

# MULTIPASS SPECIFIC DEFAULTS
MULTIPASS_DISK_SIZE=50G
MULTIPASS_IMAGE=daily:b # daily image of bionic
# you can optionally override memory cap by setting MULTIPASS_MEM_SIZE

# By default, do not exceed half the total memory or 8GiB (whichever is smaller).
# All calculations are in MiB
TOTAL_MEM=$(free -m | grep -oP '\d+' | head -n 1)
HALF_MEM="$(( $TOTAL_MEM / 2 ))"
CAP_MEM=8192

if [[ "$HALF_MEM" -lt "$CAP_MEM" ]]; then
    MULTIPASS_MEM_SIZE_DEFAULT="${HALF_MEM}M"
else
    MULTIPASS_MEM_SIZE_DEFAULT="${CAP_MEM}M"
fi

MULTIPASS_MEM_SIZE=${MULTIPASS_MEM_SIZE:-$MULTIPASS_MEM_SIZE_DEFAULT}

# AZURE SPECIFIC DEFAULTS
AZURE_URN="Canonical:UbuntuServer:18_04-daily-lts-gen2:latest"
AZURE_INSTANCE_SIZE="Standard_F8s_v2"
AZURE_LOCATION="France Central"

# GCE SPECIFIC DEFAULTS
GCE_ARM_BUILD=false
GCE_IMAGE_FAMILY="ubuntu-1804-lts"
GCE_MACHINE_TYPE="n2-standard-2"
GCE_ZONE="us-central1-a" # Low CO2 region with both ARM64 and AMD64 instances available

# We also pull in explicitly set variables on the command line

function print-usage {
  cat << EOF >&2
usage: $0 [<options>] -- --series <series> [<ubuntu-old-fashioned-options>]

    --livecd-rootfs-repo <url>              The url to the git repo hosting livecd-rootfs.
                                            Value: $LIVECD_ROOTFS_REPO

    --livecd-rootfs-branch <branch>         The branch of livecd-rootfs to use.
                                            Value: $LIVECD_ROOTFS_BRANCH

    --livecd-rootfs-dir <dir>               A local directory containing livecd-rootfs.
                                            Value: ${LIVECD_ROOTFS_DIR:-None}

    --ubuntu-old-fashioned-repo <url>       The url to the git repo hosting ubuntu-old-fashioned.
                                            Value: $UBUNTU_OLD_FASHIONED_REPO

    --ubuntu-old-fashioned-branch <branch>  The branch of ubuntu-old-fashioned to use.
                                            Value: $UBUNTU_OLD_FASHIONED_BRANCH

    --ubuntu-old-fashioned-dir <dir>        A local directory containing ubuntu-old-fashioned.
                                            Value: ${UBUNTU_OLD_FASHIONED_DIR:-None}

    --hook-extras-repo <url>                The url to a git repo hosting extra hooks.
                                            Value: ${HOOK_EXTRAS_REPO:-None}

    --hook-extras-branch <branch>           The branch of the extras-repo to use.
                                            Value: ${HOOK_EXTRAS_BRANCH:-None}

    --hook-extras-release-notes-tools-repo <url> The url to a git repo hosting release notes tools.
                                            Value: ${HOOK_EXTRAS_RELEASE_NOTES_TOOLS_REPO:-None}

    --hook-extras-release-notes-tools-branch <branch> The branch of the release-notes-tools to use.
                                            Value: ${HOOK_EXTRAS_RELEASE_NOTES_TOOLS_BRANCH:-None}

    --hook-extras-sbom-tools-repo <url>     The url to a git repo hosting sbom tools.
                                            Value: ${HOOK_EXTRAS_SBOM_TOOLS_REPO:-None}

    --hook-extras-sbom-tools-branch <branch> The branch of the sbom-tools to use.
                                            Value: ${HOOK_EXTRAS_SBOM_TOOLS_BRANCH:-None}

    --hook-extras-sbom-tools-dir <dir>      The directory in the sbom-tools to copy in to hooks.d/extra directories.
                                            This is useful as you don't always want to copy the entire sbom-tools repo.
                                            Value: ${HOOK_EXTRAS_SBOM_TOOLS_DIR:-None}

    --hook-extras-dir <dir>                 A local directory containing extra hooks.
                                            Value: ${HOOK_EXTRAS_DIR:-None}

    --chroot-archive <archive>              A local archive used to perform the build.
                                            Value: ${CHROOT_ARCHIVE:-None}

    --build-provider <provider>             The provider used to build the image.
                                            This can be multipass, aws, azure or gce.
                                            Value: $BUILD_PROVIDER

    --no-cleanup                            Don't tear down the build provider
                                            after the build completes.
                                            Value: Cleanup? $SHOULD_CLEANUP

    --aws-ami-name-filter <image-name>      Filter for finding an AWS ami by name
                                            defaults to Bionic AMD64. This is
                                            the base for the builds. for ARM:
                                            ubuntu/images-testing/hvm-ssd/ubuntu-bionic-daily-arm64-server-*
                                            Value: $AWS_AMI_NAME_FILTER

    --aws-profile <profile>                 AWS profile used with the aws cli
                                            Value: $AWS_PROFILE

    --aws-ami-owner <owner_id>              AWS OwnerId, user in conjunction with
                                            --aws-ami-filter-name to find a base image
                                            for building. Defaults to Canonical
                                            Value: $AWS_AMI_OWNER

    --aws-keypair-name                      Keypair name associated with AWS profile;
                                            local private key must be a .pem
                                            with chmod 400 in ~/.ssh/
                                            Default is $USER       
                                            Value: $AWS_KEYPAIR_NAME

    --aws-instance-type                     EC2 instance type. default is m5.large.
                                            m6g.large is a good choice for arm
                                            Value: $AWS_INSTANCE_TYPE

    --aws-arm-build                         Short hand switch for safe arm64 defaults
                                            This will override --aws-instance-type and --aws-ami-name-filter

    --azure-urn <urn>                       Image URN
                                            Value: $AZURE_URN

    --azure-instance-size <size>            Azure instance size
                                            Value: $AZURE_INSTANCE_SIZE

    --azure-location <location>             Location (region) where to create the instance
                                            on Azure
                                            Value: $AZURE_LOCATION

    --gce-machine-type <machine-type>       GCE instance machine type, defaults to n2-standard-2
                                            Value: $GCE_MACHINE_TYPE

    --gce-zone <zone>                       Zone in which to create the GCE instance,
                                            defaults to us-central1-a
                                            Value: $GCE_ZONE

    --gce-family <image-family>             Defaults to bionic amd64 (ubuntu-1804-lts).
                                            Value: $GCE_IMAGE_FAMILY

    --gce-arm-build                         Short hand switch for safe arm64 defaults in GCE
                                            (This overrides both --gce-machine-type *and* --gce-image-family)

    --temp-dir-loc                          Parent folder to build the temp dir used
                                            by bartender; note that this must be
                                            a directory that multipass (a strictly-
                                            confined snap) can access
                                            Value: $TEMP_DIR_LOC
    --help                                  Print this help message.

Options specified after '--' separator are used by Ubuntu Old Fashioned directly.

EOF
}

# These can be added to a config file as well
# Check/etc and user home. 
# user home takes precedence over etc
# Order is important as well, cli args win
POSSIBLE_CONF_FILES=("/etc/ubuntu-bartender/ubuntu-bartender.rc" ~/".ubuntu-bartender.rc")
for file in "${POSSIBLE_CONF_FILES[@]}"
do
  if [[ -f "${file}" ]]; then
    echo "Sourcing configuration file: $file"
    source $file
  fi
done

# Which we parse with the typical boilerplate:

while [[ $# -gt 0 ]]
do
  case $1 in
    --livecd-rootfs-repo)
      LIVECD_ROOTFS_REPO="$2"
      shift
      ;;
    --livecd-rootfs-branch)
      LIVECD_ROOTFS_BRANCH="$2"
      shift
      ;;
    --livecd-rootfs-dir)
      LIVECD_ROOTFS_DIR="$2"
      shift
      ;;
    --ubuntu-old-fashioned-repo)
      UBUNTU_OLD_FASHIONED_REPO="$2"
      shift
      ;;
    --ubuntu-old-fashioned-branch)
      UBUNTU_OLD_FASHIONED_BRANCH="$2"
      shift
      ;;
    --ubuntu-old-fashioned-dir)
      UBUNTU_OLD_FASHIONED_DIR="$2"
      shift
      ;;
    --hook-extras-repo)
      HOOK_EXTRAS_REPO="$2"
      shift
      ;;
    --hook-extras-branch)
      HOOK_EXTRAS_BRANCH="$2"
      shift
      ;;
    --hook-extras-dir)
      HOOK_EXTRAS_DIR="$2"
      shift
      ;;
    --hook-extras-release-notes-tools-repo)
      HOOK_EXTRAS_RELEASE_NOTES_TOOLS_REPO="$2"
      shift
      ;;
    --hook-extras-release-notes-tools-branch)
      HOOK_EXTRAS_RELEASE_NOTES_TOOLS_BRANCH="$2"
      shift
      ;;
    --hook-extras-sbom-tools-repo)
      HOOK_EXTRAS_SBOM_TOOLS_REPO="$2"
      shift
      ;;
    --hook-extras-sbom-tools-branch)
      HOOK_EXTRAS_SBOM_TOOLS_BRANCH="$2"
      shift
      ;;
    --hook-extras-sbom-tools-dir)
      HOOK_EXTRAS_SBOM_TOOLS_DIR="$2"
      shift
      ;;
    --chroot-archive)
      CHROOT_ARCHIVE="$2"
      shift
      ;;
    --build-provider)
      BUILD_PROVIDER="$2"
      shift
      ;;
    --no-cleanup)
      SHOULD_CLEANUP=NO
      ;;
    --aws-ami-name-filter)
      AWS_AMI_NAME_FILTER="$2"
      shift
      ;;
    --aws-profile)
      AWS_PROFILE="$2"
      shift
      ;;
    --aws-ami-owner)
      AWS_AMI_OWNER="$2"
      shift
      ;;
    --aws-keypair-name)
      AWS_KEYPAIR_NAME="$2"
      shift
      ;;
    --aws-instance-type)
      AWS_INSTANCE_TYPE="$2"
      shift
      ;;
    --aws-arm-build)
      AWS_ARM_BUILD=true
      ;;
    --azure-urn)
      AZURE_URN="$2"
      shift
      ;;
    --azure-instance-size)
      AZURE_INSTANCE_SIZE="$2"
      shift
      ;;
    --azure-location)
      AZURE_LOCATION="$2"
      shift
      ;;
    --gce-machine-type)
      GCE_MACHINE_TYPE="$2"
      shift
      ;;
    --gce-zone)
      GCE_ZONE="$2"
      shift
      ;;
    --gce-family)
      GCE_IMAGE_FAMILY="$2"
      shift
      ;;
    --gce-arm-build)
      GCE_ARM_BUILD=true
      ;;
    --temp-dir-loc)
      TEMP_DIR_LOC="$2"
      shift
      ;;
    --)
      shift
      break
      ;;
    --help)
      SHOW_HELP=YES
      ;;
    *)
      echo "error: unknown option '$1'" >&2
      exit 255
      ;;
  esac
  shift
done

if [ -n "$HOOK_EXTRAS_REPO" ] && [ -n "$HOOK_EXTRAS_DIR" ]
then
  echo "error: cannot specify both --hook-extras-repo and --hook-extras-dir" >&2
  print-usage
  exit 255
fi

if [ "$SHOW_HELP" = "YES" ]
then
  print-usage
  exit 255
fi

# if --aws-arm-build provided
# setup a default aws arm build
# remember -- you can always specific all your own info
if [ "$AWS_ARM_BUILD" = "true" ]
then
  AWS_INSTANCE_TYPE="m6g.large"
  AWS_AMI_NAME_FILTER="ubuntu/images-testing/hvm-ssd/ubuntu-bionic-daily-arm64-server-*"
  BUILD_PROVIDER="aws"
fi

# if --gce-arm-build is provided,
# setup the new GCE defaults.
if [ "$GCE_ARM_BUILD" = "true" ]
then
  GCE_IMAGE_FAMILY="ubuntu-1804-lts-arm64"
  GCE_MACHINE_TYPE="t2a-standard-2"
  BUILD_PROVIDER="gce"
fi


# Verify that the specified build provider is available and ready

build_provider_config="$(dirname "$(readlink -f "$0")")/$BUILD_PROVIDER-provider"
if [ ! -f "$build_provider_config" ]
then
  echo "error: invalid build provider specified: $BUILD_PROVIDER" >&2
  print-usage
  exit 255
fi

source "$build_provider_config"

build-provider-assert-ready

# The --series flag is required by ubuntu-old-fashioned for image builds,
# so we check to make sure we have it before moving on:

if [ -z "$series_flag" ] && ! echo -- $@ | grep -q -- --series
then
  echo "error: the '--series' option is required by ubuntu-old-fashioned" >&2
  print-usage
  exit 255
fi

############ Where the Magic Happens ##########################################

# Let's build all the things!

temp_dir=$(mktemp --tmpdir="$TEMP_DIR_LOC" --directory ubuntu-bartender-multipass.XXXXXXXXXX)
bartender_name=$(petname)-ubuntu-bartender
drink_name=${bartender_name/-bartender/-on-the-rocks.tar.gz}

function cleanup {
  echo "Cleaning up..."
  rm -rf $temp_dir
  build-provider-destroy $bartender_name
}

if [ "$SHOULD_CLEANUP" = "YES" ]
then
  trap cleanup EXIT
fi

build-provider-create $bartender_name

(
  echo "Preparing ingredients... in $temp_dir"

  if [ -n "$HOOK_EXTRAS_DIR" ]
  then
    cp -aR "$HOOK_EXTRAS_DIR" $temp_dir/extras
  fi

  if [ -n "$LIVECD_ROOTFS_DIR" ]
  then
    cp -aR "$LIVECD_ROOTFS_DIR" $temp_dir/livecd-rootfs
  fi

  if [ -n "$UBUNTU_OLD_FASHIONED_DIR" ]
  then
    cp -aR "$UBUNTU_OLD_FASHIONED_DIR" $temp_dir/ubuntu-old-fashioned
  fi

  if [ -n "$CHROOT_ARCHIVE" ]
  then
    cp -aR "$CHROOT_ARCHIVE" $temp_dir/chroot-archive
    chroot_archive_flag="--use-chroot-archive ../chroot-archive"
  fi

  cd $temp_dir

  if [ -z "$UBUNTU_OLD_FASHIONED_DIR" ]
  then
    git clone -qb $UBUNTU_OLD_FASHIONED_BRANCH $UBUNTU_OLD_FASHIONED_REPO
  fi

  if [ -z "$LIVECD_ROOTFS_DIR" ]
  then
    git clone -qb $LIVECD_ROOTFS_BRANCH $LIVECD_ROOTFS_REPO
  fi

  if [ -n "$HOOK_EXTRAS_REPO" ]
  then
    branch_flag=" "
    if [ -n "$HOOK_EXTRAS_BRANCH" ]
    then
      branch_flag="-b $HOOK_EXTRAS_BRANCH"
    fi
    git clone -q $branch_flag $HOOK_EXTRAS_REPO extras
  fi

  if [ -d $temp_dir/extras ]
  then
    find livecd-rootfs/live-build/ -type d -name '*hooks*' |
      xargs -I {} cp -af extras/* {}
  fi

  if [ -n "$HOOK_EXTRAS_RELEASE_NOTES_TOOLS_REPO" ]
  then
    branch_flag=" "
    if [ -n "$HOOK_EXTRAS_RELEASE_NOTES_TOOLS_BRANCH" ]
    then
      branch_flag="-b $HOOK_EXTRAS_RELEASE_NOTES_TOOLS_BRANCH"
    fi
    git clone -q $branch_flag $HOOK_EXTRAS_RELEASE_NOTES_TOOLS_REPO release-notes-tools
  fi

  if [ -d $temp_dir/release-notes-tools ]
  then
    # copy the release notes tools to all the hooks directories
    find livecd-rootfs/live-build/ -type d -name '*hooks*' |
      xargs -I {} sh -c "mkdir --parents {}/extra/release-notes-tools && cp --archive --force $temp_dir/release-notes-tools/* {}/extra/release-notes-tools/"
  fi

  if [ -n "$HOOK_EXTRAS_SBOM_TOOLS_REPO" ]
  then
    branch_flag=" "
    if [ -n "$HOOK_EXTRAS_SBOM_TOOLS_BRANCH" ]
    then
      branch_flag="-b $HOOK_EXTRAS_SBOM_TOOLS_BRANCH"
    fi
    git clone -q $branch_flag $HOOK_EXTRAS_SBOM_TOOLS_REPO sbom-tools
  fi

  if [ -d $temp_dir/sbom-tools ]
  then
    # copy the sbom tools to all the hooks directories
    find livecd-rootfs/live-build/ -type d -name '*hooks*' |
      xargs -I {} sh -c "mkdir --parents {}/extra/sbom-tools && cp --archive --force $temp_dir/sbom-tools/${HOOK_EXTRAS_SBOM_TOOLS_DIR}/* {}/extra/sbom-tools/"
  fi

  cat > mix-old-fashioned << EOF
#!/bin/bash -x
sudo add-apt-repository -y -u ppa:canonical-is-sa/ubuntu/buildd
sudo apt-get -q update
sudo apt-get -q install -y launchpad-buildd bzr git python3-ubuntutools python3-launchpadlib
cd livecd-rootfs
sudo -E ../ubuntu-old-fashioned/old-fashioned-image-build --no-cleanup $chroot_archive_flag $series_flag $@
EOF

  tar czf ingredients.tar.gz --exclude-vcs ./*
  build-provider-upload $bartender_name ingredients.tar.gz ingredients.tar.gz
  build-provider-run $bartender_name -- tar xf ingredients.tar.gz
)

echo "Mixing drink..."
echo "(See progress in $bartender_name.log)"
build-provider-run $bartender_name -- bash mix-old-fashioned &>$bartender_name.log </dev/null

echo "Pouring $drink_name..."
build-provider-run $bartender_name -- tar czf drink.tar.gz build.output
build-provider-download $bartender_name drink.tar.gz $drink_name