From 89a9aa5e160d8d243064f2ca71f06c842a766023 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Fri, 1 Apr 2022 15:52:14 -0400
Subject: [PATCH 01/14] Add a script for installing all the software mentioned
 in the README

---
 tools/install_prerequisites.sh | 44 ++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100755 tools/install_prerequisites.sh

diff --git a/tools/install_prerequisites.sh b/tools/install_prerequisites.sh
new file mode 100755
index 00000000000..05dd34100ff
--- /dev/null
+++ b/tools/install_prerequisites.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -eu
+
+# Set the CWD to Omicron's source.
+SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+cd "${SOURCE_DIR}/.."
+
+HOST_OS=$(uname -s)
+if [[ "${HOST_OS}" == "Linux" ]]; then
+  echo "Linux"
+  sudo apt-get install libpq-dev
+  sudo apt-get install pkg-config
+elif [[ "${HOST_OS}" == "SunOS" ]]; then
+  echo "illumos"
+  need=(
+    'build-essential'
+    'library/postgresql-13'
+    'pkg-config'
+    'brand/omicron1/tools'
+    'pkg:/package/pkg'
+  )
+  missing=()
+  for (( i = 0; i < ${#need[@]}; i++ )); do
+    p=${need[$i]}
+    if ! pkg info -q "$p"; then
+      missing+=( "$p" )
+    fi
+  done
+  if (( ${#missing[@]} > 0 )); then
+    pfexec pkg install -v "${missing[@]}"
+  fi
+  pkg list -v "${need[@]}"
+elif [[ "${HOST_OS}" == "Darwin" ]]; then
+  echo "Mac"
+  brew install postgresql
+  brew install pkg-config
+else
+  echo "Unsupported OS"
+  exit -1
+fi
+
+./tools/ci_download_cockroachdb
+./tools/ci_download_clickhouse

From 147c97c2df680a035775950b54ca452a3b30076f Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Fri, 1 Apr 2022 16:00:14 -0400
Subject: [PATCH 02/14] Update Readme

---
 README.adoc | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/README.adoc b/README.adoc
index ef6e5d40f65..c63d6043171 100644
--- a/README.adoc
+++ b/README.adoc
@@ -33,7 +33,15 @@ You can **format the code** using `cargo fmt`.  Make sure to run this before pus
 
 You can **run the https://github.com/rust-lang/rust-clippy[Clippy linter]** using `cargo clippy \-- -D warnings -A clippy::style`.  Make sure to run this before pushing changes.  The CI checks that the code is clippy-clean.
 
-**Prerequisites:**
+=== Installing Prerequisite Software
+
+The following software may be installed automatically with the following scripts:
+
+----
+$ ./tools/install_prerequisites.sh
+----
+
+Alternatively, the manual installation steps as follows:
 
 Both normal execution and the test suite expect certain binaries (described below) on your PATH.
 
@@ -83,6 +91,10 @@ pkg install pkg:/package/pkg
 pkg update
 ----
 
+=== Running (Simulated) Omicron
+
+NOTE: If you'd like to run on Helios, refer to <<Deploying Omicron>> below.
+
 To **run Omicron** you need to run four programs:
 
 * a CockroachDB cluster.  For development, you can use the `omicron-dev` tool in this repository to start a single-node CockroachDB cluster **that will delete the database when you shut it down.**

From 0e5d9846f1f2e29d3d5e60c6c3551ea5a2c7bb35 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Sat, 2 Apr 2022 22:28:12 -0400
Subject: [PATCH 03/14] Merge update and install

---
 tools/install_prerequisites.sh | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/tools/install_prerequisites.sh b/tools/install_prerequisites.sh
index 05dd34100ff..390094b4ef5 100755
--- a/tools/install_prerequisites.sh
+++ b/tools/install_prerequisites.sh
@@ -8,35 +8,34 @@ cd "${SOURCE_DIR}/.."
 
 HOST_OS=$(uname -s)
 if [[ "${HOST_OS}" == "Linux" ]]; then
-  echo "Linux"
   sudo apt-get install libpq-dev
   sudo apt-get install pkg-config
 elif [[ "${HOST_OS}" == "SunOS" ]]; then
-  echo "illumos"
   need=(
+    'pkg:/package/pkg'
     'build-essential'
     'library/postgresql-13'
     'pkg-config'
     'brand/omicron1/tools'
-    'pkg:/package/pkg'
   )
-  missing=()
-  for (( i = 0; i < ${#need[@]}; i++ )); do
-    p=${need[$i]}
-    if ! pkg info -q "$p"; then
-      missing+=( "$p" )
+
+  # Perform updates
+  if (( ${#need[@]} > 0 )); then
+    pfexec pkg install -v "${need[@]}" && rc=$? || rc=$?
+    # Return codes:
+    #  0: Normal Success
+    #  4: Failure because we're already up-to-date. Also acceptable.
+    if [ "$rc" -ne 4 ] && [ "$rc" -ne 0 ]; then
+      exit "$rc"
     fi
-  done
-  if (( ${#missing[@]} > 0 )); then
-    pfexec pkg install -v "${missing[@]}"
   fi
+
   pkg list -v "${need[@]}"
 elif [[ "${HOST_OS}" == "Darwin" ]]; then
-  echo "Mac"
   brew install postgresql
   brew install pkg-config
 else
-  echo "Unsupported OS"
+  echo "Unsupported OS: ${HOST_OS}"
   exit -1
 fi
 

From 6c01f56ad30a180602f11fca256a4c330428a405 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Sat, 2 Apr 2022 22:30:42 -0400
Subject: [PATCH 04/14] Make DB downloads idempotent, contained in 'out/'
 directory

---
 README.adoc                   |   4 +-
 package-manifest.toml         |   4 +-
 tools/ci_download_clickhouse  | 168 ++++++++++++++++------------
 tools/ci_download_cockroachdb | 201 +++++++++++++++++++---------------
 4 files changed, 212 insertions(+), 165 deletions(-)

diff --git a/README.adoc b/README.adoc
index c63d6043171..8f707328345 100644
--- a/README.adoc
+++ b/README.adoc
@@ -70,13 +70,13 @@ example, on Helios, you'd want `/usr/bin` on your PATH.
 . CockroachDB v21.1.10.
 +
 The build and test suite expects to be able to start a single-node CockroachDB cluster using the `cockroach` executable on your PATH.
-On illumos, MacOS, and Linux, you should be able to use the `tools/ci_download_cockroachdb` script to fetch the official CockroachDB binary.  It will be put into `./cockroachdb/bin/cockroach`.
+On illumos, MacOS, and Linux, you should be able to use the `tools/ci_download_cockroachdb` script to fetch the official CockroachDB binary.  It will be put into `./out/cockroachdb/bin/cockroach`.
 Alternatively, you can follow the https://www.cockroachlabs.com/docs/stable/install-cockroachdb.html[official CockroachDB installation instructions for your platform].
 
 . ClickHouse >= v21.7.1.
 +
 The test suite expects a running instance of the ClickHouse database server.
-The script `./tools/ci_download_clickhouse` can be used to download a pre-built binary for illumos, Linux, or macOS platforms. Once complete, you must manually add the binary (located at `clickhouse/clickhouse`) to your PATH.
+The script `./tools/ci_download_clickhouse` can be used to download a pre-built binary for illumos, Linux, or macOS platforms. Once complete, you must manually add the binary (located at `./out/clickhouse/clickhouse`) to your PATH.
 You may also install ClickHouse manually; instructions can be found https://clickhouse.tech/docs/en/getting-started/install[here].
 See <<_configuring_clickhouse>> for details on ClickHouse's setup and configuration files.
 +
diff --git a/package-manifest.toml b/package-manifest.toml
index 460b9f11a5f..03fd1256f77 100644
--- a/package-manifest.toml
+++ b/package-manifest.toml
@@ -36,7 +36,7 @@ to = "/var/svc/manifest/site/oximeter"
 service_name = "clickhouse"
 zone = true
 [[package.clickhouse.paths]]
-from = "clickhouse"
+from = "out/clickhouse"
 to = "/opt/oxide/clickhouse"
 [[package.clickhouse.paths]]
 from = "smf/clickhouse"
@@ -46,7 +46,7 @@ to = "/var/svc/manifest/site/clickhouse"
 service_name = "cockroachdb"
 zone = true
 [[package.cockroachdb.paths]]
-from = "cockroachdb"
+from = "out/cockroachdb"
 to = "/opt/oxide/cockroachdb"
 [[package.cockroachdb.paths]]
 from = "common/src/sql"
diff --git a/tools/ci_download_clickhouse b/tools/ci_download_clickhouse
index c1099216941..3bdac422115 100755
--- a/tools/ci_download_clickhouse
+++ b/tools/ci_download_clickhouse
@@ -7,112 +7,136 @@
 #
 
 set -o pipefail
-set -o xtrace
 set -o errexit
 
 SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 ARG0="$(basename ${BASH_SOURCE[0]})"
 
+TARGET_DIR="out"
+# Location where intermediate artifacts are downloaded / unpacked.
+DOWNLOAD_DIR="$TARGET_DIR/downloads"
+# Location where the final clickhouse directory should end up.
+DEST_DIR="./$TARGET_DIR/clickhouse"
+
 # If you change this, you must also update the md5sums below
 CIDL_VERSION="v21.7"
 source "$SOURCE_DIR/clickhouse_checksums"
-CIDL_ASSEMBLE_DIR="./clickhouse"
 
 # Download from manually-populated S3 bucket for now
 CIDL_URL_BASE="https://oxide-clickhouse-build.s3.us-west-2.amazonaws.com"
 
 function main
 {
-	#
-	# Process command-line arguments.  We generally don't expect any, but
-	# we allow callers to specify a value to override OSTYPE, just for
-	# testing.
-	#
-	if [[ $# != 0 ]]; then
-		CIDL_OS="$1"
-		shift
-	else
-		CIDL_OS="$OSTYPE"
-	fi
-
-	if [[ $# != 0 ]]; then
-		echo "unexpected arguments" >&2
-		exit 2
-	fi
-
-	# Configure this program
-	configure_os "$CIDL_OS"
-	CIDL_URL="$CIDL_URL_BASE/$CIDL_FILE"
-
-	# Download the file.
-	echo "URL: $CIDL_URL"
-	echo "Local file: $CIDL_FILE"
-	do_download_curl "$CIDL_URL" "$CIDL_FILE" || \
-	    fail "failed to download file"
-
-	# Verify the md5sum.
-	calculated_md5="$($CIDL_MD5FUNC "$CIDL_FILE")" || \
-	    fail "failed to calculate md5sum"
-	if [[ "$calculated_md5" != "$CIDL_MD5" ]]; then
-		fail "md5sum mismatch \
-		    (expected $CIDL_MD5, found $calculated_md5)"
-	fi
-
-	# Unpack the tarball into a local directory
-	do_untar "$CIDL_FILE" "$CIDL_ASSEMBLE_DIR"
-
-	# Run the binary as a sanity-check.
-	"$CIDL_ASSEMBLE_DIR/clickhouse" server --version
+  #
+  # Process command-line arguments.  We generally don't expect any, but
+  # we allow callers to specify a value to override OSTYPE, just for
+  # testing.
+  #
+  if [[ $# != 0 ]]; then
+    CIDL_OS="$1"
+    shift
+  else
+    CIDL_OS="$OSTYPE"
+  fi
+
+  if [[ $# != 0 ]]; then
+    echo "unexpected arguments" >&2
+    exit 2
+  fi
+
+  # Configure this program
+  configure_os "$CIDL_OS"
+  CIDL_URL="$CIDL_URL_BASE/$TARBALL_FILENAME"
+
+  # Download the file.
+  echo "URL: $CIDL_URL"
+  echo "Local file: $TARBALL_FILE"
+
+  mkdir -p "$DOWNLOAD_DIR"
+  mkdir -p "$DEST_DIR"
+
+  local DO_DOWNLOAD="true"
+  if [[ -f "$TARBALL_FILE" ]]; then
+    # If the file exists with a valid checksum, we can skip downloading.
+    calculated_md5="$($CIDL_MD5FUNC "$TARBALL_FILE")" || \
+        fail "failed to calculate md5sum"
+    if [[ "$calculated_md5" == "$CIDL_MD5" ]]; then
+      DO_DOWNLOAD="false"
+    fi
+  fi
+
+  if [ "$DO_DOWNLOAD" == "true" ]; then
+    echo "Downloading..."
+    do_download_curl "$CIDL_URL" "$TARBALL_FILE" || \
+        fail "failed to download file"
+
+    # Verify the md5sum.
+    calculated_md5="$($CIDL_MD5FUNC "$TARBALL_FILE")" || \
+        fail "failed to calculate md5sum"
+    if [[ "$calculated_md5" != "$CIDL_MD5" ]]; then
+      fail "md5sum mismatch \
+          (expected $CIDL_MD5, found $calculated_md5)"
+    fi
+  fi
+
+  # Unpack the tarball into a local directory
+  do_untar "$TARBALL_FILE" "$DEST_DIR"
+
+  # Run the binary as a sanity-check.
+  "$DEST_DIR/clickhouse" server --version
 }
 
 function fail
 {
-	echo "$ARG0: $@" >&2
-	exit 1
+  echo "$ARG0: $@" >&2
+  exit 1
 }
 
 function configure_os
 {
-	echo "current directory: $PWD"
-	echo "configuring based on OS: \"$1\""
-	case "$1" in
-		darwin*)
-            CIDL_PLATFORM="macos"
-			CIDL_MD5="$CIDL_MD5_DARWIN"
-			CIDL_MD5FUNC="do_md5"
-			;;
-		linux-gnu*) 
-            CIDL_PLATFORM="linux"
-			CIDL_MD5="$CIDL_MD5_LINUX"
-			CIDL_MD5FUNC="do_md5sum"
-			;;
-		solaris*)
-            CIDL_PLATFORM="illumos"
-			CIDL_MD5="$CIDL_MD5_ILLUMOS"
-			CIDL_MD5FUNC="do_md5sum"
-			;;
-		*)
-			fail "unsupported OS: $1"
-			;;
-	esac
-
-	CIDL_DIR="clickhouse-$CIDL_VERSION"
-	CIDL_FILE="$CIDL_DIR.$CIDL_PLATFORM.tar.gz"
+  echo "current directory: $PWD"
+  echo "configuring based on OS: \"$1\""
+  case "$1" in
+    darwin*)
+      CIDL_PLATFORM="macos"
+      CIDL_MD5="$CIDL_MD5_DARWIN"
+      CIDL_MD5FUNC="do_md5"
+      ;;
+    linux-gnu*)
+      CIDL_PLATFORM="linux"
+      CIDL_MD5="$CIDL_MD5_LINUX"
+      CIDL_MD5FUNC="do_md5sum"
+      ;;
+    solaris*)
+      CIDL_PLATFORM="illumos"
+      CIDL_MD5="$CIDL_MD5_ILLUMOS"
+      CIDL_MD5FUNC="do_md5sum"
+      ;;
+    *)
+      fail "unsupported OS: $1"
+      ;;
+  esac
+
+  TARBALL_DIRNAME="clickhouse-$CIDL_VERSION"
+  TARBALL_FILENAME="$TARBALL_DIRNAME.$CIDL_PLATFORM.tar.gz"
+
+  TARBALL_FILE="$DOWNLOAD_DIR/$TARBALL_FILENAME"
+  TARBALL_DIR="$DOWNLOAD_DIR/$TARBALL_DIRNAME"
 }
 
 function do_download_curl
 {
-	curl --silent --show-error --fail --location --output "$2" "$1"
+  curl --silent --show-error --fail --location --output "$2" "$1"
 }
 
 function do_md5
 {
-	md5 < "$1"
+  md5 < "$1"
 }
 
 function do_md5sum
 {
-	md5sum < "$1" | awk '{print $1}'
+  md5sum < "$1" | awk '{print $1}'
 }
 
 function do_untar
diff --git a/tools/ci_download_cockroachdb b/tools/ci_download_cockroachdb
index fb1a0f97c05..43680fef652 100755
--- a/tools/ci_download_cockroachdb
+++ b/tools/ci_download_cockroachdb
@@ -7,7 +7,6 @@
 #
 
 set -o pipefail
-set -o xtrace
 set -o errexit
 
 SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
@@ -17,7 +16,11 @@ ARG0="$(basename ${BASH_SOURCE[0]})"
 CIDL_VERSION="$(cat "$SOURCE_DIR/cockroachdb_version")"
 source "$SOURCE_DIR/cockroachdb_checksums"
 
-CIDL_ASSEMBLE_DIR="./cockroachdb"
+TARGET_DIR="out"
+# Location where intermediate artifacts are downloaded / unpacked.
+DOWNLOAD_DIR="$TARGET_DIR/downloads"
+# Location where the final cockroachdb directory should end up.
+DEST_DIR="./$TARGET_DIR/cockroachdb"
 
 # Official (or unofficial) download sites
 CIDL_URL_COCKROACH="https://binaries.cockroachdb.com"
@@ -25,113 +28,133 @@ CIDL_URL_ILLUMOS="https://illumos.org/downloads"
 
 function main
 {
-	#
-	# Process command-line arguments.  We generally don't expect any, but
-	# we allow callers to specify a value to override OSTYPE, just for
-	# testing.
-	#
-	if [[ $# != 0 ]]; then
-		CIDL_OS="$1"
-		shift
-	else
-		CIDL_OS="$OSTYPE"
-	fi
-
-	if [[ $# != 0 ]]; then
-		echo "unexpected arguments" >&2
-		exit 2
-	fi
-
-	# Configure this program
-	configure_os "$CIDL_OS"
-	CIDL_URL="$CIDL_URL_BASE/$CIDL_FILE"
-
-	# Download the file.
-	echo "URL: $CIDL_URL"
-	echo "Local file: $CIDL_FILE"
-	do_download_curl "$CIDL_URL" "$CIDL_FILE" || \
-	    fail "failed to download file"
-
-	# Verify the md5sum.
-	calculated_md5="$($CIDL_MD5FUNC "$CIDL_FILE")" || \
-	    fail "failed to calculate md5sum"
-	if [[ "$calculated_md5" != "$CIDL_MD5" ]]; then
-		fail "md5sum mismatch \
-		    (expected $CIDL_MD5, found $calculated_md5)"
-	fi
-
-	# Unpack the tarball.
-	do_untar "$CIDL_FILE"
-
-	# Copy the "cockroach" binary to the right spot.
-	$CIDL_ASSEMBLE "$CIDL_DIR"
-
-	# Run the binary as a sanity-check.
-	"$CIDL_ASSEMBLE_DIR/bin/cockroach" version
+  #
+  # Process command-line arguments.  We generally don't expect any, but
+  # we allow callers to specify a value to override OSTYPE, just for
+  # testing.
+  #
+  if [[ $# != 0 ]]; then
+    CIDL_OS="$1"
+    shift
+  else
+    CIDL_OS="$OSTYPE"
+  fi
+
+  if [[ $# != 0 ]]; then
+    echo "unexpected arguments" >&2
+    exit 2
+  fi
+
+  # Configure this program
+  configure_os "$CIDL_OS"
+  CIDL_URL="$CIDL_URL_BASE/$TARBALL_FILENAME"
+
+  # Download the file.
+  echo "URL: $CIDL_URL"
+  echo "Local file: $TARBALL_FILE"
+
+  mkdir -p "$DOWNLOAD_DIR"
+  mkdir -p "$DEST_DIR"
+
+  local DO_DOWNLOAD="true"
+  if [[ -f "$TARBALL_FILE" ]]; then
+    # If the file exists with a valid checksum, we can skip downloading.
+    calculated_md5="$($CIDL_MD5FUNC "$TARBALL_FILE")" || \
+        fail "failed to calculate md5sum"
+    if [[ "$calculated_md5" == "$CIDL_MD5" ]]; then
+      DO_DOWNLOAD="false"
+    fi
+  fi
+
+  if [ "$DO_DOWNLOAD" == "true" ]; then
+    echo "Downloading..."
+    do_download_curl "$CIDL_URL" "$TARBALL_FILE" || \
+        fail "failed to download file"
+
+    # Verify the md5sum.
+    calculated_md5="$($CIDL_MD5FUNC "$TARBALL_FILE")" || \
+        fail "failed to calculate md5sum"
+    if [[ "$calculated_md5" != "$CIDL_MD5" ]]; then
+      fail "md5sum mismatch \
+          (expected $CIDL_MD5, found $calculated_md5)"
+    fi
+  fi
+
+  # Unpack the tarball.
+  do_untar "$TARBALL_FILE"
+
+  # Copy the "cockroach" binary to the right spot.
+  $CIDL_ASSEMBLE
+
+  # Run the binary as a sanity-check.
+  "$DEST_DIR/bin/cockroach" version
 }
 
 function fail
 {
-	echo "$ARG0: $@" >&2
-	exit 1
+  echo "$ARG0: $@" >&2
+  exit 1
 }
 
 function configure_os
 {
-	echo "current directory: $PWD"
-	echo "configuring based on OS: \"$1\""
-	case "$1" in
-		darwin*)
-			CIDL_BUILD="darwin-10.9-amd64"
-			CIDL_SUFFIX="tgz"
-			CIDL_MD5="$CIDL_MD5_DARWIN"
-			CIDL_MD5FUNC="do_md5"
-			CIDL_URL_BASE="$CIDL_URL_COCKROACH"
-			CIDL_ASSEMBLE="do_assemble_official"
-			;;
-		linux-gnu*) 
-			CIDL_BUILD="linux-amd64"
-			CIDL_SUFFIX="tgz"
-			CIDL_MD5="$CIDL_MD5_LINUX"
-			CIDL_MD5FUNC="do_md5sum"
-			CIDL_URL_BASE="$CIDL_URL_COCKROACH"
-			CIDL_ASSEMBLE="do_assemble_official"
-			;;
-		solaris*)
-			CIDL_BUILD="illumos"
-			CIDL_SUFFIX="tar.gz"
-			CIDL_MD5="$CIDL_MD5_ILLUMOS"
-			CIDL_MD5FUNC="do_md5sum"
-			CIDL_URL_BASE="$CIDL_URL_ILLUMOS"
-			CIDL_ASSEMBLE="do_assemble_illumos"
-			;;
-		*)
-			fail "unsupported OS: $1"
-			;;
-	esac
-
-	CIDL_DIR="cockroach-$CIDL_VERSION.$CIDL_BUILD"
-	CIDL_FILE="$CIDL_DIR.$CIDL_SUFFIX"
+  echo "current directory: $PWD"
+  echo "configuring based on OS: \"$1\""
+  case "$1" in
+    darwin*)
+      CIDL_BUILD="darwin-10.9-amd64"
+      CIDL_SUFFIX="tgz"
+      CIDL_MD5="$CIDL_MD5_DARWIN"
+      CIDL_MD5FUNC="do_md5"
+      CIDL_URL_BASE="$CIDL_URL_COCKROACH"
+      CIDL_ASSEMBLE="do_assemble_official"
+      ;;
+    linux-gnu*)
+      CIDL_BUILD="linux-amd64"
+      CIDL_SUFFIX="tgz"
+      CIDL_MD5="$CIDL_MD5_LINUX"
+      CIDL_MD5FUNC="do_md5sum"
+      CIDL_URL_BASE="$CIDL_URL_COCKROACH"
+      CIDL_ASSEMBLE="do_assemble_official"
+      ;;
+    solaris*)
+      CIDL_BUILD="illumos"
+      CIDL_SUFFIX="tar.gz"
+      CIDL_MD5="$CIDL_MD5_ILLUMOS"
+      CIDL_MD5FUNC="do_md5sum"
+      CIDL_URL_BASE="$CIDL_URL_ILLUMOS"
+      CIDL_ASSEMBLE="do_assemble_illumos"
+      ;;
+    *)
+      fail "unsupported OS: $1"
+      ;;
+  esac
+
+  TARBALL_DIRNAME="cockroach-$CIDL_VERSION.$CIDL_BUILD"
+  TARBALL_FILENAME="$TARBALL_DIRNAME.$CIDL_SUFFIX"
+
+  TARBALL_FILE="$DOWNLOAD_DIR/$TARBALL_FILENAME"
+  TARBALL_DIR="$DOWNLOAD_DIR/$TARBALL_DIRNAME"
 }
 
 function do_download_curl
 {
-	curl --silent --show-error --fail --location --output "$2" "$1"
+  curl --silent --show-error --fail --location --output "$2" "$1"
 }
 
 function do_md5
 {
-	md5 < "$1"
+  md5 < "$1"
 }
 
 function do_md5sum
 {
-	md5sum < "$1" | awk '{print $1}'
+  md5sum < "$1" | awk '{print $1}'
 }
 
 function do_untar
 {
-	tar xzf "$1"
+  tar xzf "$1" -C "$DOWNLOAD_DIR"
 }
 
 #
@@ -145,14 +168,14 @@ function do_untar
 
 function do_assemble_official
 {
-	mkdir -p "$CIDL_ASSEMBLE_DIR/bin"
-	cp "$CIDL_DIR/cockroach" "$CIDL_ASSEMBLE_DIR/bin"
+  mkdir -p "$DEST_DIR/bin"
+  cp "$TARBALL_DIR/cockroach" "$DEST_DIR/bin"
 }
 
 function do_assemble_illumos
 {
-	rm -r "$CIDL_ASSEMBLE_DIR" || true
-	cp -r "cockroach-$CIDL_VERSION" "$CIDL_ASSEMBLE_DIR"
+  rm -r "$DEST_DIR" || true
+  cp -r "$DOWNLOAD_DIR/cockroach-$CIDL_VERSION" "$DEST_DIR"
 }
 
 main "$@"

From 876911f79818052bb95e66e2cd6611926aff4908 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Sat, 2 Apr 2022 23:10:47 -0400
Subject: [PATCH 05/14] Automate PATH verification, simplify README

---
 README.adoc                    | 50 ----------------------------------
 tools/install_prerequisites.sh | 30 +++++++++++++++++++-
 2 files changed, 29 insertions(+), 51 deletions(-)

diff --git a/README.adoc b/README.adoc
index 8f707328345..e3a82cf34ed 100644
--- a/README.adoc
+++ b/README.adoc
@@ -41,56 +41,6 @@ The following software may be installed automatically with the following scripts
 $ ./tools/install_prerequisites.sh
 ----
 
-Alternatively, the manual installation steps as follows:
-
-Both normal execution and the test suite expect certain binaries (described below) on your PATH.
-
-. libpq, the PostgreSQL client library
-+
---
-We use Diesel's PostgreSQL support to connect to CockroachDB (which is wire-compatible with PostgreSQL).  Diesel uses the native libpq to do this.  You can get the client library with:
-
-* Helios: `pkg install library/postgresql-13`
-* Linux: `sudo apt-get install libpq-dev`
-* Mac: `brew install postgresql`
-
-After doing this, you should have the `pg_config` command on your PATH.  For example, on Helios, you'd want `/opt/ooce/bin` on your PATH.
---
-. pkg-config, a tool for querying installed libraries
-+
---
-
-* Helios: `pkg install pkg-config`
-* Linux: `sudo apt-get install pkg-config`
-* Mac: `brew install pkg-config`
-
-After doing this, you should have the `pkg-config` command on your PATH.  For
-example, on Helios, you'd want `/usr/bin` on your PATH.
---
-. CockroachDB v21.1.10.
-+
-The build and test suite expects to be able to start a single-node CockroachDB cluster using the `cockroach` executable on your PATH.
-On illumos, MacOS, and Linux, you should be able to use the `tools/ci_download_cockroachdb` script to fetch the official CockroachDB binary.  It will be put into `./out/cockroachdb/bin/cockroach`.
-Alternatively, you can follow the https://www.cockroachlabs.com/docs/stable/install-cockroachdb.html[official CockroachDB installation instructions for your platform].
-
-. ClickHouse >= v21.7.1.
-+
-The test suite expects a running instance of the ClickHouse database server.
-The script `./tools/ci_download_clickhouse` can be used to download a pre-built binary for illumos, Linux, or macOS platforms. Once complete, you must manually add the binary (located at `./out/clickhouse/clickhouse`) to your PATH.
-You may also install ClickHouse manually; instructions can be found https://clickhouse.tech/docs/en/getting-started/install[here].
-See <<_configuring_clickhouse>> for details on ClickHouse's setup and configuration files.
-+
-. Additional software requirements:
-+
-On an illumos-based machine (Helios, OmniOS), if you want to run the real (non-simulated) Sled Agent to run actual VMs with Propolis, make sure your packages are up to date, and you have the `brand/omicron1/tools` package:
-+
-[source,text]
-----
-pkg install brand/omicron1/tools
-pkg install pkg:/package/pkg
-pkg update
-----
-
 === Running (Simulated) Omicron
 
 NOTE: If you'd like to run on Helios, refer to <<Deploying Omicron>> below.
diff --git a/tools/install_prerequisites.sh b/tools/install_prerequisites.sh
index 390094b4ef5..2648f1417dc 100755
--- a/tools/install_prerequisites.sh
+++ b/tools/install_prerequisites.sh
@@ -21,7 +21,8 @@ elif [[ "${HOST_OS}" == "SunOS" ]]; then
 
   # Perform updates
   if (( ${#need[@]} > 0 )); then
-    pfexec pkg install -v "${need[@]}" && rc=$? || rc=$?
+    rc=0
+    pfexec pkg install -v "${need[@]}" || rc=$?
     # Return codes:
     #  0: Normal Success
     #  4: Failure because we're already up-to-date. Also acceptable.
@@ -41,3 +42,30 @@ fi
 
 ./tools/ci_download_cockroachdb
 ./tools/ci_download_clickhouse
+
+# Validate the PATH:
+expected_in_path=(
+  'pg_config'
+  'pkg-config'
+)
+
+declare -A illumos_hints=(
+  ['pg_config']="On illumos, this is typically found in '/opt/ooce/bin'"
+  ['pkg-config']="On illumos, this is typically found in '/usr/bin'"
+)
+
+for command in "${expected_in_path[@]}"; do
+  rc=0
+  which "$command" &> /dev/null || rc=$?
+  if [ "$rc" -ne 0 ]; then
+    echo "$command seems installed, but not found in PATH. Please add it."
+
+
+    if [[ "${HOST_OS}" == "SunOS" ]]; then
+      if [ "${illumos_hints[$command]+_}" ]; then
+        echo "${illumos_hints[$command]}"
+      fi
+    fi
+    exit -1
+  fi
+done

From a80dd235d2b5f48153abf9391f16c114532f9af0 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Sat, 2 Apr 2022 23:14:32 -0400
Subject: [PATCH 06/14] Patch paths in github action

---
 .github/workflows/rust.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 091b5dbefd6..946033e8591 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -110,14 +110,14 @@ jobs:
       uses: actions/cache@26968a09c0ea4f3e233fdddbafd1166051a095f6
       with:
         key: ${{ runner.os }}-cockroach-binary-${{ hashFiles('tools/cockroachdb_checksums') }}
-        path: "cockroachdb"
+        path: "./out/cockroachdb"
     - name: Configure GitHub cache for ClickHouse binaries
       id: cache-clickhouse
       # actions/cache@v2.1.4
       uses: actions/cache@26968a09c0ea4f3e233fdddbafd1166051a095f6
       with:
         key: ${{ runner.os }}-clickhouse-binary-${{ hashFiles('tools/clickhouse_checksums') }}
-        path: "clickhouse"
+        path: "./out/clickhouse"
     - name: Download ClickHouse
       if: steps.cache-clickhouse.outputs.cache-hit != 'true'
       run: ./tools/ci_download_clickhouse
@@ -146,7 +146,7 @@ jobs:
       # rebuild here.
       # Put "./cockroachdb/bin" and "./clickhouse" on the PATH for the test
       # suite.
-      run: TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/cockroachdb/bin:$PWD/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" cargo test --no-fail-fast --workspace --locked --verbose
+      run: TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/out/cockroachdb/bin:$PWD/out/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" cargo test --no-fail-fast --workspace --locked --verbose
     - name: Archive any failed test results
       if: ${{ failure() }}
       # actions/upload-artifact@v2.3.1

From 215fea38f0d6eb2ca06a3cfaaed17feaa400f629 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Sat, 2 Apr 2022 23:18:15 -0400
Subject: [PATCH 07/14] stray newline

---
 tools/install_prerequisites.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tools/install_prerequisites.sh b/tools/install_prerequisites.sh
index 2648f1417dc..9ec51362fd0 100755
--- a/tools/install_prerequisites.sh
+++ b/tools/install_prerequisites.sh
@@ -60,7 +60,6 @@ for command in "${expected_in_path[@]}"; do
   if [ "$rc" -ne 0 ]; then
     echo "$command seems installed, but not found in PATH. Please add it."
 
-
     if [[ "${HOST_OS}" == "SunOS" ]]; then
       if [ "${illumos_hints[$command]+_}" ]; then
         echo "${illumos_hints[$command]}"

From 866e016fe2a3d38bc6a3fb85fa762abb41fe3342 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Sun, 3 Apr 2022 00:08:28 -0400
Subject: [PATCH 08/14] Patch more PATH

---
 .github/buildomat/jobs/build-and-test.sh | 2 +-
 .github/workflows/rust.yml               | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/buildomat/jobs/build-and-test.sh b/.github/buildomat/jobs/build-and-test.sh
index 183697e1f1f..2d204872a94 100644
--- a/.github/buildomat/jobs/build-and-test.sh
+++ b/.github/buildomat/jobs/build-and-test.sh
@@ -24,7 +24,7 @@ ptime -m bash ./tools/ci_download_cockroachdb
 # Put "./cockroachdb/bin" and "./clickhouse" on the PATH for the test
 # suite.
 #
-export PATH="$PATH:$PWD/cockroachdb/bin:$PWD/clickhouse"
+export PATH="$PATH:$PWD/out/cockroachdb/bin:$PWD/out/clickhouse"
 
 #
 # We build with:
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 946033e8591..4913737ffd8 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -140,7 +140,7 @@ jobs:
       # - TMPDIR=$OMICRON_TMP: we specify a specific temporary directory so that
       #   failed test outputs will be in a known place that we can grab at the
       #   end without also grabbing random other temporary files.
-      run: TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/cockroachdb/bin:$PWD/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" cargo build --locked --all-targets --verbose
+      run: TMPDIR=$OMICRON_TMP PATH="$PATH:$PWD/out/cockroachdb/bin:$PWD/out/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" cargo build --locked --all-targets --verbose
     - name: Run tests
       # Use the same RUSTFLAGS and RUSTDOCFLAGS as above to avoid having to
       # rebuild here.

From 40bdee81620417c91b2472644fadc1f0bba17d63 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Mon, 4 Apr 2022 13:26:07 -0400
Subject: [PATCH 09/14] Restore old formatting, xtrace

---
 tools/ci_download_clickhouse  | 183 ++++++++++++++--------------
 tools/ci_download_cockroachdb | 219 +++++++++++++++++-----------------
 2 files changed, 202 insertions(+), 200 deletions(-)

diff --git a/tools/ci_download_clickhouse b/tools/ci_download_clickhouse
index 3bdac422115..fdef07ac7ea 100755
--- a/tools/ci_download_clickhouse
+++ b/tools/ci_download_clickhouse
@@ -7,6 +7,7 @@
 #
 
 set -o pipefail
+set -o xtrace
 set -o errexit
 
 SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
@@ -27,121 +28,121 @@ CIDL_URL_BASE="https://oxide-clickhouse-build.s3.us-west-2.amazonaws.com"
 
 function main
 {
-  #
-  # Process command-line arguments.  We generally don't expect any, but
-  # we allow callers to specify a value to override OSTYPE, just for
-  # testing.
-  #
-  if [[ $# != 0 ]]; then
-    CIDL_OS="$1"
-    shift
-  else
-    CIDL_OS="$OSTYPE"
-  fi
-
-  if [[ $# != 0 ]]; then
-    echo "unexpected arguments" >&2
-    exit 2
-  fi
-
-  # Configure this program
-  configure_os "$CIDL_OS"
-  CIDL_URL="$CIDL_URL_BASE/$TARBALL_FILENAME"
-
-  # Download the file.
-  echo "URL: $CIDL_URL"
-  echo "Local file: $TARBALL_FILE"
-
-  mkdir -p "$DOWNLOAD_DIR"
-  mkdir -p "$DEST_DIR"
-
-  local DO_DOWNLOAD="true"
-  if [[ -f "$TARBALL_FILE" ]]; then
-    # If the file exists with a valid checksum, we can skip downloading.
-    calculated_md5="$($CIDL_MD5FUNC "$TARBALL_FILE")" || \
-        fail "failed to calculate md5sum"
-    if [[ "$calculated_md5" == "$CIDL_MD5" ]]; then
-      DO_DOWNLOAD="false"
-    fi
-  fi
-
-  if [ "$DO_DOWNLOAD" == "true" ]; then
-    echo "Downloading..."
-    do_download_curl "$CIDL_URL" "$TARBALL_FILE" || \
-        fail "failed to download file"
-
-    # Verify the md5sum.
-    calculated_md5="$($CIDL_MD5FUNC "$TARBALL_FILE")" || \
-        fail "failed to calculate md5sum"
-    if [[ "$calculated_md5" != "$CIDL_MD5" ]]; then
-      fail "md5sum mismatch \
-          (expected $CIDL_MD5, found $calculated_md5)"
-    fi
-  fi
-
-  # Unpack the tarball into a local directory
-  do_untar "$TARBALL_FILE" "$DEST_DIR"
-
-  # Run the binary as a sanity-check.
-  "$DEST_DIR/clickhouse" server --version
+	#
+	# Process command-line arguments. We generally don't expect any, but
+	# we allow callers to specify a value to override OSTYPE, just for
+	# testing.
+	#
+	if [[ $# != 0 ]]; then
+		CIDL_OS="$1"
+		shift
+	else
+		CIDL_OS="$OSTYPE"
+	fi
+
+	if [[ $# != 0 ]]; then
+		echo "unexpected arguments" >&2
+		exit 2
+	fi
+
+	# Configure this program
+	configure_os "$CIDL_OS"
+	CIDL_URL="$CIDL_URL_BASE/$TARBALL_FILENAME"
+
+	# Download the file.
+	echo "URL: $CIDL_URL"
+	echo "Local file: $TARBALL_FILE"
+
+	mkdir -p "$DOWNLOAD_DIR"
+	mkdir -p "$DEST_DIR"
+
+	local DO_DOWNLOAD="true"
+	if [[ -f "$TARBALL_FILE" ]]; then
+		# If the file exists with a valid checksum, we can skip downloading.
+		calculated_md5="$($CIDL_MD5FUNC "$TARBALL_FILE")" || \
+				fail "failed to calculate md5sum"
+		if [[ "$calculated_md5" == "$CIDL_MD5" ]]; then
+			DO_DOWNLOAD="false"
+		fi
+	fi
+
+	if [ "$DO_DOWNLOAD" == "true" ]; then
+		echo "Downloading..."
+		do_download_curl "$CIDL_URL" "$TARBALL_FILE" || \
+				fail "failed to download file"
+
+		# Verify the md5sum.
+		calculated_md5="$($CIDL_MD5FUNC "$TARBALL_FILE")" || \
+				fail "failed to calculate md5sum"
+		if [[ "$calculated_md5" != "$CIDL_MD5" ]]; then
+			fail "md5sum mismatch \
+					(expected $CIDL_MD5, found $calculated_md5)"
+		fi
+	fi
+
+	# Unpack the tarball into a local directory
+	do_untar "$TARBALL_FILE" "$DEST_DIR"
+
+	# Run the binary as a sanity-check.
+	"$DEST_DIR/clickhouse" server --version
 }
 
 function fail
 {
-  echo "$ARG0: $@" >&2
-  exit 1
+	echo "$ARG0: $@" >&2
+	exit 1
 }
 
 function configure_os
 {
-  echo "current directory: $PWD"
-  echo "configuring based on OS: \"$1\""
-  case "$1" in
-    darwin*)
-      CIDL_PLATFORM="macos"
-      CIDL_MD5="$CIDL_MD5_DARWIN"
-      CIDL_MD5FUNC="do_md5"
-      ;;
-    linux-gnu*)
-      CIDL_PLATFORM="linux"
-      CIDL_MD5="$CIDL_MD5_LINUX"
-      CIDL_MD5FUNC="do_md5sum"
-      ;;
-    solaris*)
-      CIDL_PLATFORM="illumos"
-      CIDL_MD5="$CIDL_MD5_ILLUMOS"
-      CIDL_MD5FUNC="do_md5sum"
-      ;;
-    *)
-      fail "unsupported OS: $1"
-      ;;
-  esac
-
-  TARBALL_DIRNAME="clickhouse-$CIDL_VERSION"
-  TARBALL_FILENAME="$TARBALL_DIRNAME.$CIDL_PLATFORM.tar.gz"
-
-  TARBALL_FILE="$DOWNLOAD_DIR/$TARBALL_FILENAME"
-  TARBALL_DIR="$DOWNLOAD_DIR/$TARBALL_DIRNAME"
+	echo "current directory: $PWD"
+	echo "configuring based on OS: \"$1\""
+	case "$1" in
+		darwin*)
+			CIDL_PLATFORM="macos"
+			CIDL_MD5="$CIDL_MD5_DARWIN"
+			CIDL_MD5FUNC="do_md5"
+			;;
+		linux-gnu*)
+			CIDL_PLATFORM="linux"
+			CIDL_MD5="$CIDL_MD5_LINUX"
+			CIDL_MD5FUNC="do_md5sum"
+			;;
+		solaris*)
+			CIDL_PLATFORM="illumos"
+			CIDL_MD5="$CIDL_MD5_ILLUMOS"
+			CIDL_MD5FUNC="do_md5sum"
+			;;
+		*)
+			fail "unsupported OS: $1"
+			;;
+	esac
+
+	TARBALL_DIRNAME="clickhouse-$CIDL_VERSION"
+	TARBALL_FILENAME="$TARBALL_DIRNAME.$CIDL_PLATFORM.tar.gz"
+
+	TARBALL_FILE="$DOWNLOAD_DIR/$TARBALL_FILENAME"
+	TARBALL_DIR="$DOWNLOAD_DIR/$TARBALL_DIRNAME"
 }
 
 function do_download_curl
 {
-  curl --silent --show-error --fail --location --output "$2" "$1"
+	curl --silent --show-error --fail --location --output "$2" "$1"
 }
 
 function do_md5
 {
-  md5 < "$1"
+	md5 < "$1"
 }
 
 function do_md5sum
 {
-  md5sum < "$1" | awk '{print $1}'
+	md5sum < "$1" | awk '{print $1}'
 }
 
 function do_untar
 {
-    mkdir -p "$2" && tar xzf "$1" -C "$2"
+		mkdir -p "$2" && tar xzf "$1" -C "$2"
 }
 
 main "$@"
diff --git a/tools/ci_download_cockroachdb b/tools/ci_download_cockroachdb
index 43680fef652..2a4c5db6c74 100755
--- a/tools/ci_download_cockroachdb
+++ b/tools/ci_download_cockroachdb
@@ -7,6 +7,7 @@
 #
 
 set -o pipefail
+set -o xtrace
 set -o errexit
 
 SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
@@ -28,139 +29,139 @@ CIDL_URL_ILLUMOS="https://illumos.org/downloads"
 
 function main
 {
-  #
-  # Process command-line arguments.  We generally don't expect any, but
-  # we allow callers to specify a value to override OSTYPE, just for
-  # testing.
-  #
-  if [[ $# != 0 ]]; then
-    CIDL_OS="$1"
-    shift
-  else
-    CIDL_OS="$OSTYPE"
-  fi
-
-  if [[ $# != 0 ]]; then
-    echo "unexpected arguments" >&2
-    exit 2
-  fi
-
-  # Configure this program
-  configure_os "$CIDL_OS"
-  CIDL_URL="$CIDL_URL_BASE/$TARBALL_FILENAME"
-
-  # Download the file.
-  echo "URL: $CIDL_URL"
-  echo "Local file: $TARBALL_FILE"
-
-  mkdir -p "$DOWNLOAD_DIR"
-  mkdir -p "$DEST_DIR"
-
-  local DO_DOWNLOAD="true"
-  if [[ -f "$TARBALL_FILE" ]]; then
-    # If the file exists with a valid checksum, we can skip downloading.
-    calculated_md5="$($CIDL_MD5FUNC "$TARBALL_FILE")" || \
-        fail "failed to calculate md5sum"
-    if [[ "$calculated_md5" == "$CIDL_MD5" ]]; then
-      DO_DOWNLOAD="false"
-    fi
-  fi
-
-  if [ "$DO_DOWNLOAD" == "true" ]; then
-    echo "Downloading..."
-    do_download_curl "$CIDL_URL" "$TARBALL_FILE" || \
-        fail "failed to download file"
-
-    # Verify the md5sum.
-    calculated_md5="$($CIDL_MD5FUNC "$TARBALL_FILE")" || \
-        fail "failed to calculate md5sum"
-    if [[ "$calculated_md5" != "$CIDL_MD5" ]]; then
-      fail "md5sum mismatch \
-          (expected $CIDL_MD5, found $calculated_md5)"
-    fi
-  fi
-
-  # Unpack the tarball.
-  do_untar "$TARBALL_FILE"
-
-  # Copy the "cockroach" binary to the right spot.
-  $CIDL_ASSEMBLE
-
-  # Run the binary as a sanity-check.
-  "$DEST_DIR/bin/cockroach" version
+	#
+	# Process command-line arguments. We generally don't expect any, but
+	# we allow callers to specify a value to override OSTYPE, just for
+	# testing.
+	#
+	if [[ $# != 0 ]]; then
+		CIDL_OS="$1"
+		shift
+	else
+		CIDL_OS="$OSTYPE"
+	fi
+
+	if [[ $# != 0 ]]; then
+		echo "unexpected arguments" >&2
+		exit 2
+	fi
+
+	# Configure this program
+	configure_os "$CIDL_OS"
+	CIDL_URL="$CIDL_URL_BASE/$TARBALL_FILENAME"
+
+	# Download the file.
+	echo "URL: $CIDL_URL"
+	echo "Local file: $TARBALL_FILE"
+
+	mkdir -p "$DOWNLOAD_DIR"
+	mkdir -p "$DEST_DIR"
+
+	local DO_DOWNLOAD="true"
+	if [[ -f "$TARBALL_FILE" ]]; then
+		# If the file exists with a valid checksum, we can skip downloading.
+		calculated_md5="$($CIDL_MD5FUNC "$TARBALL_FILE")" || \
+				fail "failed to calculate md5sum"
+		if [[ "$calculated_md5" == "$CIDL_MD5" ]]; then
+			DO_DOWNLOAD="false"
+		fi
+	fi
+
+	if [ "$DO_DOWNLOAD" == "true" ]; then
+		echo "Downloading..."
+		do_download_curl "$CIDL_URL" "$TARBALL_FILE" || \
+				fail "failed to download file"
+
+		# Verify the md5sum.
+		calculated_md5="$($CIDL_MD5FUNC "$TARBALL_FILE")" || \
+				fail "failed to calculate md5sum"
+		if [[ "$calculated_md5" != "$CIDL_MD5" ]]; then
+			fail "md5sum mismatch \
+					(expected $CIDL_MD5, found $calculated_md5)"
+		fi
+	fi
+
+	# Unpack the tarball.
+	do_untar "$TARBALL_FILE"
+
+	# Copy the "cockroach" binary to the right spot.
+	$CIDL_ASSEMBLE
+
+	# Run the binary as a sanity-check.
+	"$DEST_DIR/bin/cockroach" version
 }
 
 function fail
 {
-  echo "$ARG0: $@" >&2
-  exit 1
+	echo "$ARG0: $@" >&2
+	exit 1
 }
 
 function configure_os
 {
-  echo "current directory: $PWD"
-  echo "configuring based on OS: \"$1\""
-  case "$1" in
-    darwin*)
-      CIDL_BUILD="darwin-10.9-amd64"
-      CIDL_SUFFIX="tgz"
-      CIDL_MD5="$CIDL_MD5_DARWIN"
-      CIDL_MD5FUNC="do_md5"
-      CIDL_URL_BASE="$CIDL_URL_COCKROACH"
-      CIDL_ASSEMBLE="do_assemble_official"
-      ;;
-    linux-gnu*)
-      CIDL_BUILD="linux-amd64"
-      CIDL_SUFFIX="tgz"
-      CIDL_MD5="$CIDL_MD5_LINUX"
-      CIDL_MD5FUNC="do_md5sum"
-      CIDL_URL_BASE="$CIDL_URL_COCKROACH"
-      CIDL_ASSEMBLE="do_assemble_official"
-      ;;
-    solaris*)
-      CIDL_BUILD="illumos"
-      CIDL_SUFFIX="tar.gz"
-      CIDL_MD5="$CIDL_MD5_ILLUMOS"
-      CIDL_MD5FUNC="do_md5sum"
-      CIDL_URL_BASE="$CIDL_URL_ILLUMOS"
-      CIDL_ASSEMBLE="do_assemble_illumos"
-      ;;
-    *)
-      fail "unsupported OS: $1"
-      ;;
-  esac
-
-  TARBALL_DIRNAME="cockroach-$CIDL_VERSION.$CIDL_BUILD"
-  TARBALL_FILENAME="$TARBALL_DIRNAME.$CIDL_SUFFIX"
-
-  TARBALL_FILE="$DOWNLOAD_DIR/$TARBALL_FILENAME"
-  TARBALL_DIR="$DOWNLOAD_DIR/$TARBALL_DIRNAME"
+	echo "current directory: $PWD"
+	echo "configuring based on OS: \"$1\""
+	case "$1" in
+		darwin*)
+			CIDL_BUILD="darwin-10.9-amd64"
+			CIDL_SUFFIX="tgz"
+			CIDL_MD5="$CIDL_MD5_DARWIN"
+			CIDL_MD5FUNC="do_md5"
+			CIDL_URL_BASE="$CIDL_URL_COCKROACH"
+			CIDL_ASSEMBLE="do_assemble_official"
+			;;
+		linux-gnu*)
+			CIDL_BUILD="linux-amd64"
+			CIDL_SUFFIX="tgz"
+			CIDL_MD5="$CIDL_MD5_LINUX"
+			CIDL_MD5FUNC="do_md5sum"
+			CIDL_URL_BASE="$CIDL_URL_COCKROACH"
+			CIDL_ASSEMBLE="do_assemble_official"
+			;;
+		solaris*)
+			CIDL_BUILD="illumos"
+			CIDL_SUFFIX="tar.gz"
+			CIDL_MD5="$CIDL_MD5_ILLUMOS"
+			CIDL_MD5FUNC="do_md5sum"
+			CIDL_URL_BASE="$CIDL_URL_ILLUMOS"
+			CIDL_ASSEMBLE="do_assemble_illumos"
+			;;
+		*)
+			fail "unsupported OS: $1"
+			;;
+	esac
+
+	TARBALL_DIRNAME="cockroach-$CIDL_VERSION.$CIDL_BUILD"
+	TARBALL_FILENAME="$TARBALL_DIRNAME.$CIDL_SUFFIX"
+
+	TARBALL_FILE="$DOWNLOAD_DIR/$TARBALL_FILENAME"
+	TARBALL_DIR="$DOWNLOAD_DIR/$TARBALL_DIRNAME"
 }
 
 function do_download_curl
 {
-  curl --silent --show-error --fail --location --output "$2" "$1"
+	curl --silent --show-error --fail --location --output "$2" "$1"
 }
 
 function do_md5
 {
-  md5 < "$1"
+	md5 < "$1"
 }
 
 function do_md5sum
 {
-  md5sum < "$1" | awk '{print $1}'
+	md5sum < "$1" | awk '{print $1}'
 }
 
 function do_untar
 {
-  tar xzf "$1" -C "$DOWNLOAD_DIR"
+	tar xzf "$1" -C "$DOWNLOAD_DIR"
 }
 
 #
 # "Assembling" here is taking unpacked tarball and putting together a directory
-# structure that's common for all platforms.  This allows consumers (i.e., CI)
-# to assume the same directory structure for all platforms.  This is
+# structure that's common for all platforms. This allows consumers (i.e., CI)
+# to assume the same directory structure for all platforms. This is
 # platform-specific because on illumos, the tarball itself has a different
 # structure than the official release tarballs and the `cockroach` binary has
 # dynamic library dependencies.
@@ -168,14 +169,14 @@ function do_untar
 
 function do_assemble_official
 {
-  mkdir -p "$DEST_DIR/bin"
-  cp "$TARBALL_DIR/cockroach" "$DEST_DIR/bin"
+	mkdir -p "$DEST_DIR/bin"
+	cp "$TARBALL_DIR/cockroach" "$DEST_DIR/bin"
 }
 
 function do_assemble_illumos
 {
-  rm -r "$DEST_DIR" || true
-  cp -r "$DOWNLOAD_DIR/cockroach-$CIDL_VERSION" "$DEST_DIR"
+	rm -r "$DEST_DIR" || true
+	cp -r "$DOWNLOAD_DIR/cockroach-$CIDL_VERSION" "$DEST_DIR"
 }
 
 main "$@"

From c0be073146b3c340c88f1293c643799b5e977e4c Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Mon, 4 Apr 2022 13:28:11 -0400
Subject: [PATCH 10/14] stray tab

---
 tools/ci_download_clickhouse | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/ci_download_clickhouse b/tools/ci_download_clickhouse
index fdef07ac7ea..0eb2ff802fb 100755
--- a/tools/ci_download_clickhouse
+++ b/tools/ci_download_clickhouse
@@ -142,7 +142,7 @@ function do_md5sum
 
 function do_untar
 {
-		mkdir -p "$2" && tar xzf "$1" -C "$2"
+	mkdir -p "$2" && tar xzf "$1" -C "$2"
 }
 
 main "$@"

From a4ea71c457575dd782d0de24e571cbf3ac701ff2 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Mon, 4 Apr 2022 14:25:24 -0400
Subject: [PATCH 11/14] Add documentation, check clickhouse/cockroach in PATH

---
 tools/install_prerequisites.sh | 55 +++++++++++++++++++++++++++++++---
 1 file changed, 51 insertions(+), 4 deletions(-)

diff --git a/tools/install_prerequisites.sh b/tools/install_prerequisites.sh
index 9ec51362fd0..a24ea6e8333 100755
--- a/tools/install_prerequisites.sh
+++ b/tools/install_prerequisites.sh
@@ -6,6 +6,22 @@ set -eu
 SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 cd "${SOURCE_DIR}/.."
 
+# Packages to be installed on all OSes:
+#
+# - libpq, the PostgreSQL client lib.
+# We use Diesel's PostgreSQL support to connect to CockroachDB (which is
+# wire-compatible with PostgreSQL). Diesel uses the native libpq to do this.
+# `pg_config` is a utility which may be used to query for the installed
+# PostgreSQL libraries, and is expected by the Omicron build to exist in
+# the developer's PATH variable.
+# - pkg-config, a tool for querying installed libraries.
+#
+# Packages to be installed on Helios only:
+#
+# - pkg, the IPS client (though likely it will just be updated)
+# - build-essential: Common development tools
+# - brand/omicron1/tools: Oxide's omicron1-brand Zone
+
 HOST_OS=$(uname -s)
 if [[ "${HOST_OS}" == "Linux" ]]; then
   sudo apt-get install libpq-dev
@@ -40,6 +56,17 @@ else
   exit -1
 fi
 
+# CockroachDB and Clickhouse are used by Omicron for storage of
+# control plane metadata and metrics.
+#
+# They are used in a couple of spots within Omicron:
+#
+# - Test Suite: The test suite, regardless of host OS, builds temporary
+# databases for testing, and expects `cockroach` and `clickhouse` to
+# exist as a part of the PATH.
+# - Packaging: When constructing packages on Helios, these utilities
+# are packaged into zones which may be launched by the sled agent.
+
 ./tools/ci_download_cockroachdb
 ./tools/ci_download_clickhouse
 
@@ -47,13 +74,22 @@ fi
 expected_in_path=(
   'pg_config'
   'pkg-config'
+  'cockroach'
+  'clickhouse'
 )
 
-declare -A illumos_hints=(
+declare -A illumos_path_hints=(
   ['pg_config']="On illumos, this is typically found in '/opt/ooce/bin'"
   ['pkg-config']="On illumos, this is typically found in '/usr/bin'"
 )
 
+declare -A path_hints=(
+  ['cockroach']="This should have been installed to '$PWD/out/cockroachdb/bin'"
+  ['clickhouse']="This should have been installed to '$PWD/out/clickhouse'"
+)
+
+# Check all paths before returning an error.
+ANY_PATH_ERROR="false"
 for command in "${expected_in_path[@]}"; do
   rc=0
   which "$command" &> /dev/null || rc=$?
@@ -61,10 +97,21 @@ for command in "${expected_in_path[@]}"; do
     echo "$command seems installed, but not found in PATH. Please add it."
 
     if [[ "${HOST_OS}" == "SunOS" ]]; then
-      if [ "${illumos_hints[$command]+_}" ]; then
-        echo "${illumos_hints[$command]}"
+      if [ "${illumos_path_hints[$command]+_}" ]; then
+        echo "${illumos_path_hints[$command]}"
       fi
     fi
-    exit -1
+
+    if [ "${path_hints[$command]+_}" ]; then
+      echo "${path_hints[$command]}"
+    fi
+
+    ANY_PATH_ERROR="true"
   fi
 done
+
+if [ "$ANY_PATH_ERROR" == "true" ]; then
+  exit -1
+fi
+
+echo "All prerequisites installed successfully, and PATH looks valid"

From 3de53f182027f6030f1ec907d611ecc688b5960a Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Mon, 4 Apr 2022 14:33:58 -0400
Subject: [PATCH 12/14] More docs, hopefully more clear rc usage

---
 tools/install_prerequisites.sh | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/tools/install_prerequisites.sh b/tools/install_prerequisites.sh
index 390094b4ef5..a42cca50e19 100755
--- a/tools/install_prerequisites.sh
+++ b/tools/install_prerequisites.sh
@@ -11,7 +11,7 @@ if [[ "${HOST_OS}" == "Linux" ]]; then
   sudo apt-get install libpq-dev
   sudo apt-get install pkg-config
 elif [[ "${HOST_OS}" == "SunOS" ]]; then
-  need=(
+  packages=(
     'pkg:/package/pkg'
     'build-essential'
     'library/postgresql-13'
@@ -19,18 +19,19 @@ elif [[ "${HOST_OS}" == "SunOS" ]]; then
     'brand/omicron1/tools'
   )
 
-  # Perform updates
-  if (( ${#need[@]} > 0 )); then
-    pfexec pkg install -v "${need[@]}" && rc=$? || rc=$?
-    # Return codes:
-    #  0: Normal Success
-    #  4: Failure because we're already up-to-date. Also acceptable.
-    if [ "$rc" -ne 4 ] && [ "$rc" -ne 0 ]; then
-      exit "$rc"
-    fi
+  # Install/update the set of packages.
+  # Explicitly manage the return code using "rc" to observe the result of this
+  # command without exiting the script entirely (due to bash's "errexit").
+  rc=0
+  pfexec pkg install -v "${packages[@]}" || rc=$?
+  # Return codes:
+  #  0: Normal Success
+  #  4: Failure because we're already up-to-date. Also acceptable.
+  if [[ "$rc" -ne 4 ]] && [[ "$rc" -ne 0 ]]; then
+    exit "$rc"
   fi
 
-  pkg list -v "${need[@]}"
+  pkg list -v "${packages[@]}"
 elif [[ "${HOST_OS}" == "Darwin" ]]; then
   brew install postgresql
   brew install pkg-config

From fd77f51bd0606b910e3fcf4af2912fd02d9e5835 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Mon, 4 Apr 2022 15:52:18 -0400
Subject: [PATCH 13/14] Idempotency message on error

---
 tools/install_prerequisites.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tools/install_prerequisites.sh b/tools/install_prerequisites.sh
index a42cca50e19..9cc47afb389 100755
--- a/tools/install_prerequisites.sh
+++ b/tools/install_prerequisites.sh
@@ -2,6 +2,12 @@
 
 set -eu
 
+on_exit () {
+    echo "Something went wrong, but this script is idempotent - If you can fix the issue, try re-running"
+}
+
+trap on_exit ERR
+
 # Set the CWD to Omicron's source.
 SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 cd "${SOURCE_DIR}/.."

From 809f5f85e69ef6d754490b5676d4abe4d06fe103 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Mon, 4 Apr 2022 16:06:54 -0400
Subject: [PATCH 14/14] Avoid using associative arrays. Macs have old bash

---
 tools/install_prerequisites.sh | 46 ++++++++++++++++++----------------
 1 file changed, 25 insertions(+), 21 deletions(-)

diff --git a/tools/install_prerequisites.sh b/tools/install_prerequisites.sh
index 1df7c7f951e..646e7856ee7 100755
--- a/tools/install_prerequisites.sh
+++ b/tools/install_prerequisites.sh
@@ -84,15 +84,29 @@ expected_in_path=(
   'clickhouse'
 )
 
-declare -A illumos_path_hints=(
-  ['pg_config']="On illumos, this is typically found in '/opt/ooce/bin'"
-  ['pkg-config']="On illumos, this is typically found in '/usr/bin'"
-)
-
-declare -A path_hints=(
-  ['cockroach']="This should have been installed to '$PWD/out/cockroachdb/bin'"
-  ['clickhouse']="This should have been installed to '$PWD/out/clickhouse'"
-)
+function show_hint
+{
+  case "$1" in
+    "pg_config")
+      if [[ "${HOST_OS}" == "SunOS" ]]; then
+        echo "On illumos, $1 is typically found in '/opt/ooce/bin'"
+      fi
+      ;;
+    "pkg-config")
+      if [[ "${HOST_OS}" == "SunOS" ]]; then
+        echo "On illumos, $1 is typically found in '/usr/bin'"
+      fi
+      ;;
+    "cockroach")
+      echo "$1 should have been installed to '$PWD/out/cockroachdb/bin'"
+      ;;
+    "clickhouse")
+      echo "$1 should have been installed to '$PWD/out/clickhouse'"
+      ;;
+    *)
+      ;;
+  esac
+}
 
 # Check all paths before returning an error.
 ANY_PATH_ERROR="false"
@@ -100,18 +114,8 @@ for command in "${expected_in_path[@]}"; do
   rc=0
   which "$command" &> /dev/null || rc=$?
   if [ "$rc" -ne 0 ]; then
-    echo "$command seems installed, but not found in PATH. Please add it."
-
-    if [[ "${HOST_OS}" == "SunOS" ]]; then
-      if [ "${illumos_path_hints[$command]+_}" ]; then
-        echo "${illumos_path_hints[$command]}"
-      fi
-    fi
-
-    if [ "${path_hints[$command]+_}" ]; then
-      echo "${path_hints[$command]}"
-    fi
-
+    echo "ERROR: $command seems installed, but was not found in PATH. Please add it."
+    show_hint "$command"
     ANY_PATH_ERROR="true"
   fi
 done