Skip to content

Commit

Permalink
New driver build scripts (#13383)
Browse files Browse the repository at this point in the history 
* New driver build scripts and fix Google driver build

* Install Clojure CLI in Circle CI

* Fix driver build script

* Build script fix 🔧

* More dox

* Enforce minimum build Clojure CLI version and address feedback

* New metabuild-common directory for shared stuff. Address some PR feedback
  • Loading branch information
@camsaul
camsaul committed Oct 8, 2020
1 parent fe545c6 commit 9a82d7d
Show file tree
Hide file tree
Showing 27 changed files with 982 additions and 315 deletions.
7 changes: 6 additions & 1 deletion .circleci/config.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -566,6 +566,12 @@ jobs:
- restore_cache:
keys:
- frontend-{{ checksum "./frontend-checksums.txt" }}
- run:
name: Install Clojure CLI
command: >
curl -O https://download.clojure.org/install/linux-install-1.10.1.708.sh &&
chmod +x linux-install-1.10.1.708.sh &&
sudo ./linux-install-1.10.1.708.sh
- run:
name: Build frontend if needed
command: >
Expand Down Expand Up @@ -942,7 +948,6 @@ workflows:
requires:
- build-uberjar
- fe-deps

- fe-tests-cypress:
name: fe-tests-cypress-1
requires:
Expand Down
2 changes: 1 addition & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
*.xcworkspacedata
.DS_Store
.\#*
.cpcache
.cpcache/
.eastwood
.idea/
.nrepl-port
Expand Down
235 changes: 5 additions & 230 deletions bin/build-driver.sh
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
#! /usr/bin/env bash

set -euo pipefail

project_root=`pwd`
set -eo pipefail

driver="$1"

Expand All @@ -11,231 +9,8 @@ if [ ! "$driver" ]; then
exit -1
fi

driver_project_dir="$project_root/modules/drivers/$driver"
driver_jar="$driver.metabase-driver.jar"
dest_location="$project_root/resources/modules/$driver_jar"
metabase_uberjar="$project_root/target/uberjar/metabase.jar"
target_jar="$driver_project_dir/target/uberjar/$driver_jar"
parents=''
checksum_file="$driver_project_dir/target/checksum.txt"

################################ DELETING OLD INCORRECTLY BUILT DRIVERS ###############################

verify_existing_build() {
verification_failed=''
./bin/verify-driver "$driver" || verification_failed=true

if [ "$verification_failed" ]; then
echo 'No existing build, or existing build is invalid. (Re)building driver.'
# By removing the checksum it will force rebuilding the driver
rm -f "$checksum_file"
fi
}


######################################## CALCULATING CHECKSUMS ########################################

md5_command=''
if [ `command -v md5` ]; then
md5_command=md5
elif [ `command -v md5sum` ]; then
md5_command=md5sum
else
echo "Don't know what command to use to calculate md5sums."
exit -2
fi

# Calculate a checksum of all the driver source files. If we've already built the driver and the checksum is the same
# there's no need to build the driver a second time
calculate_checksum() {
find "$driver_project_dir" -name '*.clj' -or -name '*.yaml' | sort | xargs cat | $md5_command
}

# Check whether the saved checksum for the driver sources from the last build is the same as the current one. If so,
# we don't need to build again.
checksum_is_same() {
if [ -f "$checksum_file" ]; then
old_checksum=`cat "$checksum_file"`
current_checksum=`calculate_checksum`
echo "Checksum of source files for previous build: $old_checksum"
echo "Current checksum of source files: $current_checksum"
if [ "$current_checksum" == "$old_checksum" ]; then
# Make sure the target driver JAR actually exists as well!
if [ -f "$target_jar" ]; then
echo "$driver driver source unchanged since last build. Skipping re-build."
return 0
fi
fi
fi
return 1
}

######################################## BUILDING THE DRIVER ########################################

# Delete existing saved copies of the driver in the plugins and resources directories
delete_old_drivers() {
echo "Deleting old versions of $driver driver..."
rm -f plugins/"$driver_jar"
rm -f "$dest_location"
}

# Check if Metabase is installed locally for building drivers; install it if not
install_metabase_core() {
if [ ! "$(find ~/.m2/repository/metabase-core/metabase-core -name '*.jar')" ]; then
echo "Building Metabase and installing locally..."
lein clean
lein install-for-building-drivers
else
echo "metabase-core already installed to local Maven repo."
fi
}


# Build Metabase uberjar if needed, we'll need this for stripping duplicate classes
build_metabase_uberjar() {
if [ ! -f "$metabase_uberjar" ]; then
echo 'Building Metabase uberjar...'
lein uberjar
else
echo "Metabase uberjar already built."
fi
}

# Take a look at the `parents` file listing the parents to build, if applicable
build_parents() {
echo "Building parent drivers (if needed)..."

parents_list="$driver_project_dir"/parents

if [ -f "$parents_list" ]; then
parents=`cat "$parents_list"`
echo "Found driver parents: $parents"
fi

# Check and see if we need to recursively build or install any of our parents before proceeding
for parent in $parents; do
if [ ! -f resources/modules/"$parent.metabase-driver.jar" ]; then
echo "Building $parent..."
./bin/build-driver.sh "$parent"
fi

# Check whether built parent driver *JAR* exists in local Maven repo
parent_install_dir="~/.m2/repository/metabase/$parent-driver"
parent_installed_jar=''
if [ -f "$parent_install_dir" ]; then
parent_installed_jar=`find "$parent_install_dir" -name '*.jar'`
fi
source "./bin/check-clojure-cli.sh"
check_clojure_cli

if [ ! "$parent_installed_jar" ]; then
parent_project_dir="$project_root/modules/drivers/$parent"
echo "Installing $parent locally..."
cd "$parent_project_dir"
lein clean
lein install-for-building-drivers
cd "$project_root"
else
echo "$parent already installed to local Maven repo"
fi
done
}

# Build the driver uberjar itself
build_driver_uberjar() {
echo "Building $driver driver..."

cd "$driver_project_dir"

rm -rf target

lein clean
DEBUG=1 LEIN_SNAPSHOTS_IN_RELEASE=true lein uberjar

cd "$project_root"

if [ ! -f "$target_jar" ]; then
echo "Error: could not find $target_jar. Build failed."
return -3
fi
}

# Strip out any classes in driver JAR found in core Metabase uberjar or parent JARs; recompress with higher compression ratio
strip_and_compress() {
# ok, first things first, strip out any classes also found in the core Metabase uberjar
lein strip-and-compress "$target_jar"

# next, remove any classes also found in any of the parent JARs
for parent in $parents; do
echo "Removing duplicate classes with $parent uberjar..."
lein strip-and-compress "$target_jar" "resources/modules/$parent.metabase-driver.jar"
done
}

# copy finished JAR to the resources dir
copy_target_to_dest() {
echo "Copying $target_jar -> $dest_location"
cp "$target_jar" "$dest_location"
}

# check that JAR in resources dir looks correct
verify_build () {
verification_failed=''
./bin/verify-driver "$driver" || verification_failed=true

if [ "$verification_failed" ]; then
echo "./bin/build-driver.sh $driver FAILED."
rm -f "$checksum_file"
rm -f "$target_jar"
rm -f "$dest_location"
return -4
fi
}

# Save the checksum for the newly built JAR
save_checksum() {
echo "Saving checksum for source files to $checksum_file"
checksum=`calculate_checksum`
echo "$checksum" > "$checksum_file"
}

# Runs all the steps needed to build the driver.
build_driver() {
verify_existing_build &&
delete_old_drivers &&
install_metabase_core &&
build_metabase_uberjar &&
build_parents &&
build_driver_uberjar &&
strip_and_compress &&
copy_target_to_dest &&
verify_build &&
save_checksum
}

######################################## PUTTING IT ALL TOGETHER ########################################

clean_local_repo() {
echo "Deleting existing installed metabase-core and driver dependencies..."
rm -rf ~/.m2/repository/metabase-core
rm -rf ~/.m2/repository/metabase/*-driver
}

retry_clean_build() {
echo "Building without cleaning failed. Retrying clean build..."
clean_local_repo
build_driver
}

mkdir -p resources/modules

# run only a specific step with ./bin/build-driver.sh <driver> <step>
if [ $# -eq 2 ]; then
$2
# Build driver if checksum has changed
elif ! checksum_is_same; then
echo "Checksum has changed."
build_driver || retry_clean_build
# Either way, always copy the target uberjar to the dest location
else
echo "Checksum is unchanged."
(copy_target_to_dest && verify_build) || retry_clean_build
fi
cd bin/build-drivers
clojure -M -m build-driver "$driver"
46 changes: 5 additions & 41 deletions bin/build-drivers.sh
View file
Original file line number Diff line number Diff line change
@@ -1,45 +1,9 @@
#! /usr/bin/env bash

set -eo pipefail
set -euo pipefail

# If ran as `./bin/build-drivers.sh clean` then uninstall metabase-core from the local Maven repo and delete
if [ "$1" == clean ]; then
echo "Deleting existing installed metabase-core and driver dependencies..."
rm -rf ~/.m2/repository/metabase-core
rm -rf ~/.m2/repository/metabase/*-driver
source "./bin/check-clojure-cli.sh"
check_clojure_cli

echo "Deleting built drivers in resources/modules..."
rm -rf resources/modules
echo "Deleting build Metabase uberjar..."
rm -rf target

for target in `find modules -name 'target' -type d`; do
echo "Deleting $target..."
rm -rf "$target"
done
fi

# strip trailing slashes if `ls` is set to include them
drivers=`ls modules/drivers/ | sed 's|/$||'`

for driver in $drivers; do
echo "Build: $driver"

build_failed=''
./bin/build-driver.sh "$driver" || build_failed=true

if [ "$build_failed" ]; then
echo "Failed to build driver $driver."
exit -1
fi
done

# Double-check that all drivers were built successfully
for driver in $drivers; do
verification_failed=''
./bin/verify-driver "$driver" || verification_failed=true

if [ "$verification_failed" ]; then
exit -2
fi
done
cd bin/build-drivers
clojure -M -m build-drivers
45 changes: 45 additions & 0 deletions bin/build-drivers/README.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
# Build-drivers scripts

Scripts for building Metabase driver plugins. You must install the [Clojure CLI
tools](https://www.clojure.org/guides/getting_started) to use these.

There are three main entrypoints. Shell script wrappers are provided for convenience and compatibility.

### `build-drivers`

Builds *all* drivers as needed. If drivers were recently built and no relevant source code changed, skips rebuild.

```
cd bin/build-drivers
clojure -M -m build-drivers
# or
./bin/build-drivers.sh
```

### `build-driver`

Build a single driver as needed. Builds parent drivers if needed first.

```
cd bin/build-driver redshift
clojure -M -m build-driver redshift
# or
./bin/build-driver.sh redshift
```

### `verify-driver`

Verify that a built driver looks correctly built.

```
cd bin/verify-driver redshift
clojure -M -m verify-driver redshift
# or
./bin/verify-driver redshift
```
10 changes: 10 additions & 0 deletions bin/build-drivers/build_driver.clj
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
(ns build-driver
"Entrypoint for `bin/build-driver.sh`. Builds a single driver, if needed."
(:require [build-drivers.build-driver :as build-driver]
[metabuild-common.core :as u]))

(defn -main [& [driver]]
(u/exit-when-finished-nonzero-on-exception
(when-not (seq driver)
(throw (ex-info "Usage: clojure -m build-driver <driver>" {})))
(build-driver/build-driver! (keyword driver))))
Loading

0 comments on commit 9a82d7d

Please sign in to comment.