Skip to content

Commit

Permalink
add issue links, and further cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
@cosmicexplorer
cosmicexplorer committed Jul 23, 2019
1 parent a0bdbc3 commit e9078cb
Show file tree
Hide file tree
Showing 4 changed files with 73 additions and 54 deletions.
3 changes: 2 additions & 1 deletion build-support/native-image/README.md
View file
Expand Up @@ -38,7 +38,8 @@ After a long bootstrap process, the arguments are forwarded to a pants invocatio
Caused by: java.lang.VerifyError: class scala.tools.nsc.Global overrides final method isDeveloper.()Z
```

Please re-run the script at most two more times. This can occur nondeterministically for some reason right now.
Please re-run the script at most two more times. This can occur nondeterministically for some reason right now. https://github.com/pantsbuild/pants/issues/7955 is intended to cover solving this issue, along with others.


## Updating the zinc native-image

Expand Down
26 changes: 11 additions & 15 deletions build-support/native-image/build-zinc-native-image.bash
View file
Expand Up @@ -14,7 +14,7 @@ function _get_coursier_impl {
if [[ ! -f ./coursier ]]; then
curl -Lo coursier https://git.io/coursier-cli || return "$?"
chmod +x coursier
./coursier --help >&2 || return "$?"
./coursier --help || return "$?"
fi >&2
normalize_path_check_file coursier
}
Expand Down Expand Up @@ -47,6 +47,9 @@ function bootstrap_environment {
}

function get_base_native_image_build_script_graal_checkout {
# TODO(#7955): See https://github.com/oracle/graal/issues/1448 and
# https://github.com/pantsbuild/pants/issues/7955 to cover using a released graal instead of a
# fork!
# From https://github.com/cosmicexplorer/graal/tree/graal-make-zinc-again!
do_within_cache_dir clone_repo_somewhat_idempotently \
graal/ \
Expand All @@ -59,14 +62,7 @@ function clone_mx {
do_within_cache_dir clone_repo_somewhat_idempotently \
mx/ \
https://github.com/graalvm/mx \
c01eef6e31cd5655b1f0682c445f4ed50aa5c05e \
| while read -r mx_repo_dir; do
# Reading the mx clone dir from `do_within_cache_dir`, we make sure to call `./mx update` from
# within the mx repo, and then echo the same dir to stdout.
>&2 with_pushd "$mx_repo_dir" \
./mx update
echo "$mx_repo_dir"
done
c01eef6e31cd5655b1f0682c445f4ed50aa5c05e
}

function extract_openjdk_jvmci {
Expand All @@ -87,10 +83,9 @@ function get_substratevm_dir {
}

function build_native_image_tool {
>&2 with_pushd "$(get_substratevm_dir)" \
mx build \
|| return "$?"
get_substratevm_dir
get_substratevm_dir \
| pushd_into_command_line_with_side_effect \
mx build
}

function fetch_scala_compiler_jars {
Expand All @@ -103,9 +98,10 @@ function fetch_scala_compiler_jars {
function fetch_pants_zinc_wrapper_jars {
pants_zinc_compiler_version='0.0.15'
pants_underlying_zinc_dependency_version='1.1.7'
# NB: `native-image` emits a warning on later protobuf versions, which the pantsbuild
# TODO: `native-image` emits a warning on later protobuf versions, which the pantsbuild
# `zinc-compiler` artifact will pull in unless we exclude them here and also explicitly add a
# protobuf artifact.
# protobuf artifact. We should fix this by making the change to the org.pantsbuild:zinc-compiler
# artifact!
"$(get_coursier)" fetch \
"org.pantsbuild:zinc-compiler_2.12:${pants_zinc_compiler_version}" \
"org.scala-sbt:compiler-bridge_2.12:${pants_underlying_zinc_dependency_version}" \
Expand Down
81 changes: 43 additions & 38 deletions build-support/native-image/generate-native-image-for-pants-targets.bash
View file
Expand Up @@ -13,14 +13,17 @@ set -euxo pipefail
# information should be inferred by the native-image agent, but isn't yet.

# TODO: build off of a more recent graal sha (more recent ones fail to build scalac and more): see
# https://github.com/oracle/graal/issues/1448, as well as #7955!
# https://github.com/oracle/graal/issues/1448, as well as
# https://github.com/pantsbuild/pants/issues/7955!

# ARGUMENTS
### ARGUMENTS

# $@: Forwarded to a `./pants compile` invocation which runs with reflection tracing enabled.
# $NATIVE_IMAGE_EXTRA_ARGS: Forwarded to a `native-image` invocation.
# $ZINC_IMAGE_VERSION: Used in the compile test that runs to validate the native-image. Determines
# where the image is written to in the pants cachedir.

# CONSTANTS
### CONSTANTS

SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"

Expand All @@ -32,11 +35,13 @@ GENERATED_CONFIG_DIR="$(normalize_path_no_validation "$GENERATED_CONFIG_DIRNAME"
GENERATED_CONFIG_JSON_FILE="${GENERATED_CONFIG_DIR}/reflect-config.json"
GENERATED_BUILD_FILE="${GENERATED_CONFIG_DIR}/BUILD"

ZINC_IMAGE_VERSION="${ZINC_IMAGE_VERSION:-0.0.15-from-image-script}"

# NB: Using $REPO_ROOT as calculated in other scripts in this repo seems to make this script fail
# when run from outside the pants repo!
DOWNLOAD_BINARY_SCRIPT="$(normalize_path_check_file "${SCRIPT_DIR}/../bin/download_binary.sh")"

# FUNCTIONS
### FUNCTIONS

function download_buildozer {
"$DOWNLOAD_BINARY_SCRIPT" \
Expand All @@ -45,27 +50,25 @@ function download_buildozer {
}

function build_agent_dylib {
pushd "$(get_substratevm_dir)" >&2
if is_osx; then
dylib_extension='.dylib'
local dylib_extension='.dylib'
else
dylib_extension='.so'
local dylib_extension='.so'
fi
desired_agent_dylib_path="$(pwd)/libnative-image-agent${dylib_extension}"
local desired_agent_dylib_path="libnative-image-agent${dylib_extension}"
if [[ ! -f "$desired_agent_dylib_path" ]]; then
mx build \
&& mx native-image --tool:native-image-agent --verbose \
&& cp -v {,lib}native-image-agent"$dylib_extension" \
|| return "$?"
fi >&2
echo "$desired_agent_dylib_path"
popd >&2
normalize_path_check_file "$desired_agent_dylib_path"
}

# run with tracing on

function run_zinc_compile_with_tracing {
agent_lib_path="$(build_agent_dylib)"
local agent_lib_path="$(get_substratevm_dir | pushd_into_command_line build_agent_dylib)"
if [[ ! -f "$GENERATED_CONFIG_JSON_FILE" ]]; then
rm -rfv "$GENERATED_CONFIG_DIR"
mkdir -v "$GENERATED_CONFIG_DIR"
Expand All @@ -81,7 +84,8 @@ function run_zinc_compile_with_tracing {
--no-incremental \
--no-use-classpath-jars \
--cache-ignore \
"$@"
"$@" \
|| return "$?"
unset PANTS_COMPILE_ZINC_JVM_OPTIONS
fi >&2
normalize_path_check_dir "$GENERATED_CONFIG_DIR"
Expand Down Expand Up @@ -193,31 +197,38 @@ function exercise_native_image_for_compilation {

"$image_location" --help || return "$?"
echo >&2 "native image generated at ${image_location}. running test compile..."
sleep 1
sleep 5

# Put the native-image in the correct cache location for zinc 0.0.15, overwriting the current one
# if necessary!
if is_osx; then
zinc_image_pants_cached="${HOME}/.cache/pants/bin/zinc-pants-native-image/mac/10.13/0.0.15-from-image-script"
local zinc_image_base_pants_cachedir="${HOME}/.cache/pants/bin/zinc-pants-native-image/mac/10.13"
else
zinc_image_pants_cached="${HOME}/.cache/pants/bin/zinc-pants-native-image/linux/x86_64/0.0.15-from-image-script"
local zinc_image_base_pants_cachedir="${HOME}/.cache/pants/bin/zinc-pants-native-image/linux/x86_64"
fi
local zinc_image_pants_cached="${zinc_image_base_pants_cachedir}/${ZINC_IMAGE_VERSION}"
mkdir -pv "$zinc_image_pants_cached" || return "$?"

cp -v "$image_location" "${zinc_image_pants_cached}/zinc-pants-native-image"

export PANTS_COMPILE_ZINC_JVM_OPTIONS='[]'
./pants \
--zinc-native-image --zinc-version=0.0.15-from-image-script \
compile.zinc --execution-strategy=hermetic --no-incremental --cache-ignore \
test \
"${other_args[@]}" || return "$?"
./pants --zinc-native-image --zinc-version="$ZINC_IMAGE_VERSION" \
compile.zinc --execution-strategy=hermetic --no-incremental --cache-ignore \
test \
"${other_args[@]}" \
|| return "$?"
unset PANTS_COMPILE_ZINC_JVM_OPTIONS
}

# actually build the zinc image!!!

### EXECUTING!
# NB: We create files named `.tmp-*` in many methods in this script. These files aren't used for
# caching, just as temporary outputs for parts of the script since we can't yet do this in Pants
# directly (see https://github.com/pantsbuild/pants/pull/6893). This will remove all of those files.
_orig_pwd="$(pwd)"
trap 'find "$_orig_pwd" -maxdepth 1 -type f -name ".tmp-*" -exec rm {} "+"' EXIT

# NB: It's not clear whether it's possible to set the locale in some ubuntu containers, so we
# simply ignore it here.
export PANTS_IGNORE_UNRECOGNIZED_ENCODING=1
Expand All @@ -238,8 +249,6 @@ fi

bootstrap_environment >&2

trap 'find . -maxdepth 1 -type f -name ".tmp-*" -exec rm {} "+"' EXIT

ensure_has_executable \
'jq' \
"jq must be installed to be able to manipulate the native-image json config." \
Expand All @@ -249,29 +258,25 @@ ensure_has_executable \
export JAVA_HOME="$(extract_openjdk_jvmci)"
PATH="$(clone_mx):${PATH}"

readonly ZINC_GENERATED_CONFIG_DIR="$(run_zinc_compile_with_tracing "$@")"
with_pushd "$ZINC_GENERATED_CONFIG_DIR" \
trim_reflection_config_to_just_macros
run_zinc_compile_with_tracing "$@" \
| >&2 pushd_into_command_line \
trim_reflection_config_to_just_macros
readonly MACRO_DEPS_JAR="$(generate_macro_deps_jar "$@")"

# NB: We also echo the image location to stdout, while piping everything else to stderr. This
# means that the script can be invoked from another script to get a single line of stdout
# containing the path to the just-built native-image.
create_zinc_image \
-H:ConfigurationFileDirectories="$GENERATED_CONFIG_DIR" \
-cp "$MACRO_DEPS_JAR" \
${NATIVE_IMAGE_EXTRA_ARGS:-} \
| while read -r image_location; do
# NB: `create_zinc_image` echoes the compiled zinc native-image location to stdout, so we read
# that single line of output with `while read -r`.
exercise_native_image_for_compilation "$image_location" "$@" \
>&2
# NB: We also echo the image location to stdout, while piping everything else to stderr. This
# means that the script can be invoked from another script to get a single line of stdout
# containing the path to the just-built native-image.
echo "$image_location"
done

# NB: if the native-image build fails, and you see the following in the output:
| command_line_with_side_effect \
exercise_native_image_for_compilation "$@"

# TODO(#7955): if the native-image build fails, and you see the following in the output:
#
# Caused by: java.lang.VerifyError: class scala.tools.nsc.Global overrides final method isDeveloper.()Z
#
# Please re-run the script at most two more times. This can occur nondeterministically for some
# reason right now.
# reason right now. https://github.com/pantsbuild/pants/issues/7955 is intended to cover solving
# this issue, along with others.
17 changes: 17 additions & 0 deletions build-support/native-image/utils.bash
View file
Expand Up @@ -30,6 +30,23 @@ function do_within_cache_dir {
with_pushd "$NATIVE_IMAGE_BUILD_CACHE_DIR" "$@"
}

function pushd_into_command_line {
while read -r into_dir; do
with_pushd "$into_dir" "$@" || return "$?"
done
}

function command_line_with_side_effect {
while read -r arg; do
>&2 "$1" "$arg" "${@:2}" || return "$?"
echo "$arg"
done
}

function pushd_into_command_line_with_side_effect {
command_line_with_side_effect with_pushd "$@"
}

function normalize_path_no_validation {
if hash realpath 2>/dev/null; then
realpath "$@"
Expand Down

0 comments on commit e9078cb

Please sign in to comment.