forked from StackStorm/st2
-
Notifications
You must be signed in to change notification settings - Fork 0
/
pants
executable file
·481 lines (423 loc) · 17.9 KB
/
pants
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
#!/usr/bin/env bash
# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).
# =============================== NOTE ===============================
# This ./pants bootstrap script comes from the pantsbuild/setup
# project. It is intended to be checked into your code repository so
# that other developers have the same setup.
#
# Learn more here: https://www.pantsbuild.org/docs/installation
# ====================================================================
set -eou pipefail
# an arbitrary number: bump when there's a change that someone might want to query for
# (e.g. checking $(PANTS_BOOTSTRAP_TOOLS=1 ./pants version) >= ...)
SCRIPT_VERSION=1
# Source any custom bootstrap settings for Pants from PANTS_BOOTSTRAP if it exists.
: ${PANTS_BOOTSTRAP:=".pants.bootstrap"}
if [[ -f "${PANTS_BOOTSTRAP}" ]]; then
source "${PANTS_BOOTSTRAP}"
fi
# NOTE: To use an unreleased version of Pants from the pantsbuild/pants main branch,
# locate the main branch SHA, set PANTS_SHA=<SHA> in the environment, and run this script as usual.
#
# E.g., PANTS_SHA=725fdaf504237190f6787dda3d72c39010a4c574 ./pants --version
PYTHON_BIN_NAME="${PYTHON:-unspecified}"
# Set this to specify a non-standard location for this script to read the Pants version from.
# NB: This will *not* cause Pants itself to use this location as a config file.
# You can use PANTS_CONFIG_FILES or --pants-config-files to do so.
PANTS_TOML=${PANTS_TOML:-pants.toml}
PANTS_BIN_NAME="${PANTS_BIN_NAME:-$0}"
PANTS_SETUP_CACHE="${PANTS_SETUP_CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/pants/setup}"
# If given a relative path, we fix it to be absolute.
if [[ "$PANTS_SETUP_CACHE" != /* ]]; then
PANTS_SETUP_CACHE="${PWD}/${PANTS_SETUP_CACHE}"
fi
PANTS_BOOTSTRAP="${PANTS_SETUP_CACHE}/bootstrap-$(uname -s)-$(uname -m)"
_PEX_VERSION=2.1.103
_PEX_URL="https://github.com/pantsbuild/pex/releases/download/v${_PEX_VERSION}/pex"
_PEX_EXPECTED_SHA256="4d45336511484100ae4e2bab24542a8b86b12c8cb89230463593c60d08c4b8d3"
VIRTUALENV_VERSION=20.4.7
VIRTUALENV_REQUIREMENTS=$(
cat << EOF
virtualenv==${VIRTUALENV_VERSION} --hash sha256:2b0126166ea7c9c3661f5b8e06773d28f83322de7a3ff7d06f0aed18c9de6a76
filelock==3.0.12 --hash sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836
six==1.16.0 --hash sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
distlib==0.3.2 --hash sha256:23e223426b28491b1ced97dc3bbe183027419dfc7982b4fa2f05d5f3ff10711c
appdirs==1.4.4 --hash sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128
importlib-resources==5.1.4; python_version < "3.7" --hash sha256:e962bff7440364183203d179d7ae9ad90cb1f2b74dcb84300e88ecc42dca3351
importlib-metadata==4.5.0; python_version < "3.8" --hash sha256:833b26fb89d5de469b24a390e9df088d4e52e4ba33b01dc5e0e4f41b81a16c00
zipp==3.4.1; python_version < "3.10" --hash sha256:51cb66cc54621609dd593d1787f286ee42a5c0adbb4b29abea5a63edc3e03098
typing-extensions==3.10.0.0; python_version < "3.8" --hash sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84
EOF
)
COLOR_RED="\x1b[31m"
COLOR_GREEN="\x1b[32m"
COLOR_YELLOW="\x1b[33m"
COLOR_RESET="\x1b[0m"
INSTALL_URL="https://www.pantsbuild.org/docs/installation"
function log() {
echo -e "$@" 1>&2
}
function die() {
(($# > 0)) && log "${COLOR_RED}$*${COLOR_RESET}"
exit 1
}
function green() {
(($# > 0)) && log "${COLOR_GREEN}$*${COLOR_RESET}"
}
function warn() {
(($# > 0)) && log "${COLOR_YELLOW}$*${COLOR_RESET}"
}
function tempdir {
mkdir -p "$1"
mktemp -d "$1"/pants.XXXXXX
}
function get_exe_path_or_die {
local exe="$1"
if ! command -v "${exe}"; then
die "Could not find ${exe}. Please ensure ${exe} is on your PATH."
fi
}
function get_pants_config_string_value {
local config_key="$1"
local optional_space="[[:space:]]*"
local prefix="^${config_key}${optional_space}=${optional_space}"
local raw_value
raw_value="$(sed -ne "/${prefix}/ s|${prefix}||p" "${PANTS_TOML}")"
local optional_suffix="${optional_space}(#.*)?$"
echo "${raw_value}" \
| sed -E \
-e "s|^'([^']*)'${optional_suffix}|\1|" \
-e 's|^"([^"]*)"'"${optional_suffix}"'$|\1|' \
&& return 0
return 0
}
function get_python_major_minor_version {
local python_exe="$1"
"$python_exe" <<EOF
import sys
major_minor_version = ''.join(str(version_num) for version_num in sys.version_info[0:2])
print(major_minor_version)
EOF
}
# The high-level flow:
#
# 1.) Resolve the Pants version from config so that we know what interpreters we can use, what to name the venv,
# and what to install via pip.
# 2.) Resolve the Python interpreter, first reading from the env var $PYTHON, then using a default based on the Pants
# version.
# 3.) Check if the venv already exists via a naming convention, and create the venv if not found.
# 4.) Execute Pants with the resolved Python interpreter and venv.
#
# After that, Pants itself will handle making sure any requested plugins
# are installed and up to date.
function determine_pants_version {
if [ -n "${PANTS_SHA:-}" ]; then
# get_version_for_sha will echo the version, thus "returning" it from this function.
get_version_for_sha "$PANTS_SHA"
return
fi
pants_version="$(get_pants_config_string_value 'pants_version')"
if [[ -z "${pants_version}" ]]; then
die "Please explicitly specify the \`pants_version\` in your \`pants.toml\` under the \`[GLOBAL]\` scope.
See https://pypi.org/project/pantsbuild.pants/#history for all released versions
and ${INSTALL_URL} for more instructions."
fi
pants_major_version="$(echo "${pants_version}" | cut -d '.' -f1)"
pants_minor_version="$(echo "${pants_version}" | cut -d '.' -f2)"
# 1.26 is the first version to support `pants.toml`, so we fail eagerly if using an outdated version.
if [[ "${pants_major_version}" -eq 1 && "${pants_minor_version}" -le 25 ]]; then
die "This version of the \`./pants\` script does not work with Pants <= 1.25.0 (and it also requires using \`pants.toml\`,
rather than \`pants.ini\`). Instead, either upgrade your \`pants_version\` or use the version of the \`./pants\` script
at https://raw.githubusercontent.com/Eric-Arellano/setup/0d445edef57cb89fd830db70810e38f050b0a268/pants."
fi
echo "${pants_version}"
}
function set_supported_python_versions {
local pants_version="$1"
local pants_major_version
local pants_minor_version
pants_major_version="$(echo "${pants_version}" | cut -d '.' -f1)"
pants_minor_version="$(echo "${pants_version}" | cut -d '.' -f2)"
if [[ "${pants_major_version}" -eq 1 ]]; then
supported_python_versions_decimal=('3.6' '3.7' '3.8')
supported_python_versions_int=('36' '37' '38')
supported_message='3.6, 3.7, or 3.8'
elif [[ "${pants_major_version}" -eq 2 && "${pants_minor_version}" -eq 0 ]]; then
supported_python_versions_decimal=('3.6' '3.7' '3.8')
supported_python_versions_int=('36' '37' '38')
supported_message='3.6, 3.7, or 3.8'
elif [[ "${pants_major_version}" -eq 2 && "${pants_minor_version}" -eq 1 ]]; then
supported_python_versions_decimal=('3.7' '3.8' '3.6')
supported_python_versions_int=('37' '38' '36')
supported_message='3.7, 3.8, or 3.6 (deprecated)'
elif [[ "${pants_major_version}" -eq 2 && "${pants_minor_version}" -lt 5 ]]; then
supported_python_versions_decimal=('3.8' '3.7')
supported_python_versions_int=('38' '37')
supported_message='3.7 or 3.8'
else
# We put 3.9 first because Apple Silicon only works properly with Python 3.9, even though it's possible to have
# older Pythons installed. This makes it more likely that Pants will work out-of-the-box.
supported_python_versions_decimal=('3.9' '3.8' '3.7')
supported_python_versions_int=('39' '38' '37')
supported_message='3.7, 3.8, or 3.9'
fi
}
function check_python_exe_compatible_version {
local python_exe="$1"
local major_minor_version
major_minor_version="$(get_python_major_minor_version "${python_exe}")"
for valid_version in "${supported_python_versions_int[@]}"; do
if [[ "${major_minor_version}" == "${valid_version}" ]]; then
echo "${python_exe}" && return 0
fi
done
}
function determine_default_python_exe {
for version in "${supported_python_versions_decimal[@]}" "3" ""; do
local interpreter_path
interpreter_path="$(command -v "python${version}")"
if [[ -z "${interpreter_path}" ]]; then
continue
fi
# Check if a version is shimmed by pyenv or asdf but not configured.
if ! "${interpreter_path}" --version >/dev/null 2>&1; then
continue
fi
if [[ -n "$(check_python_exe_compatible_version "${interpreter_path}")" ]]; then
echo "${interpreter_path}" && return 0
fi
done
}
function determine_python_exe {
local pants_version="$1"
set_supported_python_versions "${pants_version}"
local requirement_str="For \`pants_version = \"${pants_version}\"\`, Pants requires Python ${supported_message} to run."
local python_exe
if [[ "${PYTHON_BIN_NAME}" != 'unspecified' ]]; then
python_exe="$(get_exe_path_or_die "${PYTHON_BIN_NAME}")" || exit 1
if [[ -z "$(check_python_exe_compatible_version "${python_exe}")" ]]; then
die "Invalid Python interpreter version for ${python_exe}. ${requirement_str}"
fi
else
python_exe="$(determine_default_python_exe)"
if [[ -z "${python_exe}" ]]; then
die "No valid Python interpreter found. ${requirement_str} Please check that a valid interpreter is installed and on your \$PATH."
fi
fi
echo "${python_exe}"
}
function compute_sha256 {
local python="$1"
local path="$2"
"$python" <<EOF
import hashlib
hasher = hashlib.sha256()
with open('${path}', 'rb') as fp:
buf = fp.read()
hasher.update(buf)
print(hasher.hexdigest())
EOF
}
# TODO(John Sirois): GC race loser tmp dirs leftover from bootstrap_XXX
# functions. Any tmp dir w/o a symlink pointing to it can go.
function bootstrap_pex {
local python="$1"
local bootstrapped="${PANTS_BOOTSTRAP}/pex-${_PEX_VERSION}/pex"
if [[ ! -f "${bootstrapped}" ]]; then
(
green "Downloading the Pex PEX."
mkdir -p "${PANTS_BOOTSTRAP}"
local staging_dir
staging_dir=$(tempdir "${PANTS_BOOTSTRAP}")
cd "${staging_dir}"
curl --proto "=https" \
--tlsv1.2 \
--silent \
--location \
--remote-name \
"${_PEX_URL}"
fingerprint="$(compute_sha256 "${python}" "pex")"
if [[ "${_PEX_EXPECTED_SHA256}" != "${fingerprint}" ]]; then
die "SHA256 of ${_PEX_URL} is not as expected. Aborting."
fi
green "SHA256 fingerprint of ${_PEX_URL} verified."
mkdir -p "$(dirname "${bootstrapped}")"
mv -f "${staging_dir}/pex" "${bootstrapped}"
rmdir "${staging_dir}"
) 1>&2 || exit 1
fi
echo "${bootstrapped}"
}
function scrub_env_vars {
# Ensure the virtualenv PEX runs as shrink-wrapped.
# See: https://github.com/pantsbuild/setup/issues/105
local -r pex_env_vars=(${!PEX_@})
if [[ ! ${#pex_env_vars[@]} -eq 0 ]]; then
local -r pex_env_vars_to_scrub="${pex_env_vars[@]/PEX_ROOT}"
if [[ -n "${pex_env_vars_to_scrub[@]}" ]]; then
warn "Scrubbing ${pex_env_vars_to_scrub[@]}"
unset ${pex_env_vars_to_scrub[@]}
fi
fi
# Also ensure pip doesn't think packages on PYTHONPATH
# are already installed.
if [ -n "${PYTHONPATH:-}" ]; then
warn "Scrubbing PYTHONPATH"
unset PYTHONPATH
fi
}
function bootstrap_virtualenv {
local python="$1"
local bootstrapped="${PANTS_BOOTSTRAP}/virtualenv-${VIRTUALENV_VERSION}/virtualenv.pex"
if [[ ! -f "${bootstrapped}" ]]; then
(
green "Creating the virtualenv PEX."
pex_path="$(bootstrap_pex "${python}")" || exit 1
mkdir -p "${PANTS_BOOTSTRAP}"
local staging_dir
staging_dir=$(tempdir "${PANTS_BOOTSTRAP}")
cd "${staging_dir}"
echo "${VIRTUALENV_REQUIREMENTS}" > requirements.txt
(
scrub_env_vars
"${python}" "${pex_path}" -r requirements.txt -c virtualenv -o virtualenv.pex
)
mkdir -p "$(dirname "${bootstrapped}")"
mv -f "${staging_dir}/virtualenv.pex" "${bootstrapped}"
rm -rf "${staging_dir}"
) 1>&2 || exit 1
fi
echo "${bootstrapped}"
}
function find_links_url {
local pants_version="$1"
local pants_sha="$2"
echo -n "https://binaries.pantsbuild.org/wheels/pantsbuild.pants/${pants_sha}/${pants_version/+/%2B}/index.html"
}
function get_version_for_sha {
local sha="$1"
# Retrieve the Pants version associated with this commit.
local pants_version
pants_version="$(curl --proto "=https" \
--tlsv1.2 \
--fail \
--silent \
--location \
"https://raw.githubusercontent.com/pantsbuild/pants/${sha}/src/python/pants/VERSION")"
# Construct the version as the release version from src/python/pants/VERSION, plus the string `+gitXXXXXXXX`,
# where the XXXXXXXX is the first 8 characters of the SHA.
echo "${pants_version}+git${sha:0:8}"
}
function bootstrap_pants {
local pants_version="$1"
local python="$2"
local pants_sha="${3:-}"
local pants_requirement="pantsbuild.pants==${pants_version}"
local maybe_find_links
if [[ -z "${pants_sha}" ]]; then
maybe_find_links=""
else
maybe_find_links="--find-links=$(find_links_url "${pants_version}" "${pants_sha}")"
fi
local python_major_minor_version
python_major_minor_version="$(get_python_major_minor_version "${python}")"
local target_folder_name="${pants_version}_py${python_major_minor_version}"
local bootstrapped="${PANTS_BOOTSTRAP}/${target_folder_name}"
if [[ ! -d "${bootstrapped}" ]]; then
(
green "Bootstrapping Pants using ${python}"
local staging_dir
staging_dir=$(tempdir "${PANTS_BOOTSTRAP}")
local virtualenv_path
virtualenv_path="$(bootstrap_virtualenv "${python}")" || exit 1
green "Installing ${pants_requirement} into a virtual environment at ${bootstrapped}"
(
scrub_env_vars
# shellcheck disable=SC2086
"${python}" "${virtualenv_path}" --quiet --no-download "${staging_dir}/install" && \
# Grab the latest pip, but don't advance setuptools past 58 which drops support for the
# `setup` kwarg `use_2to3` which Pants 1.x sdist dependencies (pystache) use.
"${staging_dir}/install/bin/pip" install --quiet -U pip "setuptools<58" && \
"${staging_dir}/install/bin/pip" install ${maybe_find_links} --quiet --progress-bar off "${pants_requirement}"
) && \
ln -s "${staging_dir}/install" "${staging_dir}/${target_folder_name}" && \
mv "${staging_dir}/${target_folder_name}" "${bootstrapped}" && \
green "New virtual environment successfully created at ${bootstrapped}."
) 1>&2 || exit 1
fi
echo "${bootstrapped}"
}
function run_bootstrap_tools {
# functionality for introspecting the bootstrapping process, without actually doing it
if [[ "${PANTS_BOOTSTRAP_TOOLS}" -gt "${SCRIPT_VERSION}" ]]; then
die "$0 script (bootstrap version ${SCRIPT_VERSION}) is too old for this invocation (with PANTS_BOOTSTRAP_TOOLS=${PANTS_BOOTSTRAP_TOOLS}).
Please update it by following ${INSTALL_URL}"
fi
case "${1:-}" in
bootstrap-cache-key)
local pants_version=$(determine_pants_version)
local python="$(determine_python_exe "${pants_version}")"
# the python above may be a shim (e.g. pyenv or homebrew), so let's get an estimate of the
# actual path, as will be symlinked in the virtualenv. (NB. virtualenv does more complicated
# things, but we at least emulate the symlink-resolution that it does.)
local python_executable_path="$("${python}" -c 'import os, sys; print(os.path.realpath(sys.executable))')"
local requirements_file="$(mktemp)"
echo "${VIRTUALENV_REQUIREMENTS}" > "${requirements_file}"
local virtualenv_requirements_sha256="$(compute_sha256 "${python}" "${requirements_file}")"
rm "${requirements_file}"
local parts=(
"os_name=$(uname -s)"
"arch=$(uname -m)"
"python_path=${python}"
"python_executable_path=${python_executable_path}"
# the full interpreter information, for maximum compatibility
"python_version=$("$python" --version)"
"pex_version=${_PEX_VERSION}"
"virtualenv_requirements_sha256=${virtualenv_requirements_sha256}"
"pants_version=${pants_version}"
)
echo "${parts[*]}"
;;
bootstrap-version)
echo "${SCRIPT_VERSION}"
;;
help|"")
cat <<EOF
Usage: PANTS_BOOTSTRAP_TOOLS=1 $0 ...
Subcommands:
bootstrap-cache-key
Print an opaque that can be used as a key for accurate and safe caching of
the pants bootstrap directories.
(Added in bootstrap version 1.)
bootstrap-version
Print a version number for the bootstrap script itself.
Distributed scripts (such as reusable CI formulae) that use these bootstrap
tools should set PANTS_BOOTSTRAP_TOOLS to the minimum script version for the
features they require. For example, if 'some-tool' was added in version 123:
PANTS_BOOTSTRAP_TOOLS=123 ./pants some-tool
(Added in bootstrap version 1.)
EOF
;;
*)
die "Unknown subcommand for bootstrap tools: $1. Do you mean to run without PANTS_BOOTSTRAP_TOOLS=1? Or, update this script ($INSTALL_URL)?"
esac
}
if [[ "${PANTS_BOOTSTRAP_TOOLS:-}" -gt 0 ]]; then
run_bootstrap_tools "$@"
exit 0
fi
# Ensure we operate from the context of the ./pants buildroot.
cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
pants_version="$(determine_pants_version)"
python="$(determine_python_exe "${pants_version}")"
pants_dir="$(bootstrap_pants "${pants_version}" "${python}" "${PANTS_SHA:-}")" || exit 1
pants_python="${pants_dir}/bin/python"
pants_binary="${pants_dir}/bin/pants"
pants_extra_args=""
if [[ -n "${PANTS_SHA:-}" ]]; then
pants_extra_args="${pants_extra_args} --python-repos-repos=$(find_links_url "$pants_version" "$PANTS_SHA")"
fi
# shellcheck disable=SC2086
exec "${pants_python}" "${pants_binary}" ${pants_extra_args} \
--pants-bin-name="${PANTS_BIN_NAME}" --pants-version=${pants_version} "$@"