diff --git a/.envrc b/.envrc index 75b29ed44ada29..2a6c34a1bf3b44 100644 --- a/.envrc +++ b/.envrc @@ -3,18 +3,13 @@ # initialization (e.g. activating the venv) or giving recommendations on how to reach the desired state. # It also sets useful environment variables. # If you'd like to override or set any custom environment variables, this .envrc will read a .env file at the end. - set -e -HERE="$( +SENTRY_ROOT="$( cd "$(dirname "${BASH_SOURCE[0]}")" pwd -P )" -bold="$(tput bold)" -red="$(tput setaf 1)" -green="$(tput setaf 2)" -yellow="$(tput setaf 3)" -reset="$(tput sgr0)" +source "${SENTRY_ROOT}/scripts/lib.sh" # XXX: we can't trap bash EXIT, because it'll override direnv's finalizing routines. # consequently, using "exit" anywhere will skip this notice from showing. @@ -58,10 +53,6 @@ report_to_sentry() { rm "$_SENTRY_LOG_FILE" } -require() { - command -v "$1" >/dev/null 2>&1 -} - debug() { if [ "${SENTRY_DIRENV_DEBUG}" ]; then echo -e "${@}" @@ -86,6 +77,27 @@ die() { return 1 } +prompt_python_venv_creation() { + echo -e "${yellow}You are missing a Python virtualenv and we ${bold}need${reset}${yellow} to run a bootstrapping script (it can take a few minutes)" + info "About to create ${venv_name}..." + echo -e "\nContinue (y/N)?" + read -r resp + case "$resp" in + y | Y) echo "Okay, let's do this." ;; + *) + die "Aborted!" + ;; + esac +} + +show_commands_info() { + echo -e "\n${red}Run the following commands to bring your environment up-to-date:" + for cmd in "${commands_to_run[@]}"; do + warn " ${red}$cmd" + done + echo "" +} + ### Environment ### commands_to_run=() @@ -111,9 +123,9 @@ export SENTRY_UI_HOT_RELOAD=1 ### You can override the exported variables with a .env file # All exports should happen before here unless they're safeguarded (see devenv error reporting below) -if [ -f "${HERE}/.env" ]; then - debug "Loading variables from ${HERE}/.env" - dotenv "${HERE}/.env" +if [ -f "${SENTRY_ROOT}/.env" ]; then + info "Loading variables from ${SENTRY_ROOT}/.env" + dotenv "${SENTRY_ROOT}/.env" fi ## Notify of reporting to Sentry @@ -156,15 +168,7 @@ make setup-git-config venv_name=".venv" if [ ! -f "${venv_name}/bin/activate" ]; then - warn "You are missing a Python virtualenv and we ${bold}need${reset} to run a bootstrapping script (it can take a few minutes)" - echo -e "\nContinue (y/N)?" - read -r resp - case "$resp" in - y | Y) echo "Okay, let's do this." ;; - *) - die "Aborted!" - ;; - esac + prompt_python_venv_creation # This is time consuming but it has to be done source ./scripts/bootstrap-py3-venv fi @@ -181,7 +185,7 @@ source "${venv_name}/bin/activate" unset PS1 debug "Ensuring proper virtualenv..." -"${HERE}/scripts/ensure-venv.sh" +"${SENTRY_ROOT}/scripts/ensure-venv.sh" if ! require sentry; then warn "Your virtualenv is activated, but sentry doesn't seem to be installed." @@ -216,11 +220,7 @@ PATH_add node_modules/.bin # These are commands that can take a significant amount of time if [ ${#commands_to_run[@]} -ne 0 ]; then - echo -e "\n${red}Run the following commands to bring your environment up-to-date:" - for cmd in "${commands_to_run[@]}"; do - warn " ${red}$cmd" - done - echo "" + show_commands_info fi if [ "${log_level}" != "info" ]; then diff --git a/.github/actions/setup-python/action.yml b/.github/actions/setup-python/action.yml index 24837ef5650d66..abd0e2f9789766 100644 --- a/.github/actions/setup-python/action.yml +++ b/.github/actions/setup-python/action.yml @@ -16,7 +16,7 @@ inputs: description: 'pip cache version in order to bust cache' required: false python-version: - description: "python version to install" + description: 'python version to install' required: false outputs: @@ -38,7 +38,7 @@ runs: shell: bash run: | if [ "${{ inputs.python-version }}" == "" ]; then - echo "::set-output name=python-version::$(cat .python-version)" + echo "::set-output name=python-version::$(SENTRY_NO_VENV_CHECK=1 ./scripts/do.sh get-pyenv-version)" else echo "::set-output name=python-version::${{ inputs.python-version }}" fi diff --git a/.github/workflows/development-environment.yml b/.github/workflows/development-environment.yml index 8e8c2dc58ce13c..00fb6bd36fdb31 100644 --- a/.github/workflows/development-environment.yml +++ b/.github/workflows/development-environment.yml @@ -2,13 +2,13 @@ name: dev env on: pull_request: paths: - - 'Makefile' - - '.github/workflows/development-environment.yml' - - '.envrc' - - 'Brewfile' - - 'scripts/*' - - 'src/sentry/runner/commands/devserver.py' - - 'src/sentry/runner/commands/devservices.py' + - 'Makefile' + - '.github/workflows/development-environment.yml' + - '.envrc' + - 'Brewfile' + - 'scripts/*' + - 'src/sentry/runner/commands/devserver.py' + - 'src/sentry/runner/commands/devservices.py' jobs: dev-environment: @@ -20,7 +20,7 @@ jobs: # macosx-11.0 is Big Sur, however, it takes long for jobs to get started # Using Ubuntu 18 until I figure out this error: # -> ImportError: libffi.so.6: cannot open shared object file: No such file or directory - os: [ macos-11.0, ubuntu-18.04 ] + os: [macos-11.0, ubuntu-18.04] fail-fast: false env: PIP_DISABLE_PIP_VERSION_CHECK: on @@ -36,7 +36,7 @@ jobs: # Sometimes, brew needs to be updated before brew bundle would work # After installing Docker (via homebrew) we need to make sure that it is properly initialized on Mac run: | - brew update -q && brew bundle -q + brew update && brew bundle -q # This code is mentioned in our dev docs. Only remove if you adjust the docs as well SENTRY_NO_VENV_CHECK=1 ./scripts/do.sh init-docker @@ -44,7 +44,7 @@ jobs: - name: Set environment variables & others id: info run: | - echo "::set-output name=python-version::$(cat .python-version)" + echo "::set-output name=python-version::$(SENTRY_NO_VENV_CHECK=1 ./scripts/do.sh get-pyenv-version)" echo "::set-output name=pip-cache-dir::$(pip3 cache dir)" echo "::set-output name=pip-version::$(pip -V | awk -F ' ' '{print $2}')" echo "::set-output name=yarn-cache-dir::$(yarn cache dir)" diff --git a/.github/workflows/python-deps.yml b/.github/workflows/python-deps.yml index 9ccbb822c948dd..654d9b8a3c7a28 100644 --- a/.github/workflows/python-deps.yml +++ b/.github/workflows/python-deps.yml @@ -2,8 +2,8 @@ name: python deps on: pull_request: paths: - - '.github/workflows/python-deps.yml' - - 'requirements*' + - '.github/workflows/python-deps.yml' + - 'requirements*' jobs: # This workflow makes sure that Python dependencies install correctly for @@ -14,8 +14,8 @@ jobs: timeout-minutes: 20 strategy: matrix: - os: [ macos-11.0, ubuntu-20.04 ] - python-version: [ 3.6.13, 3.8.10 ] + os: [macos-11.0, ubuntu-20.04] + python-version: [3.6.13, 3.8.11] fail-fast: false env: PIP_DISABLE_PIP_VERSION_CHECK: on @@ -27,25 +27,16 @@ jobs: - uses: actions/checkout@v2 - name: Install prerequisites + # Sometimes, brew needs to be updated before brew bundle would work run: | - HOMEBREW_NO_AUTO_UPDATE=1 brew bundle --no-upgrade + brew update && brew bundle -q - # Until GH composite actions can use `uses`, we need to setup python here - - uses: actions/setup-python@v2 + - name: Setup python + id: setup-python + uses: ./.github/actions/setup-python with: - python-version: ${{ matrix.python-version }} - - - name: Setup pip - uses: ./.github/actions/setup-pip - id: pip - - - name: Cache - uses: actions/cache@v2 - with: - path: | - ${{ steps.pip.outputs.pip-cache-dir }} - key: | - python-deps-${{ matrix.os }}-py${{ matrix.python-version }}-${{ hashFiles('requirements-*.txt') }} + # XXX: We need to pass this python-deps-${{ matrix.os }}-py${{ matrix.python-version }}-${{ hashFiles('requirements-*.txt') }} + cache-files-hash: ${{ hashFiles('requirements-*.txt') }} - name: Install dependencies run: | diff --git a/.python-version b/.python-version deleted file mode 100644 index 3609cf77078c22..00000000000000 --- a/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.6.13 diff --git a/requirements-base.txt b/requirements-base.txt index 485ef639a2f235..8013198faa7322 100644 --- a/requirements-base.txt +++ b/requirements-base.txt @@ -20,6 +20,9 @@ google-cloud-core==1.5.0 googleapis-common-protos==1.52.0 google-cloud-pubsub==2.2.0 google-cloud-storage==1.35.0 +# Only necessary to prevent installing the latest version +# https://github.com/googleapis/python-crc32c/issues/83 +google-crc32c==1.1.2; python_version == '3.8' jsonschema==3.2.0 lxml==4.6.3 maxminddb==2.0.3 diff --git a/scripts/bootstrap-py3-venv b/scripts/bootstrap-py3-venv index f62ca0ea3f0ace..3b8f583e12fd5b 100644 --- a/scripts/bootstrap-py3-venv +++ b/scripts/bootstrap-py3-venv @@ -11,6 +11,11 @@ # trap "echo 'bootstrap FAILED.' && trap - ERR && return" ERR # So just going to sprinkle returns everywhere. +SCRIPTS_DIR="$( + cd "$(dirname "${BASH_SOURCE[0]}")" + pwd -P +)" +source "${SCRIPTS_DIR}/lib.sh" command -v pyenv >/dev/null || { echo "You need to install pyenv. https://develop.sentry.dev/environment/#python" @@ -24,7 +29,10 @@ command -v direnv >/dev/null || { gitroot="$(git rev-parse --show-toplevel)" cd "$gitroot" -export venv_name="${PWD}/.venv" +# This executes very quickly if the Python versions are already installed +make setup-pyenv +export PYENV_VERSION=$(SENTRY_NO_VENV_CHECK=1 ./scripts/do.sh get-pyenv-version) +venv_name="${PWD}/.venv" if [[ -f "${venv_name}/bin/activate" ]]; then prompt_text="You have an existing virtualenv. This script will OVERWRITE it. Continue (y/N)?" @@ -40,9 +48,7 @@ fi deactivate 2>/dev/null || true rm -rf "$venv_name" -# XXX: You'll also need to temporarily pyenv local "$SENTRY_PYTHON_VERSION" -# and then revert changes to the .python-version after bootstrap finishes. -if [[ -z "$SENTRY_PYTHON_VERSION" ]] && ! [[ "$(python3 -V 2>&1)" = "Python $(grep "3.6" .python-version)" ]]; then +if [[ -z "$SENTRY_PYTHON_VERSION" ]] && ! query-valid-python-version; then echo "Your python3 version isn't as expected. Please run: make setup-pyenv" return 1 fi @@ -50,7 +56,7 @@ fi python3 -m pip install -U pip || { echo "bootstrap failed!"; return 1; } python3 -m venv "${venv_name}" || { echo "bootstrap failed!"; return 1; } source "${venv_name}/bin/activate" || { echo "bootstrap failed!"; return 1; } -python3 -m pip install -U pip || { echo "bootstrap failed!"; return 1; } +python3 -m pip install -U pip wheel || { echo "bootstrap failed!"; return 1; } make setup-git || { echo "bootstrap failed!"; return 1; } make install-py-dev || { echo "bootstrap failed!"; return 1; } deactivate || { echo "bootstrap failed!"; return 1; } diff --git a/scripts/do.sh b/scripts/do.sh index 734aa045b1dcac..9a2f20c744b0fb 100755 --- a/scripts/do.sh +++ b/scripts/do.sh @@ -2,8 +2,10 @@ # This script is an interface to any of the methods of lib.sh # Call this script as "do.sh method_from_lib" to execute any function from that library set -eu - -HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")"; pwd -P)" +HERE="$( + cd "$(dirname "${BASH_SOURCE[0]}")" + pwd -P +)" # shellcheck disable=SC1090 source "${HERE}/lib.sh" diff --git a/scripts/ensure-venv.sh b/scripts/ensure-venv.sh index 92e33c392e5c67..8a0800eb9c19ba 100755 --- a/scripts/ensure-venv.sh +++ b/scripts/ensure-venv.sh @@ -1,4 +1,10 @@ #!/bin/bash +HERE="$( + cd "$(dirname "${BASH_SOURCE[0]}")" || exit + pwd -P +)" +# shellcheck disable=SC1090 +source "${HERE}/lib.sh" # optionally opt out of virtualenv creation # WARNING: this will be removed (most likely renamed) soon! @@ -6,11 +12,6 @@ if [[ "$SENTRY_NO_VIRTUALENV_CREATION" == "1" ]]; then exit 0 fi -red="$(tput setaf 1)" -yellow="$(tput setaf 3)" -bold="$(tput bold)" -reset="$(tput sgr0)" - venv_name=".venv" die() { @@ -21,33 +22,31 @@ EOF } if [[ -n "$VIRTUAL_ENV" ]]; then - minor=$(python -c "import sys; print(sys.version_info[1])") - # If .venv is less than Python 3.6 fail - [[ "$minor" -lt 6 ]] && - die "Remove $VIRTUAL_ENV and try again since the Python version installed should be at least 3.6." - # If .venv is created with Python greater than 3.6 you might encounter problems and we want to ask you to downgrade - # unless you explicitely set an environment variable - if [[ "$minor" -gt 6 ]]; then - if [[ -n "$SENTRY_PYTHON_VERSION" ]]; then - cat </dev/null; then return 0 fi return 1 } +query-apple-m1() { + query-mac && [[ $(uname -m) = 'arm64' ]] +} + +get-pyenv-version() { + local PYENV_VERSION + PYENV_VERSION=3.6.13 + if query-apple-m1; then + PYENV_VERSION=3.8.11 + fi + echo "${PYENV_VERSION}" +} + +query-valid-python-version() { + python_version=$(python3 -V 2>&1 | awk '{print $2}') + if [ "${python_version}" == 3.6.13 ] || [ "${python_version}" == 3.8.11 ]; then + return 0 + else + return 1 + fi +} + sudo-askpass() { if [ -z "${sudo-askpass-x}" ]; then sudo --askpass "$@" diff --git a/scripts/pyenv_setup.sh b/scripts/pyenv_setup.sh index db8975d39a7091..7743fd7e0ad983 100755 --- a/scripts/pyenv_setup.sh +++ b/scripts/pyenv_setup.sh @@ -3,15 +3,20 @@ # # Assumptions: # - This script assumes you're calling from the top directory of the repository +# +# shellcheck disable=SC2155 # Declare and assign separately to avoid masking return values. + set -eu HERE="$( cd "$(dirname "${BASH_SOURCE[0]}")" pwd -P )" -# shellcheck disable=SC1090 source "${HERE}/lib.sh" +# We can use PYENV_VERSION to define different Python versions, otherwise, determine load default values +[ -z ${PYENV_VERSION+x} ] && export PYENV_VERSION=$(get-pyenv-version) + get_shell_startup_script() { local _startup_script='' if [[ -n "$SHELL" ]]; then @@ -44,7 +49,8 @@ _append_to_startup_script() { case "$SHELL" in */bash) # shellcheck disable=SC2016 - echo "Visit https://github.com/pyenv/pyenv#installation on how to fully set up your Bash shell.";; + echo "Visit https://github.com/pyenv/pyenv#installation on how to fully set up your Bash shell." + ;; */zsh) # shellcheck disable=SC2016 echo -e '# It is assumed that pyenv is installed via Brew, so this is all we need to do.\n' \ @@ -52,7 +58,7 @@ _append_to_startup_script() { ;; */fish) # shellcheck disable=SC2016 - echo -e '\n# pyenv init\nstatus is-login; and pyenv init --path | source' >> "${1}" + echo -e '\n# pyenv init\nstatus is-login; and pyenv init --path | source' >>"${1}" ;; esac @@ -78,12 +84,8 @@ append_to_config() { install_pyenv() { if require pyenv; then - echo "Installing Python (if missing) via pyenv" - local pyenv_version - pyenv_version=$(pyenv -v | awk '{print $2}') - python_version=$(xargs -n1 <.python-version) # NOTE: We're dropping support for older pyenv versions - if [[ "$pyenv_version" < 2.0.0 ]]; then + if [[ "$(pyenv -v | awk '{print $2}')" < 2.0.0 ]]; then echo >&2 "!!! We've dropped support for pyenv v1." \ "Run the following (this is slow) and try again." # brew upgrade does not quite do the right thing @@ -92,14 +94,16 @@ install_pyenv() { exit 1 fi - # We need to patch the source code on Big Sur before building Python - # We can remove this once we upgrade to newer versions of Python - if query_big_sur; then + if query-apple-m1; then + pyenv install --skip-existing "${PYENV_VERSION}" + elif query-big-sur; then + # We need to patch the source code on Big Sur before building Python + # We can remove this once we upgrade to newer versions of Python # cat is used since pyenv would finish to soon when the Python version is already installed curl -sSL https://github.com/python/cpython/commit/8ea6353.patch | cat | - pyenv install --skip-existing --patch "$python_version" + pyenv install --skip-existing --patch "${PYENV_VERSION}" else - pyenv install --skip-existing "$python_version" + pyenv install --skip-existing "${PYENV_VERSION}" fi else echo >&2 "!!! pyenv not found, try running bootstrap script again or run \`brew bundle\` in the sentry repo" @@ -111,16 +115,14 @@ install_pyenv() { setup_pyenv() { configure-sentry-cli install_pyenv - _startup_script=$(get_shell_startup_script) - append_to_config "$_startup_script" + append_to_config "$(get_shell_startup_script)" # If the script is called with the "dot space right" approach (. ./scripts/pyenv_setup.sh), # the effects of this will be persistent outside of this script - echo "Activating pyenv and validating Python version" # Sets up PATH for pyenv eval "$(pyenv init --path)" python_version=$(python -V | sed s/Python\ //g) - [[ $python_version == $(cat .python-version) ]] || + [[ $python_version == "${PYENV_VERSION}" ]] || (echo "Wrong Python version: $python_version. Please report in #discuss-dev-tooling" && exit 1) }