From 5aa777c7dcbde69c8540563e0988cdb3fccb56b2 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Wed, 1 Sep 2021 14:25:05 -0400 Subject: [PATCH 1/8] fix(ci): Remove automation to initializa Docker on Mac Docker has been doing all that is possible to prevent installing Docker Desktop without user interaction, thus, we can't run `sentry devservices up`. See https://github.com/docker/for-mac/issues/2359#issuecomment-910532550 for details. This PR removes all code that at times had managed to automate the process on Mac. We will rely on Ubuntu to test the `make bootstrap` path. We also add caching for Brew. --- .github/workflows/development-environment.yml | 47 +++++++++---------- .../workflows/files/com.docker.vmnetd.plist | 28 ----------- .github/workflows/python-deps.yml | 21 ++++++--- scripts/lib.sh | 47 ------------------- 4 files changed, 36 insertions(+), 107 deletions(-) delete mode 100644 .github/workflows/files/com.docker.vmnetd.plist diff --git a/.github/workflows/development-environment.yml b/.github/workflows/development-environment.yml index 00fb6bd36fdb31..85a1508166d924 100644 --- a/.github/workflows/development-environment.yml +++ b/.github/workflows/development-environment.yml @@ -3,6 +3,7 @@ on: pull_request: paths: - 'Makefile' + - '.github/actions/*' - '.github/workflows/development-environment.yml' - '.envrc' - 'Brewfile' @@ -17,7 +18,6 @@ jobs: timeout-minutes: 40 strategy: matrix: - # macosx-11.0 is Big Sur, however, it takes long for jobs to get started # Using Ubuntu 18 until I figure out this error: # -> ImportError: libffi.so.6: cannot open shared object file: No such file or directory os: [macos-11.0, ubuntu-18.04] @@ -31,30 +31,27 @@ jobs: - name: Checkout sentry uses: actions/checkout@v2 + - name: Set variables for caches + id: info + run: | + echo "::set-output name=brew-cache-dir::$(brew --cache)" + echo "::set-output name=yarn-cache-dir::$(yarn cache dir)" + + - name: Cache (brew) + uses: actions/cache@v2 + with: + path: ${{ steps.info.outputs.brew-cache-dir }} + key: devenv-${{ runner.os }}-brew-${{ hashFiles('Brewfile') }} + restore-keys: devenv-${{ runner.os }}-brew + - name: Install prerequisites # Xcode CLI & brew are already installed, thus, no need to call xcode-select install # Sometimes, brew needs to be updated before brew bundle would work - # After installing Docker (via homebrew) we need to make sure that it is properly initialized on Mac run: | - brew update && brew bundle -q - # This code is mentioned in our dev docs. Only remove if you adjust the docs as well - SENTRY_NO_VENV_CHECK=1 ./scripts/do.sh init-docker + HOMEBREW_NO_AUTO_UPDATE=1 brew bundle --no-upgrade || brew bundle -q - # The next few steps are to set up the cache quickly - - name: Set environment variables & others - id: info - run: | - echo "::set-output name=python-version::$(SENTRY_NO_VENV_CHECK=1 ./scripts/do.sh get-pyenv-version)" - echo "::set-output name=pip-cache-dir::$(pip3 cache dir)" - echo "::set-output name=pip-version::$(pip -V | awk -F ' ' '{print $2}')" - echo "::set-output name=yarn-cache-dir::$(yarn cache dir)" - - # In a sense, we set up Python two times (once here and once via pyenv). Setting - # it up here is instant and it helps us to get the cache primed sooner - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: ${{ steps.info.outputs.python-version }} + uses: ./.github/actions/setup-python - name: Cache (pyenv) uses: actions/cache@v2 @@ -62,12 +59,6 @@ jobs: path: ~/.pyenv key: devenv-${{ matrix.os }}-pyenv-${{ hashFiles('.python-version') }} - - name: Cache (pip) - uses: actions/cache@v2 - with: - path: ${{ steps.info.outputs.pip-cache-dir }} - key: devenv-${{ matrix.os }}-py${{ steps.info.outputs.python-version }}-pip${{ steps.info.outputs.pip-version }}-${{ hashFiles('**/requirements.txt') }} - - name: Cache (yarn) uses: actions/cache@v1 # We are explicitly using v1 due to perf reasons with: @@ -83,7 +74,11 @@ jobs: eval "$(pyenv init --path)" python -m venv .venv source .venv/bin/activate - make bootstrap + if [ docker system info &>/dev/null ]; then + make bootstrap + else + make develop init-config + fi - name: Test direnv run: | diff --git a/.github/workflows/files/com.docker.vmnetd.plist b/.github/workflows/files/com.docker.vmnetd.plist deleted file mode 100644 index 27bb3848d9b010..00000000000000 --- a/.github/workflows/files/com.docker.vmnetd.plist +++ /dev/null @@ -1,28 +0,0 @@ - - - - - Label - com.docker.vmnetd - Program - /Library/PrivilegedHelperTools/com.docker.vmnetd - ProgramArguments - - /Library/PrivilegedHelperTools/com.docker.vmnetd - - RunAtLoad - - Sockets - - Listener - - SockPathMode - 438 - SockPathName - /var/run/com.docker.vmnetd.sock - - - Version - 59 - - diff --git a/.github/workflows/python-deps.yml b/.github/workflows/python-deps.yml index 654d9b8a3c7a28..823b7d2ac4076d 100644 --- a/.github/workflows/python-deps.yml +++ b/.github/workflows/python-deps.yml @@ -2,6 +2,7 @@ name: python deps on: pull_request: paths: + - '.github/actions/*' - '.github/workflows/python-deps.yml' - 'requirements*' @@ -26,17 +27,25 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Set brew cache path + id: brew-info + run: | + echo "::set-output name=brew-cache-dir::$(brew --cache)" + + - name: Cache (brew) + uses: actions/cache@v2 + with: + path: ${{ steps.brew-info.outputs.brew-cache-dir }} + key: devenv-${{ runner.os }}-brew-${{ hashFiles('Brewfile') }} + restore-keys: devenv-${{ runner.os }}-brew + - name: Install prerequisites # Sometimes, brew needs to be updated before brew bundle would work run: | - brew update && brew bundle -q + HOMEBREW_NO_AUTO_UPDATE=1 brew bundle --no-upgrade || brew bundle -q - - name: Setup python - id: setup-python + - name: Setup Python uses: ./.github/actions/setup-python - with: - # XXX: We need to pass this python-deps-${{ matrix.os }}-py${{ matrix.python-version }}-${{ hashFiles('requirements-*.txt') }} - cache-files-hash: ${{ hashFiles('requirements-*.txt') }} - name: Install dependencies run: | diff --git a/scripts/lib.sh b/scripts/lib.sh index c6c45e56270735..595886f70d1c35 100755 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -70,53 +70,6 @@ sudo-askpass() { fi } -# After using homebrew to install docker, we need to do some magic to remove the need to interact with the GUI -# See: https://github.com/docker/for-mac/issues/2359#issuecomment-607154849 for why we need to do things below -init-docker() { - # Need to start docker if it was freshly installed or updated - # You will know that Docker is ready for devservices when the icon on the menu bar stops flashing - if query-mac && ! require docker && [ -d "/Applications/Docker.app" ]; then - echo "Making some changes to complete Docker initialization" - # allow the app to run without confirmation - xattr -d -r com.apple.quarantine /Applications/Docker.app - - # preemptively do docker.app's setup to avoid any gui prompts - # This path is not available for brand new MacBooks - sudo-askpass /bin/mkdir -p /Library/PrivilegedHelperTools - sudo-askpass /bin/chmod 754 /Library/PrivilegedHelperTools - sudo-askpass /bin/cp /Applications/Docker.app/Contents/Library/LaunchServices/com.docker.vmnetd /Library/PrivilegedHelperTools/ - sudo-askpass /bin/chmod 544 /Library/PrivilegedHelperTools/com.docker.vmnetd - - # This file used to be generated as part of brew's installation - if [ -f /Applications/Docker.app/Contents/Resources/com.docker.vmnetd.plist ]; then - sudo-askpass /bin/cp /Applications/Docker.app/Contents/Resources/com.docker.vmnetd.plist /Library/LaunchDaemons/ - else - sudo-askpass /bin/cp .github/workflows/files/com.docker.vmnetd.plist /Library/LaunchDaemons/ - fi - sudo-askpass /bin/chmod 644 /Library/LaunchDaemons/com.docker.vmnetd.plist - sudo-askpass /bin/launchctl load /Library/LaunchDaemons/com.docker.vmnetd.plist - fi - start-docker -} - -# This is mainly to be used by CI -# We need this for Mac since the executable docker won't work properly -# until the app is opened once -start-docker() { - if query-mac && ! docker system info &>/dev/null; then - echo "About to open Docker.app" - # At a later stage in the script, we're going to execute - # ensure_docker_server which waits for it to be ready - if ! open -g -a Docker.app; then - # If the step above fails, at least we can get some debugging information to determine why - sudo-askpass ls -l /Library/PrivilegedHelperTools/com.docker.vmnetd - ls -l /Library/LaunchDaemons/ - cat /Library/LaunchDaemons/com.docker.vmnetd.plist - ls -l /Applications/Docker.app - fi - fi -} - upgrade-pip() { pip install --upgrade "pip==21.1.2" "wheel==0.36.2" } From 817347add0364853ca533739d687a5b72e8b96ea Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Wed, 1 Sep 2021 14:41:22 -0400 Subject: [PATCH 2/8] Always update --- .github/workflows/development-environment.yml | 2 +- .github/workflows/python-deps.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/development-environment.yml b/.github/workflows/development-environment.yml index 85a1508166d924..c60ecea9b4e01d 100644 --- a/.github/workflows/development-environment.yml +++ b/.github/workflows/development-environment.yml @@ -48,7 +48,7 @@ jobs: # Xcode CLI & brew are already installed, thus, no need to call xcode-select install # Sometimes, brew needs to be updated before brew bundle would work run: | - HOMEBREW_NO_AUTO_UPDATE=1 brew bundle --no-upgrade || brew bundle -q + brew update -q && brew bundle -q - name: Setup Python uses: ./.github/actions/setup-python diff --git a/.github/workflows/python-deps.yml b/.github/workflows/python-deps.yml index 823b7d2ac4076d..b972add3f54ac8 100644 --- a/.github/workflows/python-deps.yml +++ b/.github/workflows/python-deps.yml @@ -42,7 +42,7 @@ jobs: - name: Install prerequisites # Sometimes, brew needs to be updated before brew bundle would work run: | - HOMEBREW_NO_AUTO_UPDATE=1 brew bundle --no-upgrade || brew bundle -q + brew update -q && brew bundle -q - name: Setup Python uses: ./.github/actions/setup-python From e4ce0060ddfea6dd7ed11cac98236ba9d6319909 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Wed, 1 Sep 2021 15:44:58 -0400 Subject: [PATCH 3/8] Minor change --- .github/workflows/development-environment.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/development-environment.yml b/.github/workflows/development-environment.yml index c60ecea9b4e01d..c54d7ee6c5b1f5 100644 --- a/.github/workflows/development-environment.yml +++ b/.github/workflows/development-environment.yml @@ -74,13 +74,10 @@ jobs: eval "$(pyenv init --path)" python -m venv .venv source .venv/bin/activate - if [ docker system info &>/dev/null ]; then - make bootstrap - else - make develop init-config - fi + make develop init-config - name: Test direnv run: | brew install direnv + eval "$(direnv hook zsh)" direnv allow From 7f86970137d41d3bd0085ef3ef6afb775c72342d Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Wed, 1 Sep 2021 16:29:01 -0400 Subject: [PATCH 4/8] Run zsh shell --- .github/workflows/development-environment.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/development-environment.yml b/.github/workflows/development-environment.yml index c54d7ee6c5b1f5..72808ad1831bbb 100644 --- a/.github/workflows/development-environment.yml +++ b/.github/workflows/development-environment.yml @@ -77,6 +77,8 @@ jobs: make develop init-config - name: Test direnv + shell: zsh + if: ${{ matrix.os == 'macos-11.0' }} run: | brew install direnv eval "$(direnv hook zsh)" From 5ba82706ac23c49ea62804891061d9e674afd5d5 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Thu, 2 Sep 2021 10:02:52 -0400 Subject: [PATCH 5/8] Try again --- .github/workflows/development-environment.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/development-environment.yml b/.github/workflows/development-environment.yml index 72808ad1831bbb..8be26fbeb615ce 100644 --- a/.github/workflows/development-environment.yml +++ b/.github/workflows/development-environment.yml @@ -77,9 +77,7 @@ jobs: make develop init-config - name: Test direnv - shell: zsh if: ${{ matrix.os == 'macos-11.0' }} run: | brew install direnv - eval "$(direnv hook zsh)" - direnv allow + direnv allow . From eac1a13544dd725967745da26462b4759a1f35fb Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Thu, 2 Sep 2021 10:22:19 -0400 Subject: [PATCH 6/8] More resilient brew & better python deps --- .github/workflows/development-environment.yml | 2 ++ .github/workflows/python-deps.yml | 17 ----------------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/.github/workflows/development-environment.yml b/.github/workflows/development-environment.yml index 8be26fbeb615ce..61338baf34d809 100644 --- a/.github/workflows/development-environment.yml +++ b/.github/workflows/development-environment.yml @@ -45,6 +45,8 @@ jobs: restore-keys: devenv-${{ runner.os }}-brew - name: Install prerequisites + # brew can be finicky but it does not always means that the rest of the steps would fail + if: ${{ always() }} # Xcode CLI & brew are already installed, thus, no need to call xcode-select install # Sometimes, brew needs to be updated before brew bundle would work run: | diff --git a/.github/workflows/python-deps.yml b/.github/workflows/python-deps.yml index b972add3f54ac8..b76895729b9e9e 100644 --- a/.github/workflows/python-deps.yml +++ b/.github/workflows/python-deps.yml @@ -27,23 +27,6 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set brew cache path - id: brew-info - run: | - echo "::set-output name=brew-cache-dir::$(brew --cache)" - - - name: Cache (brew) - uses: actions/cache@v2 - with: - path: ${{ steps.brew-info.outputs.brew-cache-dir }} - key: devenv-${{ runner.os }}-brew-${{ hashFiles('Brewfile') }} - restore-keys: devenv-${{ runner.os }}-brew - - - name: Install prerequisites - # Sometimes, brew needs to be updated before brew bundle would work - run: | - brew update -q && brew bundle -q - - name: Setup Python uses: ./.github/actions/setup-python From 7400365a661a8399e3b67bc89249b43176dcdfc2 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Thu, 2 Sep 2021 10:38:47 -0400 Subject: [PATCH 7/8] Do not fail --- .github/workflows/development-environment.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/development-environment.yml b/.github/workflows/development-environment.yml index 61338baf34d809..cfea5f359938f1 100644 --- a/.github/workflows/development-environment.yml +++ b/.github/workflows/development-environment.yml @@ -45,12 +45,11 @@ jobs: restore-keys: devenv-${{ runner.os }}-brew - name: Install prerequisites - # brew can be finicky but it does not always means that the rest of the steps would fail - if: ${{ always() }} + # brew can be finicky but it does not always means that the rest of the job will fail # Xcode CLI & brew are already installed, thus, no need to call xcode-select install # Sometimes, brew needs to be updated before brew bundle would work run: | - brew update -q && brew bundle -q + brew update -q && brew bundle -q || exit 0 - name: Setup Python uses: ./.github/actions/setup-python From 475dbacb5fd2617a5e254cd573d1246646e36d9b Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Thu, 2 Sep 2021 17:47:10 -0400 Subject: [PATCH 8/8] Remove direnv --- .github/workflows/development-environment.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/development-environment.yml b/.github/workflows/development-environment.yml index cfea5f359938f1..6723bfa46ccf5b 100644 --- a/.github/workflows/development-environment.yml +++ b/.github/workflows/development-environment.yml @@ -76,9 +76,3 @@ jobs: python -m venv .venv source .venv/bin/activate make develop init-config - - - name: Test direnv - if: ${{ matrix.os == 'macos-11.0' }} - run: | - brew install direnv - direnv allow .