Skip to content

Commit

Permalink
Switch the grpcio dependency to use an internally built m1 wheel for …
Browse files Browse the repository at this point in the history
…m1 macs only (#11760)

### Summary & Motivation
Providing pre-built `grpcio` wheels for m1 macs makes it easy for m1 users to install dagster. The grpc team does not publish wheels for m1 macs. This change uses our pre-built wheels from https://github.com/dagster-io/build-grpcio on m1 macs for Python 3.8 through 3.10. Note that m1 macs using rosetta python and all non m1s are not affected by this change.

### How I Tested These Changes
Local testing for m1 macs. Buildkite for everything else.
  • Loading branch information
shalabhc committed Jan 18, 2023
1 parent f6e47d8 commit a4b4295
Showing 1 changed file with 50 additions and 4 deletions.
54 changes: 50 additions & 4 deletions python_modules/dagster/setup.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import os
from pathlib import Path
from typing import Dict

Expand Down Expand Up @@ -25,6 +26,54 @@ def get_version() -> str:
return version["__version__"]


def _get_grpcio_requires():
pypi_deps = [
# grpcio>=1.48.1 has hanging/crashing issues for python 3.9 and earlier:
# https://github.com/grpc/grpc/issues/30843
# ensure version we require is >= that with which we generated the grpc code (set in dev-requirements)
"grpcio>=1.32.0,<1.48.1; python_version < '3.10'",
"grpcio>=1.32.0; python_version >= '3.10'",
]
if os.getenv("DAGSTER_M1_DEFAULT_GRPCIO"):
return pypi_deps

non_m1_marker = " and (platform_system!='Darwin' or platform_machine!='arm64')"

# on non-macs and intel-macs, we use the pypi grpcio wheel
non_m1_deps = [pypi_dep + non_m1_marker for pypi_dep in pypi_deps]
# on m1-macs, we use a grpcio wheel built by us (official wheels are not available)
m1_deps = [
(
"grpcio @"
" https://github.com/dagster-io/build-grpcio/raw/main/wheels/grpcio-1.47.2-cp38-cp38-macosx_11_0_arm64.whl"
" ; python_version == '3.8' and (platform_system=='Darwin' and"
" platform_machine=='arm64')"
),
(
"grpcio @"
" https://github.com/dagster-io/build-grpcio/raw/main/wheels/grpcio-1.47.2-cp39-cp39-macosx_11_0_arm64.whl"
" ; python_version == '3.9' and (platform_system=='Darwin' and"
" platform_machine=='arm64')"
),
(
"grpcio @"
" https://github.com/dagster-io/build-grpcio/raw/main/wheels/grpcio-1.47.2-cp310-cp310-macosx_11_0_arm64.whl"
" ; python_version == '3.10' and (platform_system=='Darwin' and"
" platform_machine=='arm64')"
),
# we do not have wheels for the following m1 python versions - fallback to PyPI
(
"grpcio>=1.32.0,<1.48.1; python_version < '3.8' and (platform_system=='Darwin' and"
" platform_machine=='arm64')"
),
(
"grpcio>=1.32.0; python_version >= '3.11' and (platform_system=='Darwin' and"
" platform_machine=='arm64')"
),
]
return non_m1_deps + m1_deps


setup(
name="dagster",
version=get_version(),
Expand Down Expand Up @@ -73,10 +122,7 @@ def get_version() -> str:
# pin around issues in specific versions of alembic that broke our migrations
"alembic>=1.2.1,!=1.6.3,!=1.7.0",
"croniter>=0.3.34",
# grpcio>=1.48.1 has hanging/crashing issues for python 3.9 and earlier: https://github.com/grpc/grpc/issues/30843
# ensure version we require is >= that with which we generated the grpc code (set in dev-requirements)
"grpcio>=1.32.0,<1.48.1; python_version < '3.10'",
"grpcio>=1.32.0; python_version >= '3.10'",
*_get_grpcio_requires(),
"grpcio-health-checking>=1.32.0,<1.44.0",
"packaging>=20.9",
"pendulum",
Expand Down

0 comments on commit a4b4295

Please sign in to comment.