From f48d3f4d05e6ea9cda41d86b71589473bb231289 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 2 May 2022 10:13:22 -0700 Subject: [PATCH 01/10] chore: Update python packaging to use build and produce sdists/wheels Signed-off-by: Achal Shah --- MANIFEST.in | 6 + Makefile | 18 +- sdk/python/README.md | 0 sdk/python/pyproject.toml | 23 -- sdk/python/setup.py | 491 -------------------------------------- setup.cfg | 22 ++ 6 files changed, 37 insertions(+), 523 deletions(-) create mode 100644 MANIFEST.in create mode 100644 sdk/python/README.md delete mode 100644 sdk/python/pyproject.toml delete mode 100644 sdk/python/setup.py create mode 100644 setup.cfg diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000000..55e7aac3ec8 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +prune sdk/python/tests +prune ui +prune java/ +prune docs +prune infra +prune examples \ No newline at end of file diff --git a/Makefile b/Makefile index a808e3f4fb1..1d9c4e3d4f3 100644 --- a/Makefile +++ b/Makefile @@ -37,24 +37,24 @@ build: protos build-java build-docker # Python SDK install-python-ci-dependencies: install-go-proto-dependencies install-go-ci-dependencies - cd sdk/python && python -m piptools sync requirements/py$(PYTHON)-ci-requirements.txt - cd sdk/python && COMPILE_GO=true python setup.py develop + python -m piptools sync sdk/python/requirements/py$(PYTHON)-ci-requirements.txt + COMPILE_GO=true python setup.py develop lock-python-ci-dependencies: - cd sdk/python && python -m piptools compile -U --extra ci --output-file requirements/py$(PYTHON)-ci-requirements.txt + python -m piptools compile -U --extra ci --output-file sdk/python/requirements/py$(PYTHON)-ci-requirements.txt package-protos: cp -r ${ROOT_DIR}/protos ${ROOT_DIR}/sdk/python/feast/protos compile-protos-python: - cd sdk/python && python setup.py build_python_protos + python setup.py build_python_protos install-python: - cd sdk/python && python -m piptools sync requirements/py$(PYTHON)-requirements.txt - cd sdk/python && python setup.py develop + python -m piptools sync sdk/python/requirements/py$(PYTHON)-requirements.txt + python setup.py develop lock-python-dependencies: - cd sdk/python && python -m piptools compile -U --output-file requirements/py$(PYTHON)-requirements.txt + python -m piptools compile -U --output-file sdk/python/requirements/py$(PYTHON)-requirements.txt benchmark-python: FEAST_USAGE=False IS_TEST=True python -m pytest --integration --benchmark --benchmark-autosave --benchmark-save-data sdk/python/tests @@ -164,10 +164,10 @@ install-protoc-dependencies: pip install grpcio-tools==1.44.0 mypy-protobuf==3.1.0 compile-protos-go: install-go-proto-dependencies install-protoc-dependencies - cd sdk/python && python setup.py build_go_protos + python setup.py build_go_protos compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies - cd sdk/python && COMPILE_GO=True python setup.py build_ext --inplace + COMPILE_GO=True python setup.py build_ext --inplace # Needs feast package to setup the feature store test-go: compile-protos-go diff --git a/sdk/python/README.md b/sdk/python/README.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml deleted file mode 100644 index 93727dc5689..00000000000 --- a/sdk/python/pyproject.toml +++ /dev/null @@ -1,23 +0,0 @@ -[tool.black] -line-length = 88 -target-version = ['py37'] -include = '\.pyi?$' -exclude = ''' -( - /( - \.eggs # exclude a few common directories in the - | \.git # root of the project - | \.hg - | \.mypy_cache - | \.tox - | \.venv - | _build - | buck-out - | build - | dist - | pb2.py - | \.pyi - | protos - )/ -) -''' diff --git a/sdk/python/setup.py b/sdk/python/setup.py deleted file mode 100644 index c9e9889d74d..00000000000 --- a/sdk/python/setup.py +++ /dev/null @@ -1,491 +0,0 @@ -# Copyright 2019 The Feast Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import copy -import glob -import json -import os -import pathlib -import re -import shutil -import subprocess -import sys -from distutils.cmd import Command -from distutils.dir_util import copy_tree -from pathlib import Path -from subprocess import CalledProcessError - -from setuptools import find_packages, Extension - -try: - from setuptools import setup - from setuptools.command.build_py import build_py - from setuptools.command.build_ext import build_ext as _build_ext - from setuptools.command.develop import develop - from setuptools.command.install import install - -except ImportError: - from distutils.command.build_py import build_py - from distutils.command.build_ext import build_ext as _build_ext - from distutils.core import setup - -NAME = "feast" -DESCRIPTION = "Python SDK for Feast" -URL = "https://github.com/feast-dev/feast" -AUTHOR = "Feast" -REQUIRES_PYTHON = ">=3.7.0" - -REQUIRED = [ - "click>=7.0.0,<8.0.2", - "colorama>=0.3.9", - "dill==0.3.*", - "fastavro>=1.1.0", - "google-api-core>=1.23.0", - "googleapis-common-protos==1.52.*", - "grpcio>=1.34.0", - "grpcio-reflection>=1.34.0", - "Jinja2>=2.0.0", - "jsonschema", - "mmh3", - "numpy<1.22", # 1.22 drops support for python 3.7. - "pandas>=1.0.0", - "pandavro==1.5.*", - "protobuf>=3.10,<3.20", - "proto-plus<1.19.7", - "pyarrow>=4.0.0", - "pydantic>=1.0.0", - "PyYAML>=5.4.*", - "tabulate==0.8.*", - "tenacity>=7.*", - "toml==0.10.*", - "tqdm==4.*", - "fastapi>=0.68.0", - "uvicorn[standard]>=0.14.0", - "proto-plus<1.19.7", - "tensorflow-metadata>=1.0.0,<2.0.0", - "dask>=2021.*,<2022.02.0", -] - -GCP_REQUIRED = [ - "google-cloud-bigquery>=2,<3", - "google-cloud-bigquery-storage >= 2.0.0", - "google-cloud-datastore>=2.1.*", - "google-cloud-storage>=1.34.*,<1.41", - "google-cloud-core>=1.4.0,<2.0.0", -] - -REDIS_REQUIRED = [ - "redis==4.2.2", - "hiredis>=2.0.0", -] - -AWS_REQUIRED = [ - "boto3>=1.17.0", - "docker>=5.0.2", -] - -SNOWFLAKE_REQUIRED = [ - "snowflake-connector-python[pandas]>=2.7.3", -] - -SPARK_REQUIRED = [ - "pyspark>=3.0.0", -] - -TRINO_REQUIRED = [ - "trino>=0.305.0,<0.400.0", -] - -POSTGRES_REQUIRED = [ - "psycopg2-binary>=2.8.3", -] - -HBASE_REQUIRED = [ - "happybase>=1.2.0", -] - -GE_REQUIRED = [ - "great_expectations>=0.14.0,<0.15.0" -] - -GO_REQUIRED = [ - "cffi==1.15.*", -] - -CI_REQUIRED = ( - [ - "cryptography==3.4.8", - "flake8", - "black==19.10b0", - "isort>=5", - "grpcio-tools==1.44.0", - "grpcio-testing==1.44.0", - "minio==7.1.0", - "mock==2.0.0", - "moto", - "mypy==0.931", - "mypy-protobuf==3.1", - "avro==1.10.0", - "gcsfs", - "urllib3>=1.25.4", - "psutil==5.9.0", - "pytest>=6.0.0", - "pytest-cov", - "pytest-xdist", - "pytest-benchmark>=3.4.1", - "pytest-lazy-fixture==0.6.3", - "pytest-timeout==1.4.2", - "pytest-ordering==0.6.*", - "pytest-mock==1.10.4", - "Sphinx!=4.0.0,<4.4.0", - "sphinx-rtd-theme", - "testcontainers>=3.5", - "adlfs==0.5.9", - "firebase-admin==4.5.2", - "pre-commit", - "assertpy==1.1", - "pip-tools", - "pybindgen", - "types-protobuf", - "types-python-dateutil", - "types-pytz", - "types-PyYAML", - "types-redis", - "types-requests", - "types-setuptools", - "types-tabulate", - ] - + GCP_REQUIRED - + REDIS_REQUIRED - + AWS_REQUIRED - + SNOWFLAKE_REQUIRED - + SPARK_REQUIRED - + POSTGRES_REQUIRED - + TRINO_REQUIRED - + GE_REQUIRED - + HBASE_REQUIRED -) - -DEV_REQUIRED = ["mypy-protobuf==3.1", "grpcio-testing==1.*"] + CI_REQUIRED - -# Get git repo root directory -repo_root = str(pathlib.Path(__file__).resolve().parent.parent.parent) - -# README file from Feast repo root directory -README_FILE = os.path.join(repo_root, "README.md") -with open(README_FILE, "r", encoding="utf8") as f: - LONG_DESCRIPTION = f.read() - -# Add Support for parsing tags that have a prefix containing '/' (ie 'sdk/go') to setuptools_scm. -# Regex modified from default tag regex in: -# https://github.com/pypa/setuptools_scm/blob/2a1b46d38fb2b8aeac09853e660bcd0d7c1bc7be/src/setuptools_scm/config.py#L9 -TAG_REGEX = re.compile( - r"^(?:[\/\w-]+)?(?P[vV]?\d+(?:\.\d+){0,2}[^\+]*)(?:\+.*)?$" -) - -# Only set use_scm_version if git executable exists (setting this variable causes pip to use git under the hood) -if shutil.which("git"): - use_scm_version = {"root": "../..", "relative_to": __file__, "tag_regex": TAG_REGEX} -else: - use_scm_version = None - -PROTO_SUBDIRS = ["core", "serving", "types", "storage"] - - -class BuildPythonProtosCommand(Command): - description = "Builds the proto files into Python files." - user_options = [] - - def initialize_options(self): - self.python_protoc = [ - sys.executable, - "-m", - "grpc_tools.protoc", - ] # find_executable("protoc") - self.proto_folder = os.path.join(repo_root, "protos") - self.python_folder = os.path.join( - os.path.dirname(__file__) or os.getcwd(), "feast/protos" - ) - self.sub_folders = PROTO_SUBDIRS - - def finalize_options(self): - pass - - def _generate_python_protos(self, path: str): - proto_files = glob.glob(os.path.join(self.proto_folder, path)) - Path(self.python_folder).mkdir(exist_ok=True) - subprocess.check_call( - self.python_protoc - + [ - "-I", - self.proto_folder, - "--python_out", - self.python_folder, - "--grpc_python_out", - self.python_folder, - "--mypy_out", - self.python_folder, - ] - + proto_files, - ) - - def run(self): - for sub_folder in self.sub_folders: - self._generate_python_protos(f"feast/{sub_folder}/*.proto") - # We need the __init__ files for each of the generated subdirs - # so that they are regular packages, and don't need the `--namespace-packages` flags - # when being typechecked using mypy. BUT, we need to exclude `types` because that clashes - # with an existing module in the python standard library. - if sub_folder == "types": - continue - with open(f"{self.python_folder}/feast/{sub_folder}/__init__.py", 'w'): - pass - - with open(f"{self.python_folder}/__init__.py", 'w'): - pass - with open(f"{self.python_folder}/feast/__init__.py", 'w'): - pass - - for path in Path("feast/protos").rglob("*.py"): - for folder in self.sub_folders: - # Read in the file - with open(path, "r") as file: - filedata = file.read() - - # Replace the target string - filedata = filedata.replace( - f"from feast.{folder}", f"from feast.protos.feast.{folder}" - ) - - # Write the file out again - with open(path, "w") as file: - file.write(filedata) - - -def _generate_path_with_gopath(): - go_path = subprocess.check_output(["go", "env", "GOPATH"]).decode("utf-8") - go_path = go_path.strip() - path_val = os.getenv("PATH") - path_val = f"{path_val}:{go_path}/bin" - - return path_val - - -def _ensure_go_and_proto_toolchain(): - try: - version = subprocess.check_output(["go", "version"]) - except Exception as e: - raise RuntimeError("Unable to find go toolchain") from e - - semver_string = re.search(r"go[\S]+", str(version)).group().lstrip("go") - parts = semver_string.split(".") - if not (int(parts[0]) >= 1 and int(parts[1]) >= 16): - raise RuntimeError(f"Go compiler too old; expected 1.16+ found {semver_string}") - - path_val = _generate_path_with_gopath() - - try: - subprocess.check_call(["protoc-gen-go", "--version"], env={ - "PATH": path_val - }) - subprocess.check_call(["protoc-gen-go-grpc", "--version"], env={ - "PATH": path_val - }) - except Exception as e: - raise RuntimeError("Unable to find go/grpc extensions for protoc") from e - - -class BuildGoProtosCommand(Command): - description = "Builds the proto files into Go files." - user_options = [] - - def initialize_options(self): - self.go_protoc = [ - sys.executable, - "-m", - "grpc_tools.protoc", - ] # find_executable("protoc") - self.proto_folder = os.path.join(repo_root, "protos") - self.go_folder = os.path.join(repo_root, "go/protos") - self.sub_folders = PROTO_SUBDIRS - self.path_val = _generate_path_with_gopath() - - def finalize_options(self): - pass - - def _generate_go_protos(self, path: str): - proto_files = glob.glob(os.path.join(self.proto_folder, path)) - - try: - subprocess.check_call( - self.go_protoc - + ["-I", self.proto_folder, - "--go_out", self.go_folder, - "--go_opt=module=github.com/feast-dev/feast/go/protos", - "--go-grpc_out", self.go_folder, - "--go-grpc_opt=module=github.com/feast-dev/feast/go/protos"] - + proto_files, - env={ - "PATH": self.path_val - } - ) - except CalledProcessError as e: - print(f"Stderr: {e.stderr}") - print(f"Stdout: {e.stdout}") - - def run(self): - go_dir = Path(repo_root) / "go" / "protos" - go_dir.mkdir(exist_ok=True) - for sub_folder in self.sub_folders: - self._generate_go_protos(f"feast/{sub_folder}/*.proto") - - -class BuildCommand(build_py): - """Custom build command.""" - - def run(self): - self.run_command("build_python_protos") - if os.getenv("COMPILE_GO", "false").lower() == "true": - _ensure_go_and_proto_toolchain() - self.run_command("build_go_protos") - - build_py.run(self) - - -class DevelopCommand(develop): - """Custom develop command.""" - - def run(self): - self.run_command("build_python_protos") - if os.getenv("COMPILE_GO", "false").lower() == "true": - _ensure_go_and_proto_toolchain() - self.run_command("build_go_protos") - - develop.run(self) - - -class build_ext(_build_ext): - def finalize_options(self) -> None: - super().finalize_options() - if os.getenv("COMPILE_GO", "false").lower() == "false": - self.extensions = [e for e in self.extensions if not self._is_go_ext(e)] - - def _is_go_ext(self, ext: Extension): - return any(source.endswith('.go') or source.startswith('github') for source in ext.sources) - - def build_extension(self, ext: Extension): - if not self._is_go_ext(ext): - # the base class may mutate `self.compiler` - compiler = copy.deepcopy(self.compiler) - self.compiler, compiler = compiler, self.compiler - try: - return _build_ext.build_extension(self, ext) - finally: - self.compiler, compiler = compiler, self.compiler - - bin_path = _generate_path_with_gopath() - go_env = json.loads( - subprocess.check_output(["go", "env", "-json"]).decode("utf-8").strip() - ) - - destination = os.path.dirname(os.path.abspath(self.get_ext_fullpath(ext.name))) - subprocess.check_call([ - "gopy", - "build", - "-output", - destination, - "-vm", - sys.executable, - "-no-make", - *ext.sources - ], env={ - "PATH": bin_path, - "CGO_LDFLAGS_ALLOW": ".*", - **go_env, - }) - - def copy_extensions_to_source(self): - build_py = self.get_finalized_command('build_py') - for ext in self.extensions: - fullname = self.get_ext_fullname(ext.name) - modpath = fullname.split('.') - package = '.'.join(modpath[:-1]) - package_dir = build_py.get_package_dir(package) - src = os.path.join(self.build_lib, package_dir) - - # copy whole directory - copy_tree(src, package_dir) - - -setup( - name=NAME, - author=AUTHOR, - description=DESCRIPTION, - long_description=LONG_DESCRIPTION, - long_description_content_type="text/markdown", - python_requires=REQUIRES_PYTHON, - url=URL, - packages=find_packages(exclude=("tests",)), - install_requires=REQUIRED, - # https://stackoverflow.com/questions/28509965/setuptools-development-requirements - # Install dev requirements with: pip install -e .[dev] - extras_require={ - "dev": DEV_REQUIRED, - "ci": CI_REQUIRED, - "gcp": GCP_REQUIRED, - "aws": AWS_REQUIRED, - "redis": REDIS_REQUIRED, - "snowflake": SNOWFLAKE_REQUIRED, - "spark": SPARK_REQUIRED, - "trino": TRINO_REQUIRED, - "postgres": POSTGRES_REQUIRED, - "ge": GE_REQUIRED, - "hbase": HBASE_REQUIRED, - "go": GO_REQUIRED, - }, - include_package_data=True, - license="Apache", - classifiers=[ - # Trove classifiers - # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - ], - entry_points={"console_scripts": ["feast=feast.cli:cli"]}, - use_scm_version=use_scm_version, - setup_requires=[ - "setuptools_scm", - "grpcio", - "grpcio-tools==1.44.0", - "mypy-protobuf==3.1", - "sphinx!=4.0.0", - ], - package_data={ - "": [ - "protos/feast/**/*.proto", - "protos/feast/third_party/grpc/health/v1/*.proto", - "feast/protos/feast/**/*.py", - ], - }, - cmdclass={ - "build_python_protos": BuildPythonProtosCommand, - "build_go_protos": BuildGoProtosCommand, - "build_py": BuildCommand, - "develop": DevelopCommand, - "build_ext": build_ext, - }, - ext_modules=[Extension('feast.embedded_go.lib._embedded', - ["github.com/feast-dev/feast/go/embedded"])], -) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000000..e2d707e2720 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,22 @@ +[isort] +src_paths = feast,tests +multi_line_output=3 +include_trailing_comma=True +force_grid_wrap=0 +use_parentheses=True +line_length=88 +skip=feast/protos,feast/embedded_go/lib +known_first_party=feast,feast_serving_server,feast_core_server +default_section=THIRDPARTY + +[flake8] +ignore = E203, E266, E501, W503 +max-line-length = 88 +max-complexity = 20 +select = B,C,E,F,W,T4 +exclude = .git,__pycache__,docs/conf.py,dist,feast/protos,feast/embedded_go/lib + +[mypy] +files=feast,tests +ignore_missing_imports=true +exclude=feast/embedded_go/lib From 4ce68f91566e697111b525b3ea49c554d57d9e1d Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 2 May 2022 10:42:47 -0700 Subject: [PATCH 02/10] D'oh Signed-off-by: Achal Shah --- pyproject.toml | 23 +++ setup.py | 497 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 520 insertions(+) create mode 100644 pyproject.toml create mode 100644 setup.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000000..93727dc5689 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,23 @@ +[tool.black] +line-length = 88 +target-version = ['py37'] +include = '\.pyi?$' +exclude = ''' +( + /( + \.eggs # exclude a few common directories in the + | \.git # root of the project + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | _build + | buck-out + | build + | dist + | pb2.py + | \.pyi + | protos + )/ +) +''' diff --git a/setup.py b/setup.py new file mode 100644 index 00000000000..d34f03e4e60 --- /dev/null +++ b/setup.py @@ -0,0 +1,497 @@ +# Copyright 2019 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import copy +import glob +import json +import os +import pathlib +import re +import shutil +import subprocess +import sys +from distutils.cmd import Command +from distutils.dir_util import copy_tree +from pathlib import Path +from subprocess import CalledProcessError + +from setuptools import find_packages, Extension + +try: + from setuptools import setup + from setuptools.command.build_py import build_py + from setuptools.command.build_ext import build_ext as _build_ext + from setuptools.command.develop import develop + from setuptools.command.install import install + +except ImportError: + from distutils.command.build_py import build_py + from distutils.command.build_ext import build_ext as _build_ext + from distutils.core import setup + +NAME = "feast" +DESCRIPTION = "Python SDK for Feast" +URL = "https://github.com/feast-dev/feast" +AUTHOR = "Feast" +REQUIRES_PYTHON = ">=3.7.0" + +REQUIRED = [ + "click>=7.0.0,<8.0.2", + "colorama>=0.3.9", + "dill==0.3.*", + "fastavro>=1.1.0", + "google-api-core>=1.23.0", + "googleapis-common-protos==1.52.*", + "grpcio>=1.34.0", + "grpcio-reflection>=1.34.0", + "Jinja2>=2.0.0", + "jsonschema", + "mmh3", + "pandas>=1.0.0", + "pandavro==1.5.*", + "protobuf>=3.10,<3.20", + "proto-plus<1.19.7", + "pyarrow>=4.0.0", + "pydantic>=1.0.0", + "PyYAML>=5.4.*", + "tabulate==0.8.*", + "tenacity>=7.*", + "toml==0.10.*", + "tqdm==4.*", + "fastapi>=0.68.0", + "uvicorn[standard]>=0.14.0", + "proto-plus<1.19.7", + "tensorflow-metadata>=1.0.0,<2.0.0", + "dask>=2021.*,<2022.02.0", +] + +GCP_REQUIRED = [ + "google-cloud-bigquery>=2,<3", + "google-cloud-bigquery-storage >= 2.0.0", + "google-cloud-datastore>=2.1.*", + "google-cloud-storage>=1.34.*,<1.41", + "google-cloud-core>=1.4.0,<2.0.0", +] + +REDIS_REQUIRED = [ + "redis==4.2.2", + "hiredis>=2.0.0", +] + +AWS_REQUIRED = [ + "boto3>=1.17.0", + "docker>=5.0.2", +] + +SNOWFLAKE_REQUIRED = [ + "snowflake-connector-python[pandas]>=2.7.3", +] + +SPARK_REQUIRED = [ + "pyspark>=3.0.0", +] + +TRINO_REQUIRED = [ + "trino>=0.305.0,<0.400.0", +] + +POSTGRES_REQUIRED = [ + "psycopg2-binary>=2.8.3", +] + +HBASE_REQUIRED = [ + "happybase>=1.2.0", +] + +GE_REQUIRED = [ + "great_expectations>=0.14.0,<0.15.0" +] + +GO_REQUIRED = [ + "cffi==1.15.*", +] + +CI_REQUIRED = ( + [ + "build", + "cryptography==3.4.8", + "flake8", + "black==19.10b0", + "isort>=5", + "grpcio-tools==1.44.0", + "grpcio-testing==1.44.0", + "minio==7.1.0", + "mock==2.0.0", + "moto", + "mypy==0.931", + "mypy-protobuf==3.1", + "avro==1.10.0", + "gcsfs", + "urllib3>=1.25.4", + "psutil==5.9.0", + "pytest>=6.0.0", + "pytest-cov", + "pytest-xdist", + "pytest-benchmark>=3.4.1", + "pytest-lazy-fixture==0.6.3", + "pytest-timeout==1.4.2", + "pytest-ordering==0.6.*", + "pytest-mock==1.10.4", + "Sphinx!=4.0.0,<4.4.0", + "sphinx-rtd-theme", + "testcontainers>=3.5", + "adlfs==0.5.9", + "firebase-admin==4.5.2", + "pre-commit", + "assertpy==1.1", + "pip-tools", + "pybindgen", + "types-protobuf", + "types-python-dateutil", + "types-pytz", + "types-PyYAML", + "types-redis", + "types-requests", + "types-setuptools", + "types-tabulate", + ] + + GCP_REQUIRED + + REDIS_REQUIRED + + AWS_REQUIRED + + SNOWFLAKE_REQUIRED + + SPARK_REQUIRED + + POSTGRES_REQUIRED + + TRINO_REQUIRED + + GE_REQUIRED + + HBASE_REQUIRED +) + +DEV_REQUIRED = ["mypy-protobuf==3.1", "grpcio-testing==1.*"] + CI_REQUIRED + +# Get git repo root directory +repo_root = str(pathlib.Path(__file__).resolve().parent) + +# README file from Feast repo root directory +README_FILE = os.path.join(repo_root, "README.md") +with open(README_FILE, "r", encoding="utf8") as f: + LONG_DESCRIPTION = f.read() + +# Add Support for parsing tags that have a prefix containing '/' (ie 'sdk/go') to setuptools_scm. +# Regex modified from default tag regex in: +# https://github.com/pypa/setuptools_scm/blob/2a1b46d38fb2b8aeac09853e660bcd0d7c1bc7be/src/setuptools_scm/config.py#L9 +TAG_REGEX = re.compile( + r"^(?:[\/\w-]+)?(?P[vV]?\d+(?:\.\d+){0,2}[^\+]*)(?:\+.*)?$" +) + +# Only set use_scm_version if git executable exists (setting this variable causes pip to use git under the hood) +if shutil.which("git"): + use_scm_version = {"root": ".", "relative_to": __file__, "tag_regex": TAG_REGEX} +else: + use_scm_version = None + +PROTO_SUBDIRS = ["core", "serving", "types", "storage"] + + +class BuildPythonProtosCommand(Command): + description = "Builds the proto files into Python files." + user_options = [] + + def initialize_options(self): + self.python_protoc = [ + sys.executable, + "-m", + "grpc_tools.protoc", + ] # find_executable("protoc") + self.proto_folder = os.path.join(repo_root, "protos") + self.python_folder = os.path.join( + os.path.dirname(__file__) or os.getcwd(), "sdk/python/feast/protos" + ) + self.sub_folders = PROTO_SUBDIRS + + def finalize_options(self): + pass + + def _generate_python_protos(self, path: str): + proto_files = glob.glob(os.path.join(self.proto_folder, path)) + Path(self.python_folder).mkdir(parents=True, exist_ok=True) + subprocess.check_call( + self.python_protoc + + [ + "-I", + self.proto_folder, + "--python_out", + self.python_folder, + "--grpc_python_out", + self.python_folder, + "--mypy_out", + self.python_folder, + ] + + proto_files, + ) + + def run(self): + for sub_folder in self.sub_folders: + self._generate_python_protos(f"feast/{sub_folder}/*.proto") + # We need the __init__ files for each of the generated subdirs + # so that they are regular packages, and don't need the `--namespace-packages` flags + # when being typechecked using mypy. BUT, we need to exclude `types` because that clashes + # with an existing module in the python standard library. + + # if sub_folder == "types": + # continue + with open(f"{self.python_folder}/feast/{sub_folder}/__init__.py", 'w'): + pass + + + with open(f"{self.python_folder}/__init__.py", 'w'): + pass + with open(f"{self.python_folder}/feast/__init__.py", 'w'): + pass + + gend_files = list(Path(self.python_folder).rglob("*.py")) + + for path in Path(self.python_folder).rglob("*.py"): + for folder in self.sub_folders: + # Read in the file + with open(path, "r") as file: + filedata = file.read() + + # Replace the target string + filedata = filedata.replace( + f"from feast.{folder}", f"from feast.protos.feast.{folder}" + ) + + # Write the file out again + with open(path, "w") as file: + file.write(filedata) + print(f"Generated files: {gend_files}") + + +def _generate_path_with_gopath(): + go_path = subprocess.check_output(["go", "env", "GOPATH"]).decode("utf-8") + go_path = go_path.strip() + path_val = os.getenv("PATH") + path_val = f"{path_val}:{go_path}/bin" + + return path_val + + +def _ensure_go_and_proto_toolchain(): + try: + version = subprocess.check_output(["go", "version"]) + except Exception as e: + raise RuntimeError("Unable to find go toolchain") from e + + semver_string = re.search(r"go[\S]+", str(version)).group().lstrip("go") + parts = semver_string.split(".") + if not (int(parts[0]) >= 1 and int(parts[1]) >= 16): + raise RuntimeError(f"Go compiler too old; expected 1.16+ found {semver_string}") + + path_val = _generate_path_with_gopath() + + try: + subprocess.check_call(["protoc-gen-go", "--version"], env={ + "PATH": path_val + }) + subprocess.check_call(["protoc-gen-go-grpc", "--version"], env={ + "PATH": path_val + }) + except Exception as e: + raise RuntimeError("Unable to find go/grpc extensions for protoc") from e + + +class BuildGoProtosCommand(Command): + description = "Builds the proto files into Go files." + user_options = [] + + def initialize_options(self): + self.go_protoc = [ + sys.executable, + "-m", + "grpc_tools.protoc", + ] # find_executable("protoc") + self.proto_folder = os.path.join(repo_root, "protos") + self.go_folder = os.path.join(repo_root, "go/protos") + self.sub_folders = PROTO_SUBDIRS + self.path_val = _generate_path_with_gopath() + + def finalize_options(self): + pass + + def _generate_go_protos(self, path: str): + proto_files = glob.glob(os.path.join(self.proto_folder, path)) + + try: + subprocess.check_call( + self.go_protoc + + ["-I", self.proto_folder, + "--go_out", self.go_folder, + "--go_opt=module=github.com/feast-dev/feast/go/protos", + "--go-grpc_out", self.go_folder, + "--go-grpc_opt=module=github.com/feast-dev/feast/go/protos"] + + proto_files, + env={ + "PATH": self.path_val + } + ) + except CalledProcessError as e: + print(f"Stderr: {e.stderr}") + print(f"Stdout: {e.stdout}") + + def run(self): + go_dir = Path(repo_root) / "go" / "protos" + go_dir.mkdir(exist_ok=True) + for sub_folder in self.sub_folders: + self._generate_go_protos(f"feast/{sub_folder}/*.proto") + + +class BuildCommand(build_py): + """Custom build command.""" + + def run(self): + self.run_command("build_python_protos") + if os.getenv("COMPILE_GO", "false").lower() == "true": + _ensure_go_and_proto_toolchain() + self.run_command("build_go_protos") + + build_py.run(self) + + +class DevelopCommand(develop): + """Custom develop command.""" + + def run(self): + self.run_command("build_python_protos") + if os.getenv("COMPILE_GO", "false").lower() == "true": + _ensure_go_and_proto_toolchain() + self.run_command("build_go_protos") + + develop.run(self) + + +class build_ext(_build_ext): + def finalize_options(self) -> None: + super().finalize_options() + if os.getenv("COMPILE_GO", "false").lower() == "false": + self.extensions = [e for e in self.extensions if not self._is_go_ext(e)] + + def _is_go_ext(self, ext: Extension): + return any(source.endswith('.go') or source.startswith('github') for source in ext.sources) + + def build_extension(self, ext: Extension): + if not self._is_go_ext(ext): + # the base class may mutate `self.compiler` + compiler = copy.deepcopy(self.compiler) + self.compiler, compiler = compiler, self.compiler + try: + return _build_ext.build_extension(self, ext) + finally: + self.compiler, compiler = compiler, self.compiler + + bin_path = _generate_path_with_gopath() + go_env = json.loads( + subprocess.check_output(["go", "env", "-json"]).decode("utf-8").strip() + ) + + destination = os.path.dirname(os.path.abspath(self.get_ext_fullpath(ext.name))) + subprocess.check_call([ + "gopy", + "build", + "-output", + destination, + "-vm", + sys.executable, + "-no-make", + *ext.sources + ], env={ + "PATH": bin_path, + "CGO_LDFLAGS_ALLOW": ".*", + **go_env, + }) + + def copy_extensions_to_source(self): + build_py = self.get_finalized_command('build_py') + for ext in self.extensions: + fullname = self.get_ext_fullname(ext.name) + modpath = fullname.split('.') + package = '.'.join(modpath[:-1]) + package_dir = build_py.get_package_dir(package) + src = os.path.join(self.build_lib, package_dir) + + # copy whole directory + copy_tree(src, package_dir) + + +setup( + name=NAME, + author=AUTHOR, + description=DESCRIPTION, + long_description=LONG_DESCRIPTION, + long_description_content_type="text/markdown", + python_requires=REQUIRES_PYTHON, + url=URL, + packages=find_packages(where="sdk/python", exclude=("java", "infra", "sdk/python/tests", "ui")), + package_dir={"": "sdk/python"}, + install_requires=REQUIRED, + # https://stackoverflow.com/questions/28509965/setuptools-development-requirements + # Install dev requirements with: pip install -e .[dev] + extras_require={ + "dev": DEV_REQUIRED, + "ci": CI_REQUIRED, + "gcp": GCP_REQUIRED, + "aws": AWS_REQUIRED, + "redis": REDIS_REQUIRED, + "snowflake": SNOWFLAKE_REQUIRED, + "spark": SPARK_REQUIRED, + "trino": TRINO_REQUIRED, + "postgres": POSTGRES_REQUIRED, + "ge": GE_REQUIRED, + "hbase": HBASE_REQUIRED, + "go": GO_REQUIRED, + }, + include_package_data=True, + license="Apache", + classifiers=[ + # Trove classifiers + # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + ], + entry_points={"console_scripts": ["feast=feast.cli:cli"]}, + use_scm_version=use_scm_version, + setup_requires=[ + "setuptools_scm", + "grpcio", + "grpcio-tools==1.44.0", + "mypy-protobuf==3.1", + "sphinx!=4.0.0", + ], + package_data={ + "": [ + "sdk/python/**/*.proto", + "sdk/python/**/*.py", + "protos/**/*.proto", + ], + }, + cmdclass={ + "build_python_protos": BuildPythonProtosCommand, + "build_go_protos": BuildGoProtosCommand, + "build_py": BuildCommand, + "develop": DevelopCommand, + "build_ext": build_ext, + }, + ext_modules=[Extension('feast.embedded_go.lib._embedded', + ["github.com/feast-dev/feast/go/embedded"])], +) From 0678f1518b2d0ffe522ca72d02604480086dcb67 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 2 May 2022 10:46:42 -0700 Subject: [PATCH 03/10] fix path Signed-off-by: Achal Shah --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1d9c4e3d4f3..0ee6fcb6857 100644 --- a/Makefile +++ b/Makefile @@ -171,7 +171,7 @@ compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies # Needs feast package to setup the feature store test-go: compile-protos-go - pip install -e "sdk/python[ci]" + pip install -e ".[ci]" go test ./... format-go: From 24eecf7db20609a8432830122733d214d2782877 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 2 May 2022 12:07:12 -0700 Subject: [PATCH 04/10] fix path Signed-off-by: Achal Shah --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d34f03e4e60..258a034cecc 100644 --- a/setup.py +++ b/setup.py @@ -426,8 +426,9 @@ def copy_extensions_to_source(self): modpath = fullname.split('.') package = '.'.join(modpath[:-1]) package_dir = build_py.get_package_dir(package) - src = os.path.join(self.build_lib, package_dir) + src = os.path.dirname(os.path.abspath(self.get_ext_fullpath(ext.name))) + # src = os.path.join(self.build_lib, package_dir) # copy whole directory copy_tree(src, package_dir) From 03a1f328e0f5838a01658089fc0e9b5bf7ed5be0 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 2 May 2022 15:30:08 -0700 Subject: [PATCH 05/10] fix path again Signed-off-by: Achal Shah --- setup.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 258a034cecc..6685575ef09 100644 --- a/setup.py +++ b/setup.py @@ -200,6 +200,7 @@ use_scm_version = None PROTO_SUBDIRS = ["core", "serving", "types", "storage"] +PYTHON_CODE_PREFIX = "sdk/python" class BuildPythonProtosCommand(Command): @@ -426,9 +427,13 @@ def copy_extensions_to_source(self): modpath = fullname.split('.') package = '.'.join(modpath[:-1]) package_dir = build_py.get_package_dir(package) - src = os.path.dirname(os.path.abspath(self.get_ext_fullpath(ext.name))) - # src = os.path.join(self.build_lib, package_dir) + if package_dir.startswith(PYTHON_CODE_PREFIX): + package_dir = package_dir[len(PYTHON_CODE_PREFIX):] + package_dir = package_dir.lstrip("/") + + src = os.path.join(self.build_lib, package_dir) + # copy whole directory copy_tree(src, package_dir) @@ -441,8 +446,8 @@ def copy_extensions_to_source(self): long_description_content_type="text/markdown", python_requires=REQUIRES_PYTHON, url=URL, - packages=find_packages(where="sdk/python", exclude=("java", "infra", "sdk/python/tests", "ui")), - package_dir={"": "sdk/python"}, + packages=find_packages(where=PYTHON_CODE_PREFIX, exclude=("java", "infra", "sdk/python/tests", "ui")), + package_dir={"": PYTHON_CODE_PREFIX}, install_requires=REQUIRED, # https://stackoverflow.com/questions/28509965/setuptools-development-requirements # Install dev requirements with: pip install -e .[dev] From d6910877cbb052fc5f4fcec27af92e5377613266 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 2 May 2022 15:36:31 -0700 Subject: [PATCH 06/10] fix Dockerfile Signed-off-by: Achal Shah --- sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile | 2 +- sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile b/sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile index 0c342a77ce4..fc326982c4f 100644 --- a/sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile @@ -12,7 +12,7 @@ COPY README.md README.md # Install Feast for AWS with Lambda dependencies # TODO(felixwang9817): Remove Snowflake dependencies once lazy loading of offline stores is supported. # See https://github.com/feast-dev/feast/issues/2566 for more details. -RUN pip3 install -e 'sdk/python[aws,redis,snowflake]' +RUN pip3 install -e '.[aws,redis,snowflake]' RUN pip3 install -r sdk/python/feast/infra/feature_servers/aws_lambda/requirements.txt --target "${LAMBDA_TASK_ROOT}" # Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile) diff --git a/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile index b4e1e4adb97..0427dbb6d5d 100644 --- a/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile @@ -17,7 +17,7 @@ COPY README.md ./README.md # Install production dependencies. RUN pip install --no-cache-dir \ - -e 'sdk/python[gcp,redis]' \ + -e '.[gcp,redis]' \ -r ./sdk/python/feast/infra/feature_servers/gcp_cloudrun/requirements.txt # Run the web service on container startup. Here we use the gunicorn From f1b5e7e1240b08ec923b03e2ee33f80f97687a44 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 2 May 2022 15:49:51 -0700 Subject: [PATCH 07/10] invoke build_ext in build Signed-off-by: Achal Shah --- setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 6685575ef09..c7e1b604d25 100644 --- a/setup.py +++ b/setup.py @@ -259,8 +259,6 @@ def run(self): with open(f"{self.python_folder}/feast/__init__.py", 'w'): pass - gend_files = list(Path(self.python_folder).rglob("*.py")) - for path in Path(self.python_folder).rglob("*.py"): for folder in self.sub_folders: # Read in the file @@ -275,7 +273,6 @@ def run(self): # Write the file out again with open(path, "w") as file: file.write(filedata) - print(f"Generated files: {gend_files}") def _generate_path_with_gopath(): @@ -361,6 +358,7 @@ class BuildCommand(build_py): def run(self): self.run_command("build_python_protos") + self.run_command("build_ext") if os.getenv("COMPILE_GO", "false").lower() == "true": _ensure_go_and_proto_toolchain() self.run_command("build_go_protos") From 6bf0e00c052bfee88a6a0853d334d0758b402e51 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 2 May 2022 16:08:19 -0700 Subject: [PATCH 08/10] fix src_dir and dest_dir Signed-off-by: Achal Shah --- setup.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index c7e1b604d25..74ed43e9cee 100644 --- a/setup.py +++ b/setup.py @@ -426,14 +426,16 @@ def copy_extensions_to_source(self): package = '.'.join(modpath[:-1]) package_dir = build_py.get_package_dir(package) - if package_dir.startswith(PYTHON_CODE_PREFIX): - package_dir = package_dir[len(PYTHON_CODE_PREFIX):] - package_dir = package_dir.lstrip("/") + src_dir = dest_dir = package_dir - src = os.path.join(self.build_lib, package_dir) + if src_dir.startswith(PYTHON_CODE_PREFIX): + src_dir = package_dir[len(PYTHON_CODE_PREFIX):] + src_dir = src_dir.lstrip("/") + + src_dir = os.path.join(self.build_lib, src_dir) # copy whole directory - copy_tree(src, package_dir) + copy_tree(src_dir, dest_dir) setup( From 232e25ef6fe4cf7774dc7e9d43e8451c51d9dcca Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 2 May 2022 21:19:25 -0700 Subject: [PATCH 09/10] fix dockerfiles Signed-off-by: Achal Shah --- sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile | 2 ++ sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile | 2 ++ 2 files changed, 4 insertions(+) diff --git a/sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile b/sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile index fc326982c4f..cf86218d584 100644 --- a/sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile @@ -7,6 +7,8 @@ COPY sdk/python/feast/infra/feature_servers/aws_lambda/app.py ${LAMBDA_TASK_ROOT COPY sdk/python sdk/python COPY protos protos COPY go go +COPY setup.py setup.py +COPY pyproject.toml pyproject.toml COPY README.md README.md # Install Feast for AWS with Lambda dependencies diff --git a/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile index 0427dbb6d5d..e0e16f6a14f 100644 --- a/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile @@ -13,6 +13,8 @@ COPY sdk/python/feast/infra/feature_servers/gcp_cloudrun/app.py ./app.py # Copy necessary parts of the Feast codebase COPY sdk/python ./sdk/python COPY protos ./protos +COPY setup.py setup.py +COPY pyproject.toml pyproject.toml COPY README.md ./README.md # Install production dependencies. From 86cb3438080a9d4d0c652146ac33f8165230fe59 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 3 May 2022 11:49:30 -0700 Subject: [PATCH 10/10] CR Signed-off-by: Achal Shah --- setup.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 74ed43e9cee..1ecc3df8209 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ "Jinja2>=2.0.0", "jsonschema", "mmh3", + "numpy<1.22", "pandas>=1.0.0", "pandavro==1.5.*", "protobuf>=3.10,<3.20", @@ -245,11 +246,7 @@ def run(self): self._generate_python_protos(f"feast/{sub_folder}/*.proto") # We need the __init__ files for each of the generated subdirs # so that they are regular packages, and don't need the `--namespace-packages` flags - # when being typechecked using mypy. BUT, we need to exclude `types` because that clashes - # with an existing module in the python standard library. - - # if sub_folder == "types": - # continue + # when being typechecked using mypy. with open(f"{self.python_folder}/feast/{sub_folder}/__init__.py", 'w'): pass