diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000000..55e7aac3ec8 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +prune sdk/python/tests +prune ui +prune java/ +prune docs +prune infra +prune examples \ No newline at end of file diff --git a/Makefile b/Makefile index a808e3f4fb1..0ee6fcb6857 100644 --- a/Makefile +++ b/Makefile @@ -37,24 +37,24 @@ build: protos build-java build-docker # Python SDK install-python-ci-dependencies: install-go-proto-dependencies install-go-ci-dependencies - cd sdk/python && python -m piptools sync requirements/py$(PYTHON)-ci-requirements.txt - cd sdk/python && COMPILE_GO=true python setup.py develop + python -m piptools sync sdk/python/requirements/py$(PYTHON)-ci-requirements.txt + COMPILE_GO=true python setup.py develop lock-python-ci-dependencies: - cd sdk/python && python -m piptools compile -U --extra ci --output-file requirements/py$(PYTHON)-ci-requirements.txt + python -m piptools compile -U --extra ci --output-file sdk/python/requirements/py$(PYTHON)-ci-requirements.txt package-protos: cp -r ${ROOT_DIR}/protos ${ROOT_DIR}/sdk/python/feast/protos compile-protos-python: - cd sdk/python && python setup.py build_python_protos + python setup.py build_python_protos install-python: - cd sdk/python && python -m piptools sync requirements/py$(PYTHON)-requirements.txt - cd sdk/python && python setup.py develop + python -m piptools sync sdk/python/requirements/py$(PYTHON)-requirements.txt + python setup.py develop lock-python-dependencies: - cd sdk/python && python -m piptools compile -U --output-file requirements/py$(PYTHON)-requirements.txt + python -m piptools compile -U --output-file sdk/python/requirements/py$(PYTHON)-requirements.txt benchmark-python: FEAST_USAGE=False IS_TEST=True python -m pytest --integration --benchmark --benchmark-autosave --benchmark-save-data sdk/python/tests @@ -164,14 +164,14 @@ install-protoc-dependencies: pip install grpcio-tools==1.44.0 mypy-protobuf==3.1.0 compile-protos-go: install-go-proto-dependencies install-protoc-dependencies - cd sdk/python && python setup.py build_go_protos + python setup.py build_go_protos compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies - cd sdk/python && COMPILE_GO=True python setup.py build_ext --inplace + COMPILE_GO=True python setup.py build_ext --inplace # Needs feast package to setup the feature store test-go: compile-protos-go - pip install -e "sdk/python[ci]" + pip install -e ".[ci]" go test ./... format-go: diff --git a/sdk/python/pyproject.toml b/pyproject.toml similarity index 100% rename from sdk/python/pyproject.toml rename to pyproject.toml diff --git a/sdk/python/README.md b/sdk/python/README.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile b/sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile index 0c342a77ce4..cf86218d584 100644 --- a/sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile @@ -7,12 +7,14 @@ COPY sdk/python/feast/infra/feature_servers/aws_lambda/app.py ${LAMBDA_TASK_ROOT COPY sdk/python sdk/python COPY protos protos COPY go go +COPY setup.py setup.py +COPY pyproject.toml pyproject.toml COPY README.md README.md # Install Feast for AWS with Lambda dependencies # TODO(felixwang9817): Remove Snowflake dependencies once lazy loading of offline stores is supported. # See https://github.com/feast-dev/feast/issues/2566 for more details. -RUN pip3 install -e 'sdk/python[aws,redis,snowflake]' +RUN pip3 install -e '.[aws,redis,snowflake]' RUN pip3 install -r sdk/python/feast/infra/feature_servers/aws_lambda/requirements.txt --target "${LAMBDA_TASK_ROOT}" # Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile) diff --git a/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile index b4e1e4adb97..e0e16f6a14f 100644 --- a/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile @@ -13,11 +13,13 @@ COPY sdk/python/feast/infra/feature_servers/gcp_cloudrun/app.py ./app.py # Copy necessary parts of the Feast codebase COPY sdk/python ./sdk/python COPY protos ./protos +COPY setup.py setup.py +COPY pyproject.toml pyproject.toml COPY README.md ./README.md # Install production dependencies. RUN pip install --no-cache-dir \ - -e 'sdk/python[gcp,redis]' \ + -e '.[gcp,redis]' \ -r ./sdk/python/feast/infra/feature_servers/gcp_cloudrun/requirements.txt # Run the web service on container startup. Here we use the gunicorn diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000000..e2d707e2720 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,22 @@ +[isort] +src_paths = feast,tests +multi_line_output=3 +include_trailing_comma=True +force_grid_wrap=0 +use_parentheses=True +line_length=88 +skip=feast/protos,feast/embedded_go/lib +known_first_party=feast,feast_serving_server,feast_core_server +default_section=THIRDPARTY + +[flake8] +ignore = E203, E266, E501, W503 +max-line-length = 88 +max-complexity = 20 +select = B,C,E,F,W,T4 +exclude = .git,__pycache__,docs/conf.py,dist,feast/protos,feast/embedded_go/lib + +[mypy] +files=feast,tests +ignore_missing_imports=true +exclude=feast/embedded_go/lib diff --git a/sdk/python/setup.py b/setup.py similarity index 92% rename from sdk/python/setup.py rename to setup.py index c9e9889d74d..1ecc3df8209 100644 --- a/sdk/python/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ "Jinja2>=2.0.0", "jsonschema", "mmh3", - "numpy<1.22", # 1.22 drops support for python 3.7. + "numpy<1.22", "pandas>=1.0.0", "pandavro==1.5.*", "protobuf>=3.10,<3.20", @@ -124,6 +124,7 @@ CI_REQUIRED = ( [ + "build", "cryptography==3.4.8", "flake8", "black==19.10b0", @@ -179,7 +180,7 @@ DEV_REQUIRED = ["mypy-protobuf==3.1", "grpcio-testing==1.*"] + CI_REQUIRED # Get git repo root directory -repo_root = str(pathlib.Path(__file__).resolve().parent.parent.parent) +repo_root = str(pathlib.Path(__file__).resolve().parent) # README file from Feast repo root directory README_FILE = os.path.join(repo_root, "README.md") @@ -195,11 +196,12 @@ # Only set use_scm_version if git executable exists (setting this variable causes pip to use git under the hood) if shutil.which("git"): - use_scm_version = {"root": "../..", "relative_to": __file__, "tag_regex": TAG_REGEX} + use_scm_version = {"root": ".", "relative_to": __file__, "tag_regex": TAG_REGEX} else: use_scm_version = None PROTO_SUBDIRS = ["core", "serving", "types", "storage"] +PYTHON_CODE_PREFIX = "sdk/python" class BuildPythonProtosCommand(Command): @@ -214,7 +216,7 @@ def initialize_options(self): ] # find_executable("protoc") self.proto_folder = os.path.join(repo_root, "protos") self.python_folder = os.path.join( - os.path.dirname(__file__) or os.getcwd(), "feast/protos" + os.path.dirname(__file__) or os.getcwd(), "sdk/python/feast/protos" ) self.sub_folders = PROTO_SUBDIRS @@ -223,7 +225,7 @@ def finalize_options(self): def _generate_python_protos(self, path: str): proto_files = glob.glob(os.path.join(self.proto_folder, path)) - Path(self.python_folder).mkdir(exist_ok=True) + Path(self.python_folder).mkdir(parents=True, exist_ok=True) subprocess.check_call( self.python_protoc + [ @@ -244,19 +246,17 @@ def run(self): self._generate_python_protos(f"feast/{sub_folder}/*.proto") # We need the __init__ files for each of the generated subdirs # so that they are regular packages, and don't need the `--namespace-packages` flags - # when being typechecked using mypy. BUT, we need to exclude `types` because that clashes - # with an existing module in the python standard library. - if sub_folder == "types": - continue + # when being typechecked using mypy. with open(f"{self.python_folder}/feast/{sub_folder}/__init__.py", 'w'): pass + with open(f"{self.python_folder}/__init__.py", 'w'): pass with open(f"{self.python_folder}/feast/__init__.py", 'w'): pass - for path in Path("feast/protos").rglob("*.py"): + for path in Path(self.python_folder).rglob("*.py"): for folder in self.sub_folders: # Read in the file with open(path, "r") as file: @@ -355,6 +355,7 @@ class BuildCommand(build_py): def run(self): self.run_command("build_python_protos") + self.run_command("build_ext") if os.getenv("COMPILE_GO", "false").lower() == "true": _ensure_go_and_proto_toolchain() self.run_command("build_go_protos") @@ -421,10 +422,17 @@ def copy_extensions_to_source(self): modpath = fullname.split('.') package = '.'.join(modpath[:-1]) package_dir = build_py.get_package_dir(package) - src = os.path.join(self.build_lib, package_dir) + + src_dir = dest_dir = package_dir + + if src_dir.startswith(PYTHON_CODE_PREFIX): + src_dir = package_dir[len(PYTHON_CODE_PREFIX):] + src_dir = src_dir.lstrip("/") + + src_dir = os.path.join(self.build_lib, src_dir) # copy whole directory - copy_tree(src, package_dir) + copy_tree(src_dir, dest_dir) setup( @@ -435,7 +443,8 @@ def copy_extensions_to_source(self): long_description_content_type="text/markdown", python_requires=REQUIRES_PYTHON, url=URL, - packages=find_packages(exclude=("tests",)), + packages=find_packages(where=PYTHON_CODE_PREFIX, exclude=("java", "infra", "sdk/python/tests", "ui")), + package_dir={"": PYTHON_CODE_PREFIX}, install_requires=REQUIRED, # https://stackoverflow.com/questions/28509965/setuptools-development-requirements # Install dev requirements with: pip install -e .[dev] @@ -474,9 +483,9 @@ def copy_extensions_to_source(self): ], package_data={ "": [ - "protos/feast/**/*.proto", - "protos/feast/third_party/grpc/health/v1/*.proto", - "feast/protos/feast/**/*.py", + "sdk/python/**/*.proto", + "sdk/python/**/*.py", + "protos/**/*.proto", ], }, cmdclass={