From 87d59fe92ecb3e47ec002d26fd09651fecbfd642 Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 23 Oct 2024 13:04:49 -0400 Subject: [PATCH 01/18] rebase Signed-off-by: Rob Howley --- sdk/python/feast/feature_server.py | 2 + sdk/python/feast/feature_store.py | 8 ++ .../infra/offline_stores/offline_store.py | 6 ++ .../feast/infra/online_stores/dynamodb.py | 74 ++++++++++++++----- .../feast/infra/online_stores/online_store.py | 6 ++ .../feast/infra/passthrough_provider.py | 8 ++ sdk/python/feast/infra/provider.py | 8 ++ sdk/python/tests/foo_provider.py | 6 ++ 8 files changed, 100 insertions(+), 18 deletions(-) diff --git a/sdk/python/feast/feature_server.py b/sdk/python/feast/feature_server.py index b4ed591b04c..e6e42718abd 100644 --- a/sdk/python/feast/feature_server.py +++ b/sdk/python/feast/feature_server.py @@ -101,8 +101,10 @@ def async_refresh(): @asynccontextmanager async def lifespan(app: FastAPI): async_refresh() + await store.initialize() yield stop_refresh() + await store.close() app = FastAPI(lifespan=lifespan) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index f9fa0a78819..21dec7b06f6 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -2157,6 +2157,14 @@ def list_saved_datasets( self.project, allow_cache=allow_cache, tags=tags ) + async def initialize(self) -> None: + """Initialize long-lived clients and/or resources needed for accessing datastores""" + await self._get_provider().initialize(self.config) + + async def close(self) -> None: + """Cleanup any long-lived clients and/or resources""" + await self._get_provider().close() + def _print_materialization_log( start_date, end_date, num_feature_views: int, online_store: str diff --git a/sdk/python/feast/infra/offline_stores/offline_store.py b/sdk/python/feast/infra/offline_stores/offline_store.py index 69d6bb278b7..b3bd61b95f4 100644 --- a/sdk/python/feast/infra/offline_stores/offline_store.py +++ b/sdk/python/feast/infra/offline_stores/offline_store.py @@ -388,3 +388,9 @@ def get_table_column_names_and_types_from_data_source( data_source: DataSource object """ return data_source.get_table_column_names_and_types(config=config) + + async def initialize(self, config: RepoConfig) -> None: + pass + + async def close(self) -> None: + pass diff --git a/sdk/python/feast/infra/online_stores/dynamodb.py b/sdk/python/feast/infra/online_stores/dynamodb.py index a915d2ee34b..a97e81bc447 100644 --- a/sdk/python/feast/infra/online_stores/dynamodb.py +++ b/sdk/python/feast/infra/online_stores/dynamodb.py @@ -12,11 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. import asyncio +import contextlib import itertools import logging from datetime import datetime from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union +from aiobotocore.config import AioConfig from pydantic import StrictBool, StrictStr from feast import Entity, FeatureView, utils @@ -75,6 +77,9 @@ class DynamoDBOnlineStoreConfig(FeastConfigBaseModel): session_based_auth: bool = False """AWS session based client authentication""" + max_pool_connections: int = 10 + """Max number of connections for async Dynamodb operations""" + class DynamoDBOnlineStore(OnlineStore): """ @@ -87,7 +92,14 @@ class DynamoDBOnlineStore(OnlineStore): _dynamodb_client = None _dynamodb_resource = None - _aioboto_session = None + + async def initialize(self, config: RepoConfig): + await _get_aiodynamodb_client( + config.online_store.region, config.online_store.max_pool_connections + ) + + async def close(self): + await _aiodynamodb_close() @property def async_supported(self) -> SupportedAsyncMethods: @@ -326,15 +338,17 @@ def to_tbl_resp(raw_client_response): batches.append(batch) entity_id_batches.append(entity_id_batch) - async with self._get_aiodynamodb_client(online_config.region) as client: - response_batches = await asyncio.gather( - *[ - client.batch_get_item( - RequestItems=entity_id_batch, - ) - for entity_id_batch in entity_id_batches - ] - ) + client = await _get_aiodynamodb_client( + online_config.region, online_config.max_pool_connections + ) + response_batches = await asyncio.gather( + *[ + client.batch_get_item( + RequestItems=entity_id_batch, + ) + for entity_id_batch in entity_id_batches + ] + ) result_batches = [] for batch, response in zip(batches, response_batches): @@ -349,14 +363,6 @@ def to_tbl_resp(raw_client_response): return list(itertools.chain(*result_batches)) - def _get_aioboto_session(self): - if self._aioboto_session is None: - self._aioboto_session = session.get_session() - return self._aioboto_session - - def _get_aiodynamodb_client(self, region: str): - return self._get_aioboto_session().create_client("dynamodb", region_name=region) - def _get_dynamodb_client( self, region: str, @@ -489,6 +495,38 @@ def _to_client_batch_get_payload(online_config, table_name, batch): } +_aioboto_session = None +_aioboto_client = None + + +def _get_aioboto_session(): + global _aioboto_session + if _aioboto_session is None: + logger.debug("initializing the aiobotocore session") + _aioboto_session = session.get_session() + return _aioboto_session + + +async def _get_aiodynamodb_client(region: str, max_pool_connections: int): + global _aioboto_client + if _aioboto_client is None: + logger.debug("initializing the aiobotocore dynamodb client") + client_context = _get_aioboto_session().create_client( + "dynamodb", + region_name=region, + config=AioConfig(max_pool_connections=max_pool_connections), + ) + context_stack = contextlib.AsyncExitStack() + _aioboto_client = await context_stack.enter_async_context(client_context) + return _aioboto_client + + +async def _aiodynamodb_close(): + global _aioboto_client + if _aioboto_client: + await _aioboto_client.close() + + def _initialize_dynamodb_client( region: str, endpoint_url: Optional[str] = None, diff --git a/sdk/python/feast/infra/online_stores/online_store.py b/sdk/python/feast/infra/online_stores/online_store.py index 15dd843ba82..789885f82bc 100644 --- a/sdk/python/feast/infra/online_stores/online_store.py +++ b/sdk/python/feast/infra/online_stores/online_store.py @@ -422,3 +422,9 @@ def retrieve_online_documents( raise NotImplementedError( f"Online store {self.__class__.__name__} does not support online retrieval" ) + + async def initialize(self, config: RepoConfig) -> None: + pass + + async def close(self) -> None: + pass diff --git a/sdk/python/feast/infra/passthrough_provider.py b/sdk/python/feast/infra/passthrough_provider.py index 9482b808a9b..f59d457ec09 100644 --- a/sdk/python/feast/infra/passthrough_provider.py +++ b/sdk/python/feast/infra/passthrough_provider.py @@ -518,3 +518,11 @@ def get_table_column_names_and_types_from_data_source( return self.offline_store.get_table_column_names_and_types_from_data_source( config=config, data_source=data_source ) + + async def initialize(self, config: RepoConfig) -> None: + await self.online_store.initialize(config) + await self.offline_store.initialize(config) + + async def close(self) -> None: + await self.online_store.close() + await self.offline_store.close() diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 47b7c65ef00..8351f389ad9 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -476,6 +476,14 @@ def get_table_column_names_and_types_from_data_source( """ pass + @abstractmethod + async def initialize(self, config: RepoConfig) -> None: + pass + + @abstractmethod + async def close(self) -> None: + pass + def get_provider(config: RepoConfig) -> Provider: if "." not in config.provider: diff --git a/sdk/python/tests/foo_provider.py b/sdk/python/tests/foo_provider.py index bc30d3ef88e..570a6d4f8d5 100644 --- a/sdk/python/tests/foo_provider.py +++ b/sdk/python/tests/foo_provider.py @@ -213,3 +213,9 @@ async def online_write_batch_async( progress: Optional[Callable[[int], Any]], ) -> None: pass + + async def initialize(self, config: RepoConfig) -> None: + pass + + async def close(self) -> None: + pass From 7793d99f3f39b834f87b3f3818757025031011f2 Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Thu, 17 Oct 2024 15:40:23 -0400 Subject: [PATCH 02/18] offline store init doesnt make sense Signed-off-by: Rob Howley --- sdk/python/feast/infra/offline_stores/offline_store.py | 6 ------ sdk/python/feast/infra/passthrough_provider.py | 2 -- 2 files changed, 8 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/offline_store.py b/sdk/python/feast/infra/offline_stores/offline_store.py index b3bd61b95f4..69d6bb278b7 100644 --- a/sdk/python/feast/infra/offline_stores/offline_store.py +++ b/sdk/python/feast/infra/offline_stores/offline_store.py @@ -388,9 +388,3 @@ def get_table_column_names_and_types_from_data_source( data_source: DataSource object """ return data_source.get_table_column_names_and_types(config=config) - - async def initialize(self, config: RepoConfig) -> None: - pass - - async def close(self) -> None: - pass diff --git a/sdk/python/feast/infra/passthrough_provider.py b/sdk/python/feast/infra/passthrough_provider.py index f59d457ec09..a1e9ef82ad7 100644 --- a/sdk/python/feast/infra/passthrough_provider.py +++ b/sdk/python/feast/infra/passthrough_provider.py @@ -521,8 +521,6 @@ def get_table_column_names_and_types_from_data_source( async def initialize(self, config: RepoConfig) -> None: await self.online_store.initialize(config) - await self.offline_store.initialize(config) async def close(self) -> None: await self.online_store.close() - await self.offline_store.close() From f04711eb6c95f5c488d3598c133e819207ed1711 Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 23 Oct 2024 15:24:09 -0400 Subject: [PATCH 03/18] dont init or close Signed-off-by: Rob Howley --- sdk/python/feast/feature_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/feature_server.py b/sdk/python/feast/feature_server.py index e6e42718abd..e5e7410715e 100644 --- a/sdk/python/feast/feature_server.py +++ b/sdk/python/feast/feature_server.py @@ -101,10 +101,10 @@ def async_refresh(): @asynccontextmanager async def lifespan(app: FastAPI): async_refresh() - await store.initialize() + # await store.initialize() yield stop_refresh() - await store.close() + # await store.close() app = FastAPI(lifespan=lifespan) From d1094adc039f80eb113a0d7613a1d32fcb68dc0e Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 23 Oct 2024 16:13:02 -0400 Subject: [PATCH 04/18] update test to handle event loop for dynamo case Signed-off-by: Rob Howley --- .../integration/online_store/test_python_feature_server.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdk/python/tests/integration/online_store/test_python_feature_server.py b/sdk/python/tests/integration/online_store/test_python_feature_server.py index d08e1104eb0..8a150d0cc90 100644 --- a/sdk/python/tests/integration/online_store/test_python_feature_server.py +++ b/sdk/python/tests/integration/online_store/test_python_feature_server.py @@ -1,3 +1,4 @@ +import asyncio import json from typing import List @@ -137,5 +138,7 @@ def python_fs_client(environment, universal_data_sources, request): feast_objects.extend([driver(), customer(), location()]) fs.apply(feast_objects) fs.materialize(environment.start_date, environment.end_date) + asyncio.run(fs.initalize()) client = TestClient(get_app(fs)) yield client + asyncio.run(fs.close()) From afaa4120107e9eddb3e9acd781a57021b5ceda4b Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 23 Oct 2024 16:36:06 -0400 Subject: [PATCH 05/18] use run util complete Signed-off-by: Rob Howley --- sdk/python/feast/feature_server.py | 4 ++-- .../integration/online_store/test_python_feature_server.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/sdk/python/feast/feature_server.py b/sdk/python/feast/feature_server.py index e5e7410715e..e6e42718abd 100644 --- a/sdk/python/feast/feature_server.py +++ b/sdk/python/feast/feature_server.py @@ -101,10 +101,10 @@ def async_refresh(): @asynccontextmanager async def lifespan(app: FastAPI): async_refresh() - # await store.initialize() + await store.initialize() yield stop_refresh() - # await store.close() + await store.close() app = FastAPI(lifespan=lifespan) diff --git a/sdk/python/tests/integration/online_store/test_python_feature_server.py b/sdk/python/tests/integration/online_store/test_python_feature_server.py index 8a150d0cc90..7e6c09c1de9 100644 --- a/sdk/python/tests/integration/online_store/test_python_feature_server.py +++ b/sdk/python/tests/integration/online_store/test_python_feature_server.py @@ -138,7 +138,8 @@ def python_fs_client(environment, universal_data_sources, request): feast_objects.extend([driver(), customer(), location()]) fs.apply(feast_objects) fs.materialize(environment.start_date, environment.end_date) - asyncio.run(fs.initalize()) + loop = asyncio.get_event_loop() + loop.run_until_complete(fs.initalize()) client = TestClient(get_app(fs)) yield client - asyncio.run(fs.close()) + loop.run_until_complete(fs.close()) From 2245caa10e431a9d65af8ba14a3d0264df684900 Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 23 Oct 2024 17:05:14 -0400 Subject: [PATCH 06/18] fix: spelling sigh Signed-off-by: Rob Howley --- .../integration/online_store/test_python_feature_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/tests/integration/online_store/test_python_feature_server.py b/sdk/python/tests/integration/online_store/test_python_feature_server.py index 7e6c09c1de9..159660d20e7 100644 --- a/sdk/python/tests/integration/online_store/test_python_feature_server.py +++ b/sdk/python/tests/integration/online_store/test_python_feature_server.py @@ -139,7 +139,7 @@ def python_fs_client(environment, universal_data_sources, request): fs.apply(feast_objects) fs.materialize(environment.start_date, environment.end_date) loop = asyncio.get_event_loop() - loop.run_until_complete(fs.initalize()) + loop.run_until_complete(fs.initialize()) client = TestClient(get_app(fs)) yield client loop.run_until_complete(fs.close()) From eebf71de944c297c20c0cbfee74d47a2366838a8 Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 23 Oct 2024 20:28:26 -0400 Subject: [PATCH 07/18] run integration test as async since that is default for read Signed-off-by: Rob Howley --- sdk/python/feast/feature_server.py | 2 +- .../requirements/py3.10-ci-requirements.txt | 172 +++++++++++++--- .../requirements/py3.11-ci-requirements.txt | 191 ++++++++++++++++-- .../requirements/py3.9-ci-requirements.txt | 183 +++++++++++++---- .../test_python_feature_server.py | 9 +- 5 files changed, 471 insertions(+), 86 deletions(-) diff --git a/sdk/python/feast/feature_server.py b/sdk/python/feast/feature_server.py index e6e42718abd..f485d874e14 100644 --- a/sdk/python/feast/feature_server.py +++ b/sdk/python/feast/feature_server.py @@ -100,8 +100,8 @@ def async_refresh(): @asynccontextmanager async def lifespan(app: FastAPI): - async_refresh() await store.initialize() + async_refresh() yield stop_refresh() await store.close() diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 57a21cd6d95..0041e21703f 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -1,6 +1,7 @@ # This file was autogenerated by uv via the following command: -# uv pip compile -p 3.10 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt +# uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt aiobotocore==2.15.1 + # via feast (setup.py) aiohappyeyeballs==2.4.0 # via aiohttp aiohttp==3.10.5 @@ -21,6 +22,8 @@ anyio==4.5.0 # jupyter-server # starlette # watchfiles +appnope==0.1.4 + # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -30,6 +33,7 @@ arrow==1.3.0 asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 + # via feast (setup.py) asttokens==2.4.1 # via stack-data async-lru==2.0.4 @@ -52,7 +56,9 @@ azure-core==1.31.0 # azure-identity # azure-storage-blob azure-identity==1.18.0 + # via feast (setup.py) azure-storage-blob==12.23.0 + # via feast (setup.py) babel==2.16.0 # via # jupyterlab-server @@ -60,10 +66,13 @@ babel==2.16.0 beautifulsoup4==4.12.3 # via nbconvert bigtree==0.21.1 + # via feast (setup.py) bleach==6.1.0 # via nbconvert boto3==1.35.23 - # via moto + # via + # feast (setup.py) + # moto botocore==1.35.23 # via # aiobotocore @@ -72,11 +81,13 @@ botocore==1.35.23 # s3transfer build==1.2.2 # via + # feast (setup.py) # pip-tools # singlestoredb cachetools==5.5.0 # via google-auth cassandra-driver==3.29.2 + # via feast (setup.py) certifi==2024.8.30 # via # elastic-transport @@ -99,6 +110,7 @@ charset-normalizer==3.3.2 # snowflake-connector-python click==8.1.7 # via + # feast (setup.py) # dask # geomet # great-expectations @@ -107,15 +119,18 @@ click==8.1.7 cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via great-expectations + # via + # feast (setup.py) + # great-expectations comm==0.2.2 # via # ipykernel # ipywidgets coverage[toml]==7.6.1 # via pytest-cov -cryptography==43.0.1 +cryptography==42.0.8 # via + # feast (setup.py) # azure-identity # azure-storage-blob # great-expectations @@ -130,7 +145,9 @@ cryptography==43.0.1 cython==3.0.11 # via thriftpy2 dask[dataframe]==2024.9.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.14 # via dask db-dtypes==1.3.0 @@ -142,9 +159,11 @@ decorator==5.1.1 defusedxml==0.7.1 # via nbconvert deltalake==0.20.0 + # via feast (setup.py) deprecation==2.1.0 # via python-keycloak dill==0.3.8 + # via feast (setup.py) distlib==0.3.8 # via virtualenv docker==7.1.0 @@ -156,6 +175,7 @@ duckdb==1.1.0 elastic-transport==8.15.0 # via elasticsearch elasticsearch==8.15.1 + # via feast (setup.py) entrypoints==0.4 # via altair exceptiongroup==1.2.2 @@ -167,7 +187,10 @@ execnet==2.1.1 # via pytest-xdist executing==2.1.0 # via stack-data +faiss-cpu==1.9.0 + # via feast (setup.py) fastapi==0.115.2 + # via feast (setup.py) fastjsonschema==2.20.0 # via nbformat filelock==3.16.1 @@ -181,11 +204,14 @@ frozenlist==1.4.1 # aiohttp # aiosignal fsspec==2024.9.0 - # via dask + # via + # feast (setup.py) + # dask geomet==0.2.1.post1 # via cassandra-driver google-api-core[grpc]==2.20.0 # via + # feast (setup.py) # google-cloud-bigquery # google-cloud-bigquery-storage # google-cloud-bigtable @@ -203,8 +229,11 @@ google-auth==2.35.0 # google-cloud-storage # kubernetes google-cloud-bigquery[pandas]==3.25.0 + # via feast (setup.py) google-cloud-bigquery-storage==2.26.0 + # via feast (setup.py) google-cloud-bigtable==2.26.0 + # via feast (setup.py) google-cloud-core==2.4.1 # via # google-cloud-bigquery @@ -212,7 +241,9 @@ google-cloud-core==2.4.1 # google-cloud-datastore # google-cloud-storage google-cloud-datastore==2.20.1 + # via feast (setup.py) google-cloud-storage==2.18.2 + # via feast (setup.py) google-crc32c==1.6.0 # via # google-cloud-storage @@ -223,16 +254,17 @@ google-resumable-media==2.7.2 # google-cloud-storage googleapis-common-protos[grpc]==1.65.0 # via + # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status great-expectations==0.18.21 -greenlet==3.1.0 - # via sqlalchemy + # via feast (setup.py) grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable grpcio==1.66.1 # via + # feast (setup.py) # google-api-core # googleapis-common-protos # grpc-google-iam-v1 @@ -242,30 +274,44 @@ grpcio==1.66.1 # grpcio-testing # grpcio-tools grpcio-health-checking==1.62.3 + # via feast (setup.py) grpcio-reflection==1.62.3 + # via feast (setup.py) grpcio-status==1.62.3 # via google-api-core grpcio-testing==1.62.3 + # via feast (setup.py) grpcio-tools==1.62.3 + # via feast (setup.py) gunicorn==23.0.0 + # via + # feast (setup.py) + # uvicorn-worker h11==0.14.0 # via # httpcore # uvicorn happybase==1.2.0 + # via feast (setup.py) hazelcast-python-client==5.5.0 + # via feast (setup.py) hiredis==2.4.0 + # via feast (setup.py) httpcore==1.0.5 # via httpx httptools==0.6.1 # via uvicorn httpx==0.27.2 # via + # feast (setup.py) # jupyterlab # python-keycloak ibis-framework[duckdb]==9.5.0 - # via ibis-substrait + # via + # feast (setup.py) + # ibis-substrait ibis-substrait==4.0.1 + # via feast (setup.py) identify==2.6.1 # via pre-commit idna==3.10 @@ -279,9 +325,7 @@ idna==3.10 imagesize==1.4.1 # via sphinx importlib-metadata==8.5.0 - # via - # build - # dask + # via dask iniconfig==2.0.0 # via pytest ipykernel==6.29.5 @@ -301,6 +345,7 @@ jedi==0.19.1 # via ipython jinja2==3.1.4 # via + # feast (setup.py) # altair # great-expectations # jupyter-server @@ -323,6 +368,7 @@ jsonpointer==3.0.0 # jsonschema jsonschema[format-nongpl]==4.23.0 # via + # feast (setup.py) # altair # great-expectations # jupyter-events @@ -370,6 +416,7 @@ jupyterlab-widgets==3.0.13 jwcrypto==1.5.6 # via python-keycloak kubernetes==20.13.0 + # via feast (setup.py) locket==1.0.0 # via partd makefun==1.15.4 @@ -390,13 +437,17 @@ matplotlib-inline==0.1.7 mdurl==0.1.2 # via markdown-it-py minio==7.1.0 + # via feast (setup.py) mistune==3.0.2 # via # great-expectations # nbconvert mmh3==5.0.0 + # via feast (setup.py) mock==2.0.0 + # via feast (setup.py) moto==4.2.14 + # via feast (setup.py) msal==1.31.0 # via # azure-identity @@ -408,10 +459,13 @@ multidict==6.1.0 # aiohttp # yarl mypy==1.11.2 - # via sqlalchemy + # via + # feast (setup.py) + # sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 + # via feast (setup.py) nbclient==0.10.0 # via nbconvert nbconvert==7.16.4 @@ -434,9 +488,11 @@ notebook-shim==0.2.4 # notebook numpy==1.26.4 # via + # feast (setup.py) # altair # dask # db-dtypes + # faiss-cpu # great-expectations # ibis-framework # pandas @@ -452,6 +508,7 @@ packaging==24.1 # dask # db-dtypes # deprecation + # faiss-cpu # google-cloud-bigquery # great-expectations # gunicorn @@ -468,6 +525,7 @@ packaging==24.1 # sphinx pandas==2.2.2 # via + # feast (setup.py) # altair # dask # dask-expr @@ -493,6 +551,7 @@ pexpect==4.9.0 pip==24.2 # via pip-tools pip-tools==7.4.1 + # via feast (setup.py) platformdirs==3.11.0 # via # jupyter-core @@ -505,8 +564,11 @@ ply==3.11 portalocker==2.10.1 # via msal-extensions pre-commit==3.3.1 + # via feast (setup.py) prometheus-client==0.20.0 - # via jupyter-server + # via + # feast (setup.py) + # jupyter-server prompt-toolkit==3.0.47 # via ipython proto-plus==1.24.0 @@ -517,6 +579,7 @@ proto-plus==1.24.0 # google-cloud-datastore protobuf==4.25.5 # via + # feast (setup.py) # google-api-core # google-cloud-bigquery-storage # google-cloud-bigtable @@ -532,9 +595,12 @@ protobuf==4.25.5 # proto-plus # substrait psutil==5.9.0 - # via ipykernel -psycopg[binary, pool]==3.2.2 -psycopg-binary==3.2.2 + # via + # feast (setup.py) + # ipykernel +psycopg[binary, pool]==3.1.19 + # via feast (setup.py) +psycopg-binary==3.1.19 # via psycopg psycopg-pool==3.2.3 # via psycopg @@ -545,12 +611,14 @@ ptyprocess==0.7.0 pure-eval==0.2.3 # via stack-data py==1.11.0 + # via feast (setup.py) py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark pyarrow==17.0.0 # via + # feast (setup.py) # dask-expr # db-dtypes # deltalake @@ -566,28 +634,35 @@ pyasn1==0.6.1 pyasn1-modules==0.4.1 # via google-auth pybindgen==0.22.1 + # via feast (setup.py) pycparser==2.22 # via cffi pydantic==2.9.2 # via + # feast (setup.py) # fastapi # great-expectations pydantic-core==2.23.4 # via pydantic pygments==2.18.0 # via + # feast (setup.py) # ipython # nbconvert # rich # sphinx pyjwt[crypto]==2.9.0 # via + # feast (setup.py) # msal # singlestoredb # snowflake-connector-python pymssql==2.3.1 + # via feast (setup.py) pymysql==1.1.1 + # via feast (setup.py) pyodbc==5.1.0 + # via feast (setup.py) pyopenssl==24.2.1 # via snowflake-connector-python pyparsing==3.1.4 @@ -597,8 +672,10 @@ pyproject-hooks==1.1.0 # build # pip-tools pyspark==3.5.2 + # via feast (setup.py) pytest==7.4.4 # via + # feast (setup.py) # pytest-benchmark # pytest-cov # pytest-env @@ -608,13 +685,21 @@ pytest==7.4.4 # pytest-timeout # pytest-xdist pytest-benchmark==3.4.1 + # via feast (setup.py) pytest-cov==5.0.0 + # via feast (setup.py) pytest-env==1.1.3 + # via feast (setup.py) pytest-lazy-fixture==0.6.3 + # via feast (setup.py) pytest-mock==1.10.4 + # via feast (setup.py) pytest-ordering==0.6 + # via feast (setup.py) pytest-timeout==1.4.2 + # via feast (setup.py) pytest-xdist==3.6.1 + # via feast (setup.py) python-dateutil==2.9.0.post0 # via # arrow @@ -632,6 +717,7 @@ python-dotenv==1.0.1 python-json-logger==2.0.7 # via jupyter-events python-keycloak==4.2.2 + # via feast (setup.py) pytz==2024.2 # via # great-expectations @@ -641,6 +727,7 @@ pytz==2024.2 # trino pyyaml==6.0.2 # via + # feast (setup.py) # dask # ibis-substrait # jupyter-events @@ -654,15 +741,19 @@ pyzmq==26.2.0 # jupyter-client # jupyter-server redis==4.6.0 + # via feast (setup.py) referencing==0.35.1 # via # jsonschema # jsonschema-specifications # jupyter-events regex==2024.9.11 - # via parsimonious + # via + # feast (setup.py) + # parsimonious requests==2.32.3 # via + # feast (setup.py) # azure-core # docker # google-api-core @@ -708,6 +799,7 @@ ruamel-yaml==0.17.40 ruamel-yaml-clib==0.2.8 # via ruamel-yaml ruff==0.6.6 + # via feast (setup.py) s3transfer==0.10.2 # via boto3 scipy==1.14.1 @@ -722,6 +814,7 @@ setuptools==75.1.0 # pip-tools # singlestoredb singlestoredb==1.6.3 + # via feast (setup.py) six==1.16.0 # via # asttokens @@ -742,11 +835,13 @@ sniffio==1.3.1 snowballstemmer==2.2.0 # via sphinx snowflake-connector-python[pandas]==3.12.2 + # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.6 # via beautifulsoup4 sphinx==6.2.1 + # via feast (setup.py) sphinxcontrib-applehelp==2.0.0 # via sphinx sphinxcontrib-devhelp==2.0.0 @@ -760,9 +855,11 @@ sphinxcontrib-qthelp==2.0.0 sphinxcontrib-serializinghtml==2.0.0 # via sphinx sqlalchemy[mypy]==2.0.35 + # via feast (setup.py) sqlglot==25.20.1 # via ibis-framework sqlite-vec==0.1.1 + # via feast (setup.py) sqlparams==6.1.0 # via singlestoredb stack-data==0.6.3 @@ -772,17 +869,21 @@ starlette==0.40.0 substrait==0.23.0 # via ibis-substrait tabulate==0.9.0 + # via feast (setup.py) tenacity==8.5.0 + # via feast (setup.py) terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals testcontainers==4.4.0 + # via feast (setup.py) thriftpy2==0.5.2 # via happybase tinycss2==1.3.0 # via nbconvert toml==0.10.2 + # via feast (setup.py) tomli==2.0.1 # via # build @@ -810,7 +911,9 @@ tornado==6.4.1 # notebook # terminado tqdm==4.66.5 - # via great-expectations + # via + # feast (setup.py) + # great-expectations traitlets==5.14.3 # via # comm @@ -827,23 +930,37 @@ traitlets==5.14.3 # nbconvert # nbformat trino==0.329.0 + # via feast (setup.py) typeguard==4.3.0 + # via feast (setup.py) types-cffi==1.16.0.20240331 # via types-pyopenssl types-protobuf==3.19.22 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf types-pymysql==1.1.0.20240524 + # via feast (setup.py) types-pyopenssl==24.1.0.20240722 # via types-redis types-python-dateutil==2.9.0.20240906 - # via arrow + # via + # feast (setup.py) + # arrow types-pytz==2024.2.0.20240913 + # via feast (setup.py) types-pyyaml==6.0.12.20240917 + # via feast (setup.py) types-redis==4.6.0.20240903 + # via feast (setup.py) types-requests==2.30.0.0 + # via feast (setup.py) types-setuptools==75.1.0.20240917 - # via types-cffi + # via + # feast (setup.py) + # types-cffi types-tabulate==0.9.0.20240106 + # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.12.2 @@ -879,6 +996,7 @@ uri-template==1.3.0 # via jsonschema urllib3==2.2.3 # via + # feast (setup.py) # botocore # docker # elastic-transport @@ -889,11 +1007,17 @@ urllib3==2.2.3 # responses # testcontainers uvicorn[standard]==0.30.6 -uvicorn-worker + # via + # feast (setup.py) + # uvicorn-worker +uvicorn-worker==0.2.0 + # via feast (setup.py) uvloop==0.20.0 # via uvicorn virtualenv==20.23.0 - # via pre-commit + # via + # feast (setup.py) + # pre-commit watchfiles==0.24.0 # via uvicorn wcwidth==0.2.13 diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index ed6dc239d37..9e1225d9edf 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -1,6 +1,7 @@ # This file was autogenerated by uv via the following command: -# uv pip compile -p 3.11 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt +# uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt aiobotocore==2.15.1 + # via feast (setup.py) aiohappyeyeballs==2.4.0 # via aiohttp aiohttp==3.10.5 @@ -21,6 +22,8 @@ anyio==4.5.0 # jupyter-server # starlette # watchfiles +appnope==0.1.4 + # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -30,6 +33,7 @@ arrow==1.3.0 asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 + # via feast (setup.py) asttokens==2.4.1 # via stack-data async-lru==2.0.4 @@ -37,7 +41,9 @@ async-lru==2.0.4 async-property==0.2.2 # via python-keycloak async-timeout==4.0.3 - # via redis + # via + # aiohttp + # redis atpublic==5.0 # via ibis-framework attrs==24.2.0 @@ -50,7 +56,9 @@ azure-core==1.31.0 # azure-identity # azure-storage-blob azure-identity==1.18.0 + # via feast (setup.py) azure-storage-blob==12.23.0 + # via feast (setup.py) babel==2.16.0 # via # jupyterlab-server @@ -58,10 +66,13 @@ babel==2.16.0 beautifulsoup4==4.12.3 # via nbconvert bigtree==0.21.1 + # via feast (setup.py) bleach==6.1.0 # via nbconvert boto3==1.35.23 - # via moto + # via + # feast (setup.py) + # moto botocore==1.35.23 # via # aiobotocore @@ -70,11 +81,13 @@ botocore==1.35.23 # s3transfer build==1.2.2 # via + # feast (setup.py) # pip-tools # singlestoredb cachetools==5.5.0 # via google-auth cassandra-driver==3.29.2 + # via feast (setup.py) certifi==2024.8.30 # via # elastic-transport @@ -97,6 +110,7 @@ charset-normalizer==3.3.2 # snowflake-connector-python click==8.1.7 # via + # feast (setup.py) # dask # geomet # great-expectations @@ -105,15 +119,18 @@ click==8.1.7 cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via great-expectations + # via + # feast (setup.py) + # great-expectations comm==0.2.2 # via # ipykernel # ipywidgets coverage[toml]==7.6.1 # via pytest-cov -cryptography==43.0.1 +cryptography==42.0.8 # via + # feast (setup.py) # azure-identity # azure-storage-blob # great-expectations @@ -128,7 +145,9 @@ cryptography==43.0.1 cython==3.0.11 # via thriftpy2 dask[dataframe]==2024.9.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.14 # via dask db-dtypes==1.3.0 @@ -140,9 +159,11 @@ decorator==5.1.1 defusedxml==0.7.1 # via nbconvert deltalake==0.20.0 + # via feast (setup.py) deprecation==2.1.0 # via python-keycloak dill==0.3.8 + # via feast (setup.py) distlib==0.3.8 # via virtualenv docker==7.1.0 @@ -154,13 +175,22 @@ duckdb==1.1.0 elastic-transport==8.15.0 # via elasticsearch elasticsearch==8.15.1 + # via feast (setup.py) entrypoints==0.4 # via altair +exceptiongroup==1.2.2 + # via + # anyio + # ipython + # pytest execnet==2.1.1 # via pytest-xdist executing==2.1.0 # via stack-data +faiss-cpu==1.9.0 + # via feast (setup.py) fastapi==0.115.2 + # via feast (setup.py) fastjsonschema==2.20.0 # via nbformat filelock==3.16.1 @@ -174,11 +204,14 @@ frozenlist==1.4.1 # aiohttp # aiosignal fsspec==2024.9.0 - # via dask + # via + # feast (setup.py) + # dask geomet==0.2.1.post1 # via cassandra-driver google-api-core[grpc]==2.20.0 # via + # feast (setup.py) # google-cloud-bigquery # google-cloud-bigquery-storage # google-cloud-bigtable @@ -196,8 +229,11 @@ google-auth==2.35.0 # google-cloud-storage # kubernetes google-cloud-bigquery[pandas]==3.25.0 + # via feast (setup.py) google-cloud-bigquery-storage==2.26.0 + # via feast (setup.py) google-cloud-bigtable==2.26.0 + # via feast (setup.py) google-cloud-core==2.4.1 # via # google-cloud-bigquery @@ -205,7 +241,9 @@ google-cloud-core==2.4.1 # google-cloud-datastore # google-cloud-storage google-cloud-datastore==2.20.1 + # via feast (setup.py) google-cloud-storage==2.18.2 + # via feast (setup.py) google-crc32c==1.6.0 # via # google-cloud-storage @@ -216,16 +254,17 @@ google-resumable-media==2.7.2 # google-cloud-storage googleapis-common-protos[grpc]==1.65.0 # via + # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status great-expectations==0.18.21 -greenlet==3.1.0 - # via sqlalchemy + # via feast (setup.py) grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable grpcio==1.66.1 # via + # feast (setup.py) # google-api-core # googleapis-common-protos # grpc-google-iam-v1 @@ -235,30 +274,44 @@ grpcio==1.66.1 # grpcio-testing # grpcio-tools grpcio-health-checking==1.62.3 + # via feast (setup.py) grpcio-reflection==1.62.3 + # via feast (setup.py) grpcio-status==1.62.3 # via google-api-core grpcio-testing==1.62.3 + # via feast (setup.py) grpcio-tools==1.62.3 + # via feast (setup.py) gunicorn==23.0.0 + # via + # feast (setup.py) + # uvicorn-worker h11==0.14.0 # via # httpcore # uvicorn happybase==1.2.0 + # via feast (setup.py) hazelcast-python-client==5.5.0 + # via feast (setup.py) hiredis==2.4.0 + # via feast (setup.py) httpcore==1.0.5 # via httpx httptools==0.6.1 # via uvicorn httpx==0.27.2 # via + # feast (setup.py) # jupyterlab # python-keycloak ibis-framework[duckdb]==9.5.0 - # via ibis-substrait + # via + # feast (setup.py) + # ibis-substrait ibis-substrait==4.0.1 + # via feast (setup.py) identify==2.6.1 # via pre-commit idna==3.10 @@ -292,6 +345,7 @@ jedi==0.19.1 # via ipython jinja2==3.1.4 # via + # feast (setup.py) # altair # great-expectations # jupyter-server @@ -314,6 +368,7 @@ jsonpointer==3.0.0 # jsonschema jsonschema[format-nongpl]==4.23.0 # via + # feast (setup.py) # altair # great-expectations # jupyter-events @@ -361,6 +416,7 @@ jupyterlab-widgets==3.0.13 jwcrypto==1.5.6 # via python-keycloak kubernetes==20.13.0 + # via feast (setup.py) locket==1.0.0 # via partd makefun==1.15.4 @@ -381,13 +437,17 @@ matplotlib-inline==0.1.7 mdurl==0.1.2 # via markdown-it-py minio==7.1.0 + # via feast (setup.py) mistune==3.0.2 # via # great-expectations # nbconvert mmh3==5.0.0 + # via feast (setup.py) mock==2.0.0 + # via feast (setup.py) moto==4.2.14 + # via feast (setup.py) msal==1.31.0 # via # azure-identity @@ -399,10 +459,13 @@ multidict==6.1.0 # aiohttp # yarl mypy==1.11.2 - # via sqlalchemy + # via + # feast (setup.py) + # sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 + # via feast (setup.py) nbclient==0.10.0 # via nbconvert nbconvert==7.16.4 @@ -425,9 +488,11 @@ notebook-shim==0.2.4 # notebook numpy==1.26.4 # via + # feast (setup.py) # altair # dask # db-dtypes + # faiss-cpu # great-expectations # ibis-framework # pandas @@ -443,6 +508,7 @@ packaging==24.1 # dask # db-dtypes # deprecation + # faiss-cpu # google-cloud-bigquery # great-expectations # gunicorn @@ -459,6 +525,7 @@ packaging==24.1 # sphinx pandas==2.2.2 # via + # feast (setup.py) # altair # dask # dask-expr @@ -484,6 +551,7 @@ pexpect==4.9.0 pip==24.2 # via pip-tools pip-tools==7.4.1 + # via feast (setup.py) platformdirs==3.11.0 # via # jupyter-core @@ -496,8 +564,11 @@ ply==3.11 portalocker==2.10.1 # via msal-extensions pre-commit==3.3.1 + # via feast (setup.py) prometheus-client==0.20.0 - # via jupyter-server + # via + # feast (setup.py) + # jupyter-server prompt-toolkit==3.0.47 # via ipython proto-plus==1.24.0 @@ -508,6 +579,7 @@ proto-plus==1.24.0 # google-cloud-datastore protobuf==4.25.5 # via + # feast (setup.py) # google-api-core # google-cloud-bigquery-storage # google-cloud-bigtable @@ -523,9 +595,12 @@ protobuf==4.25.5 # proto-plus # substrait psutil==5.9.0 - # via ipykernel -psycopg[binary, pool]==3.2.2 -psycopg-binary==3.2.2 + # via + # feast (setup.py) + # ipykernel +psycopg[binary, pool]==3.1.19 + # via feast (setup.py) +psycopg-binary==3.1.19 # via psycopg psycopg-pool==3.2.3 # via psycopg @@ -536,12 +611,14 @@ ptyprocess==0.7.0 pure-eval==0.2.3 # via stack-data py==1.11.0 + # via feast (setup.py) py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark pyarrow==17.0.0 # via + # feast (setup.py) # dask-expr # db-dtypes # deltalake @@ -557,28 +634,35 @@ pyasn1==0.6.1 pyasn1-modules==0.4.1 # via google-auth pybindgen==0.22.1 + # via feast (setup.py) pycparser==2.22 # via cffi pydantic==2.9.2 # via + # feast (setup.py) # fastapi # great-expectations pydantic-core==2.23.4 # via pydantic pygments==2.18.0 # via + # feast (setup.py) # ipython # nbconvert # rich # sphinx pyjwt[crypto]==2.9.0 # via + # feast (setup.py) # msal # singlestoredb # snowflake-connector-python pymssql==2.3.1 + # via feast (setup.py) pymysql==1.1.1 + # via feast (setup.py) pyodbc==5.1.0 + # via feast (setup.py) pyopenssl==24.2.1 # via snowflake-connector-python pyparsing==3.1.4 @@ -588,8 +672,10 @@ pyproject-hooks==1.1.0 # build # pip-tools pyspark==3.5.2 + # via feast (setup.py) pytest==7.4.4 # via + # feast (setup.py) # pytest-benchmark # pytest-cov # pytest-env @@ -599,13 +685,21 @@ pytest==7.4.4 # pytest-timeout # pytest-xdist pytest-benchmark==3.4.1 + # via feast (setup.py) pytest-cov==5.0.0 + # via feast (setup.py) pytest-env==1.1.3 + # via feast (setup.py) pytest-lazy-fixture==0.6.3 + # via feast (setup.py) pytest-mock==1.10.4 + # via feast (setup.py) pytest-ordering==0.6 + # via feast (setup.py) pytest-timeout==1.4.2 + # via feast (setup.py) pytest-xdist==3.6.1 + # via feast (setup.py) python-dateutil==2.9.0.post0 # via # arrow @@ -623,6 +717,7 @@ python-dotenv==1.0.1 python-json-logger==2.0.7 # via jupyter-events python-keycloak==4.2.2 + # via feast (setup.py) pytz==2024.2 # via # great-expectations @@ -632,6 +727,7 @@ pytz==2024.2 # trino pyyaml==6.0.2 # via + # feast (setup.py) # dask # ibis-substrait # jupyter-events @@ -645,15 +741,19 @@ pyzmq==26.2.0 # jupyter-client # jupyter-server redis==4.6.0 + # via feast (setup.py) referencing==0.35.1 # via # jsonschema # jsonschema-specifications # jupyter-events regex==2024.9.11 - # via parsimonious + # via + # feast (setup.py) + # parsimonious requests==2.32.3 # via + # feast (setup.py) # azure-core # docker # google-api-core @@ -699,6 +799,7 @@ ruamel-yaml==0.17.40 ruamel-yaml-clib==0.2.8 # via ruamel-yaml ruff==0.6.6 + # via feast (setup.py) s3transfer==0.10.2 # via boto3 scipy==1.14.1 @@ -713,6 +814,7 @@ setuptools==75.1.0 # pip-tools # singlestoredb singlestoredb==1.6.3 + # via feast (setup.py) six==1.16.0 # via # asttokens @@ -733,11 +835,13 @@ sniffio==1.3.1 snowballstemmer==2.2.0 # via sphinx snowflake-connector-python[pandas]==3.12.2 + # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.6 # via beautifulsoup4 sphinx==6.2.1 + # via feast (setup.py) sphinxcontrib-applehelp==2.0.0 # via sphinx sphinxcontrib-devhelp==2.0.0 @@ -751,9 +855,11 @@ sphinxcontrib-qthelp==2.0.0 sphinxcontrib-serializinghtml==2.0.0 # via sphinx sqlalchemy[mypy]==2.0.35 + # via feast (setup.py) sqlglot==25.20.1 # via ibis-framework sqlite-vec==0.1.1 + # via feast (setup.py) sqlparams==6.1.0 # via singlestoredb stack-data==0.6.3 @@ -763,17 +869,31 @@ starlette==0.40.0 substrait==0.23.0 # via ibis-substrait tabulate==0.9.0 + # via feast (setup.py) tenacity==8.5.0 + # via feast (setup.py) terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals testcontainers==4.4.0 + # via feast (setup.py) thriftpy2==0.5.2 # via happybase tinycss2==1.3.0 # via nbconvert toml==0.10.2 + # via feast (setup.py) +tomli==2.0.2 + # via + # build + # coverage + # jupyterlab + # mypy + # pip-tools + # pytest + # pytest-env + # singlestoredb tomlkit==0.13.2 # via snowflake-connector-python toolz==0.12.1 @@ -791,7 +911,9 @@ tornado==6.4.1 # notebook # terminado tqdm==4.66.5 - # via great-expectations + # via + # feast (setup.py) + # great-expectations traitlets==5.14.3 # via # comm @@ -808,27 +930,43 @@ traitlets==5.14.3 # nbconvert # nbformat trino==0.329.0 + # via feast (setup.py) typeguard==4.3.0 + # via feast (setup.py) types-cffi==1.16.0.20240331 # via types-pyopenssl types-protobuf==3.19.22 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf types-pymysql==1.1.0.20240524 + # via feast (setup.py) types-pyopenssl==24.1.0.20240722 # via types-redis types-python-dateutil==2.9.0.20240906 - # via arrow + # via + # feast (setup.py) + # arrow types-pytz==2024.2.0.20240913 + # via feast (setup.py) types-pyyaml==6.0.12.20240917 + # via feast (setup.py) types-redis==4.6.0.20240903 + # via feast (setup.py) types-requests==2.30.0.0 + # via feast (setup.py) types-setuptools==75.1.0.20240917 - # via types-cffi + # via + # feast (setup.py) + # types-cffi types-tabulate==0.9.0.20240106 + # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.12.2 # via + # anyio + # async-lru # azure-core # azure-identity # azure-storage-blob @@ -837,6 +975,7 @@ typing-extensions==4.12.2 # ibis-framework # ipython # jwcrypto + # multidict # mypy # psycopg # psycopg-pool @@ -846,6 +985,7 @@ typing-extensions==4.12.2 # sqlalchemy # testcontainers # typeguard + # uvicorn tzdata==2024.1 # via pandas tzlocal==5.2 @@ -856,6 +996,7 @@ uri-template==1.3.0 # via jsonschema urllib3==2.2.3 # via + # feast (setup.py) # botocore # docker # elastic-transport @@ -866,11 +1007,17 @@ urllib3==2.2.3 # responses # testcontainers uvicorn[standard]==0.30.6 -uvicorn-worker + # via + # feast (setup.py) + # uvicorn-worker +uvicorn-worker==0.2.0 + # via feast (setup.py) uvloop==0.20.0 # via uvicorn virtualenv==20.23.0 - # via pre-commit + # via + # feast (setup.py) + # pre-commit watchfiles==0.24.0 # via uvicorn wcwidth==0.2.13 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index e7d6686b4d2..e7989843a88 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -1,6 +1,7 @@ # This file was autogenerated by uv via the following command: -# uv pip compile -p 3.9 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt +# uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt aiobotocore==2.15.1 + # via feast (setup.py) aiohappyeyeballs==2.4.0 # via aiohttp aiohttp==3.10.5 @@ -21,6 +22,8 @@ anyio==4.5.0 # jupyter-server # starlette # watchfiles +appnope==0.1.4 + # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -30,6 +33,7 @@ arrow==1.3.0 asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 + # via feast (setup.py) asttokens==2.4.1 # via stack-data async-lru==2.0.4 @@ -52,7 +56,9 @@ azure-core==1.31.0 # azure-identity # azure-storage-blob azure-identity==1.18.0 + # via feast (setup.py) azure-storage-blob==12.23.0 + # via feast (setup.py) babel==2.16.0 # via # jupyterlab-server @@ -62,10 +68,13 @@ beautifulsoup4==4.12.3 bidict==0.23.1 # via ibis-framework bigtree==0.21.1 + # via feast (setup.py) bleach==6.1.0 # via nbconvert boto3==1.35.23 - # via moto + # via + # feast (setup.py) + # moto botocore==1.35.23 # via # aiobotocore @@ -74,11 +83,13 @@ botocore==1.35.23 # s3transfer build==1.2.2 # via + # feast (setup.py) # pip-tools # singlestoredb cachetools==5.5.0 # via google-auth cassandra-driver==3.29.2 + # via feast (setup.py) certifi==2024.8.30 # via # elastic-transport @@ -101,6 +112,7 @@ charset-normalizer==3.3.2 # snowflake-connector-python click==8.1.7 # via + # feast (setup.py) # dask # geomet # great-expectations @@ -109,15 +121,18 @@ click==8.1.7 cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via great-expectations + # via + # feast (setup.py) + # great-expectations comm==0.2.2 # via # ipykernel # ipywidgets coverage[toml]==7.6.1 # via pytest-cov -cryptography==43.0.1 +cryptography==42.0.8 # via + # feast (setup.py) # azure-identity # azure-storage-blob # great-expectations @@ -132,7 +147,9 @@ cryptography==43.0.1 cython==3.0.11 # via thriftpy2 dask[dataframe]==2024.8.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.10 # via dask db-dtypes==1.3.0 @@ -144,9 +161,11 @@ decorator==5.1.1 defusedxml==0.7.1 # via nbconvert deltalake==0.20.0 + # via feast (setup.py) deprecation==2.1.0 # via python-keycloak dill==0.3.8 + # via feast (setup.py) distlib==0.3.8 # via virtualenv docker==7.1.0 @@ -158,6 +177,7 @@ duckdb==0.10.3 elastic-transport==8.15.0 # via elasticsearch elasticsearch==8.15.1 + # via feast (setup.py) entrypoints==0.4 # via altair exceptiongroup==1.2.2 @@ -169,7 +189,10 @@ execnet==2.1.1 # via pytest-xdist executing==2.1.0 # via stack-data +faiss-cpu==1.9.0 + # via feast (setup.py) fastapi==0.115.2 + # via feast (setup.py) fastjsonschema==2.20.0 # via nbformat filelock==3.16.1 @@ -183,11 +206,14 @@ frozenlist==1.4.1 # aiohttp # aiosignal fsspec==2024.9.0 - # via dask + # via + # feast (setup.py) + # dask geomet==0.2.1.post1 # via cassandra-driver google-api-core[grpc]==2.20.0 # via + # feast (setup.py) # google-cloud-bigquery # google-cloud-bigquery-storage # google-cloud-bigtable @@ -205,8 +231,11 @@ google-auth==2.35.0 # google-cloud-storage # kubernetes google-cloud-bigquery[pandas]==3.25.0 + # via feast (setup.py) google-cloud-bigquery-storage==2.26.0 + # via feast (setup.py) google-cloud-bigtable==2.26.0 + # via feast (setup.py) google-cloud-core==2.4.1 # via # google-cloud-bigquery @@ -214,7 +243,9 @@ google-cloud-core==2.4.1 # google-cloud-datastore # google-cloud-storage google-cloud-datastore==2.20.1 + # via feast (setup.py) google-cloud-storage==2.18.2 + # via feast (setup.py) google-crc32c==1.6.0 # via # google-cloud-storage @@ -225,16 +256,17 @@ google-resumable-media==2.7.2 # google-cloud-storage googleapis-common-protos[grpc]==1.65.0 # via + # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status great-expectations==0.18.21 -greenlet==3.1.0 - # via sqlalchemy + # via feast (setup.py) grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable grpcio==1.66.1 # via + # feast (setup.py) # google-api-core # googleapis-common-protos # grpc-google-iam-v1 @@ -244,30 +276,44 @@ grpcio==1.66.1 # grpcio-testing # grpcio-tools grpcio-health-checking==1.62.3 + # via feast (setup.py) grpcio-reflection==1.62.3 + # via feast (setup.py) grpcio-status==1.62.3 # via google-api-core grpcio-testing==1.62.3 + # via feast (setup.py) grpcio-tools==1.62.3 + # via feast (setup.py) gunicorn==23.0.0 + # via + # feast (setup.py) + # uvicorn-worker h11==0.14.0 # via # httpcore # uvicorn happybase==1.2.0 + # via feast (setup.py) hazelcast-python-client==5.5.0 + # via feast (setup.py) hiredis==2.4.0 + # via feast (setup.py) httpcore==1.0.5 # via httpx httptools==0.6.1 # via uvicorn httpx==0.27.2 # via + # feast (setup.py) # jupyterlab # python-keycloak ibis-framework[duckdb]==9.0.0 - # via ibis-substrait + # via + # feast (setup.py) + # ibis-substrait ibis-substrait==4.0.1 + # via feast (setup.py) identify==2.6.1 # via pre-commit idna==3.10 @@ -281,16 +327,7 @@ idna==3.10 imagesize==1.4.1 # via sphinx importlib-metadata==8.5.0 - # via - # build - # dask - # jupyter-client - # jupyter-lsp - # jupyterlab - # jupyterlab-server - # nbconvert - # sphinx - # typeguard + # via dask iniconfig==2.0.0 # via pytest ipykernel==6.29.5 @@ -310,6 +347,7 @@ jedi==0.19.1 # via ipython jinja2==3.1.4 # via + # feast (setup.py) # altair # great-expectations # jupyter-server @@ -332,6 +370,7 @@ jsonpointer==3.0.0 # jsonschema jsonschema[format-nongpl]==4.23.0 # via + # feast (setup.py) # altair # great-expectations # jupyter-events @@ -379,6 +418,7 @@ jupyterlab-widgets==3.0.13 jwcrypto==1.5.6 # via python-keycloak kubernetes==20.13.0 + # via feast (setup.py) locket==1.0.0 # via partd makefun==1.15.4 @@ -399,13 +439,17 @@ matplotlib-inline==0.1.7 mdurl==0.1.2 # via markdown-it-py minio==7.1.0 + # via feast (setup.py) mistune==3.0.2 # via # great-expectations # nbconvert mmh3==5.0.0 + # via feast (setup.py) mock==2.0.0 + # via feast (setup.py) moto==4.2.14 + # via feast (setup.py) msal==1.31.0 # via # azure-identity @@ -417,10 +461,13 @@ multidict==6.1.0 # aiohttp # yarl mypy==1.11.2 - # via sqlalchemy + # via + # feast (setup.py) + # sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 + # via feast (setup.py) nbclient==0.10.0 # via nbconvert nbconvert==7.16.4 @@ -443,9 +490,11 @@ notebook-shim==0.2.4 # notebook numpy==1.26.4 # via + # feast (setup.py) # altair # dask # db-dtypes + # faiss-cpu # great-expectations # ibis-framework # pandas @@ -461,6 +510,7 @@ packaging==24.1 # dask # db-dtypes # deprecation + # faiss-cpu # google-cloud-bigquery # great-expectations # gunicorn @@ -476,6 +526,7 @@ packaging==24.1 # sphinx pandas==2.2.2 # via + # feast (setup.py) # altair # dask # dask-expr @@ -501,6 +552,7 @@ pexpect==4.9.0 pip==24.2 # via pip-tools pip-tools==7.4.1 + # via feast (setup.py) platformdirs==3.11.0 # via # jupyter-core @@ -513,8 +565,11 @@ ply==3.11 portalocker==2.10.1 # via msal-extensions pre-commit==3.3.1 + # via feast (setup.py) prometheus-client==0.20.0 - # via jupyter-server + # via + # feast (setup.py) + # jupyter-server prompt-toolkit==3.0.47 # via ipython proto-plus==1.24.0 @@ -525,6 +580,7 @@ proto-plus==1.24.0 # google-cloud-datastore protobuf==4.25.5 # via + # feast (setup.py) # google-api-core # google-cloud-bigquery-storage # google-cloud-bigtable @@ -540,9 +596,12 @@ protobuf==4.25.5 # proto-plus # substrait psutil==5.9.0 - # via ipykernel -psycopg[binary, pool]==3.2.2 -psycopg-binary==3.2.2 + # via + # feast (setup.py) + # ipykernel +psycopg[binary, pool]==3.1.19 + # via feast (setup.py) +psycopg-binary==3.1.19 # via psycopg psycopg-pool==3.2.3 # via psycopg @@ -553,12 +612,14 @@ ptyprocess==0.7.0 pure-eval==0.2.3 # via stack-data py==1.11.0 + # via feast (setup.py) py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark pyarrow==16.1.0 # via + # feast (setup.py) # dask-expr # db-dtypes # deltalake @@ -574,28 +635,35 @@ pyasn1==0.6.1 pyasn1-modules==0.4.1 # via google-auth pybindgen==0.22.1 + # via feast (setup.py) pycparser==2.22 # via cffi pydantic==2.9.2 # via + # feast (setup.py) # fastapi # great-expectations pydantic-core==2.23.4 # via pydantic pygments==2.18.0 # via + # feast (setup.py) # ipython # nbconvert # rich # sphinx pyjwt[crypto]==2.9.0 # via + # feast (setup.py) # msal # singlestoredb # snowflake-connector-python pymssql==2.3.1 + # via feast (setup.py) pymysql==1.1.1 + # via feast (setup.py) pyodbc==5.1.0 + # via feast (setup.py) pyopenssl==24.2.1 # via snowflake-connector-python pyparsing==3.1.4 @@ -605,8 +673,10 @@ pyproject-hooks==1.1.0 # build # pip-tools pyspark==3.5.2 + # via feast (setup.py) pytest==7.4.4 # via + # feast (setup.py) # pytest-benchmark # pytest-cov # pytest-env @@ -616,13 +686,21 @@ pytest==7.4.4 # pytest-timeout # pytest-xdist pytest-benchmark==3.4.1 + # via feast (setup.py) pytest-cov==5.0.0 + # via feast (setup.py) pytest-env==1.1.3 + # via feast (setup.py) pytest-lazy-fixture==0.6.3 + # via feast (setup.py) pytest-mock==1.10.4 + # via feast (setup.py) pytest-ordering==0.6 + # via feast (setup.py) pytest-timeout==1.4.2 + # via feast (setup.py) pytest-xdist==3.6.1 + # via feast (setup.py) python-dateutil==2.9.0.post0 # via # arrow @@ -640,6 +718,7 @@ python-dotenv==1.0.1 python-json-logger==2.0.7 # via jupyter-events python-keycloak==4.2.2 + # via feast (setup.py) pytz==2024.2 # via # great-expectations @@ -649,6 +728,7 @@ pytz==2024.2 # trino pyyaml==6.0.2 # via + # feast (setup.py) # dask # ibis-substrait # jupyter-events @@ -662,15 +742,19 @@ pyzmq==26.2.0 # jupyter-client # jupyter-server redis==4.6.0 + # via feast (setup.py) referencing==0.35.1 # via # jsonschema # jsonschema-specifications # jupyter-events regex==2024.9.11 - # via parsimonious + # via + # feast (setup.py) + # parsimonious requests==2.32.3 # via + # feast (setup.py) # azure-core # docker # google-api-core @@ -716,6 +800,7 @@ ruamel-yaml==0.17.40 ruamel-yaml-clib==0.2.8 # via ruamel-yaml ruff==0.6.6 + # via feast (setup.py) s3transfer==0.10.2 # via boto3 scipy==1.13.1 @@ -730,6 +815,7 @@ setuptools==75.1.0 # pip-tools # singlestoredb singlestoredb==1.6.3 + # via feast (setup.py) six==1.16.0 # via # asttokens @@ -750,11 +836,13 @@ sniffio==1.3.1 snowballstemmer==2.2.0 # via sphinx snowflake-connector-python[pandas]==3.12.2 + # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.6 # via beautifulsoup4 sphinx==6.2.1 + # via feast (setup.py) sphinxcontrib-applehelp==2.0.0 # via sphinx sphinxcontrib-devhelp==2.0.0 @@ -768,9 +856,11 @@ sphinxcontrib-qthelp==2.0.0 sphinxcontrib-serializinghtml==2.0.0 # via sphinx sqlalchemy[mypy]==2.0.35 + # via feast (setup.py) sqlglot==23.12.2 # via ibis-framework sqlite-vec==0.1.1 + # via feast (setup.py) sqlparams==6.1.0 # via singlestoredb stack-data==0.6.3 @@ -780,17 +870,21 @@ starlette==0.40.0 substrait==0.23.0 # via ibis-substrait tabulate==0.9.0 + # via feast (setup.py) tenacity==8.5.0 + # via feast (setup.py) terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals testcontainers==4.4.0 + # via feast (setup.py) thriftpy2==0.5.2 # via happybase tinycss2==1.3.0 # via nbconvert toml==0.10.2 + # via feast (setup.py) tomli==2.0.1 # via # build @@ -818,7 +912,9 @@ tornado==6.4.1 # notebook # terminado tqdm==4.66.5 - # via great-expectations + # via + # feast (setup.py) + # great-expectations traitlets==5.14.3 # via # comm @@ -835,28 +931,41 @@ traitlets==5.14.3 # nbconvert # nbformat trino==0.329.0 + # via feast (setup.py) typeguard==4.3.0 + # via feast (setup.py) types-cffi==1.16.0.20240331 # via types-pyopenssl types-protobuf==3.19.22 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf types-pymysql==1.1.0.20240524 + # via feast (setup.py) types-pyopenssl==24.1.0.20240722 # via types-redis types-python-dateutil==2.9.0.20240906 - # via arrow + # via + # feast (setup.py) + # arrow types-pytz==2024.2.0.20240913 + # via feast (setup.py) types-pyyaml==6.0.12.20240917 + # via feast (setup.py) types-redis==4.6.0.20240903 + # via feast (setup.py) types-requests==2.30.0.0 + # via feast (setup.py) types-setuptools==75.1.0.20240917 - # via types-cffi + # via + # feast (setup.py) + # types-cffi types-tabulate==0.9.0.20240106 + # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.12.2 # via - # aioitertools # anyio # async-lru # azure-core @@ -865,7 +974,6 @@ typing-extensions==4.12.2 # fastapi # great-expectations # ibis-framework - # ipython # jwcrypto # multidict # mypy @@ -875,7 +983,6 @@ typing-extensions==4.12.2 # pydantic-core # snowflake-connector-python # sqlalchemy - # starlette # testcontainers # typeguard # uvicorn @@ -889,6 +996,7 @@ uri-template==1.3.0 # via jsonschema urllib3==1.26.20 # via + # feast (setup.py) # botocore # docker # elastic-transport @@ -897,14 +1005,19 @@ urllib3==1.26.20 # minio # requests # responses - # snowflake-connector-python # testcontainers uvicorn[standard]==0.30.6 -uvicorn-worker + # via + # feast (setup.py) + # uvicorn-worker +uvicorn-worker==0.2.0 + # via feast (setup.py) uvloop==0.20.0 # via uvicorn virtualenv==20.23.0 - # via pre-commit + # via + # feast (setup.py) + # pre-commit watchfiles==0.24.0 # via uvicorn wcwidth==0.2.13 diff --git a/sdk/python/tests/integration/online_store/test_python_feature_server.py b/sdk/python/tests/integration/online_store/test_python_feature_server.py index 159660d20e7..f86d87ff2d9 100644 --- a/sdk/python/tests/integration/online_store/test_python_feature_server.py +++ b/sdk/python/tests/integration/online_store/test_python_feature_server.py @@ -59,8 +59,9 @@ def test_get_online_features(python_fs_client): @pytest.mark.integration @pytest.mark.universal_online_stores -def test_push(python_fs_client): - initial_temp = _get_temperatures_from_feature_server( +@pytest.mark.asyncio +async def test_push(python_fs_client): + initial_temp = await _get_temperatures_from_feature_server( python_fs_client, location_ids=[1] )[0] json_data = json.dumps( @@ -81,7 +82,7 @@ def test_push(python_fs_client): # Check new pushed temperature is fetched assert response.status_code == 200 - assert _get_temperatures_from_feature_server( + assert await _get_temperatures_from_feature_server( python_fs_client, location_ids=[1] ) == [initial_temp * 100] @@ -112,7 +113,7 @@ def test_push_source_does_not_exist(python_fs_client): ) -def _get_temperatures_from_feature_server(client, location_ids: List[int]): +async def _get_temperatures_from_feature_server(client, location_ids: List[int]): get_request_data = { "features": ["pushable_location_stats:temperature"], "entities": {"location_id": location_ids}, From e593c1adb6bf58ca0e06e1cc0d149a21f55a052f Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 23 Oct 2024 20:34:02 -0400 Subject: [PATCH 08/18] add pytest async to ci reqs Signed-off-by: Rob Howley --- sdk/python/requirements/py3.10-ci-requirements.txt | 3 +++ sdk/python/requirements/py3.11-ci-requirements.txt | 3 +++ sdk/python/requirements/py3.9-ci-requirements.txt | 3 +++ setup.py | 1 + 4 files changed, 10 insertions(+) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 0041e21703f..7109a6feaea 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -676,6 +676,7 @@ pyspark==3.5.2 pytest==7.4.4 # via # feast (setup.py) + # pytest-asyncio # pytest-benchmark # pytest-cov # pytest-env @@ -684,6 +685,8 @@ pytest==7.4.4 # pytest-ordering # pytest-timeout # pytest-xdist +pytest-asyncio==0.23.8 + # via feast (setup.py) pytest-benchmark==3.4.1 # via feast (setup.py) pytest-cov==5.0.0 diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 9e1225d9edf..eb8bbc280b4 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -676,6 +676,7 @@ pyspark==3.5.2 pytest==7.4.4 # via # feast (setup.py) + # pytest-asyncio # pytest-benchmark # pytest-cov # pytest-env @@ -684,6 +685,8 @@ pytest==7.4.4 # pytest-ordering # pytest-timeout # pytest-xdist +pytest-asyncio==0.23.8 + # via feast (setup.py) pytest-benchmark==3.4.1 # via feast (setup.py) pytest-cov==5.0.0 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index e7989843a88..2a380d6fc34 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -677,6 +677,7 @@ pyspark==3.5.2 pytest==7.4.4 # via # feast (setup.py) + # pytest-asyncio # pytest-benchmark # pytest-cov # pytest-env @@ -685,6 +686,8 @@ pytest==7.4.4 # pytest-ordering # pytest-timeout # pytest-xdist +pytest-asyncio==0.23.8 + # via feast (setup.py) pytest-benchmark==3.4.1 # via feast (setup.py) pytest-cov==5.0.0 diff --git a/setup.py b/setup.py index 7c75625d302..5ab57c906cf 100644 --- a/setup.py +++ b/setup.py @@ -165,6 +165,7 @@ "psutil==5.9.0", "py>=1.11.0", # https://github.com/pytest-dev/pytest/issues/10420 "pytest>=6.0.0,<8", + "pytest-asyncio<=0.24.0", "pytest-cov", "pytest-xdist", "pytest-benchmark>=3.4.1,<4", From f50c6cb68101b1937120ed470ba764a849d82f5c Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 23 Oct 2024 20:52:24 -0400 Subject: [PATCH 09/18] be safe w cleanup in test fixture Signed-off-by: Rob Howley --- .../online_store/test_python_feature_server.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/sdk/python/tests/integration/online_store/test_python_feature_server.py b/sdk/python/tests/integration/online_store/test_python_feature_server.py index f86d87ff2d9..89f881f37d8 100644 --- a/sdk/python/tests/integration/online_store/test_python_feature_server.py +++ b/sdk/python/tests/integration/online_store/test_python_feature_server.py @@ -63,13 +63,13 @@ def test_get_online_features(python_fs_client): async def test_push(python_fs_client): initial_temp = await _get_temperatures_from_feature_server( python_fs_client, location_ids=[1] - )[0] + ) json_data = json.dumps( { "push_source_name": "location_stats_push_source", "df": { "location_id": [1], - "temperature": [initial_temp * 100], + "temperature": [initial_temp[0] * 100], "event_timestamp": [str(_utc_now())], "created": [str(_utc_now())], }, @@ -82,9 +82,10 @@ async def test_push(python_fs_client): # Check new pushed temperature is fetched assert response.status_code == 200 - assert await _get_temperatures_from_feature_server( + actual = await _get_temperatures_from_feature_server( python_fs_client, location_ids=[1] - ) == [initial_temp * 100] + ) + assert actual == [initial_temp[0] * 100] @pytest.mark.integration @@ -142,5 +143,7 @@ def python_fs_client(environment, universal_data_sources, request): loop = asyncio.get_event_loop() loop.run_until_complete(fs.initialize()) client = TestClient(get_app(fs)) - yield client - loop.run_until_complete(fs.close()) + try: + yield client + finally: + loop.run_until_complete(fs.close()) From 5c4a88a463fc355914d7a1390128461c92ba9bff Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 23 Oct 2024 20:53:57 -0400 Subject: [PATCH 10/18] be safe w cleanup in test fixture Signed-off-by: Rob Howley --- .../integration/online_store/test_python_feature_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/tests/integration/online_store/test_python_feature_server.py b/sdk/python/tests/integration/online_store/test_python_feature_server.py index 89f881f37d8..082271ccdb7 100644 --- a/sdk/python/tests/integration/online_store/test_python_feature_server.py +++ b/sdk/python/tests/integration/online_store/test_python_feature_server.py @@ -146,4 +146,4 @@ def python_fs_client(environment, universal_data_sources, request): try: yield client finally: - loop.run_until_complete(fs.close()) + asyncio.get_event_loop().run_until_complete(fs.close()) From 92dc4bde3b1959aae004bdde9cd0e185c34cc487 Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 23 Oct 2024 21:20:29 -0400 Subject: [PATCH 11/18] update pytest ini Signed-off-by: Rob Howley --- sdk/python/pytest.ini | 4 ++++ .../online_store/test_python_feature_server.py | 8 ++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/python/pytest.ini b/sdk/python/pytest.ini index d87e4c07cb3..a0736767601 100644 --- a/sdk/python/pytest.ini +++ b/sdk/python/pytest.ini @@ -1,4 +1,6 @@ [pytest] +asyncio_mode = auto + markers = universal_offline_stores: mark a test as using all offline stores. universal_online_stores: mark a test as using all online stores. @@ -7,6 +9,8 @@ env = IS_TEST=True filterwarnings = + error::_pytest.warning_types.PytestConfigWarning + error::_pytest.warning_types.PytestUnhandledCoroutineWarning ignore::DeprecationWarning:pyspark.sql.pandas.*: ignore::DeprecationWarning:pyspark.sql.connect.*: ignore::DeprecationWarning:httpx.*: diff --git a/sdk/python/tests/integration/online_store/test_python_feature_server.py b/sdk/python/tests/integration/online_store/test_python_feature_server.py index 082271ccdb7..ee1d27d7e8a 100644 --- a/sdk/python/tests/integration/online_store/test_python_feature_server.py +++ b/sdk/python/tests/integration/online_store/test_python_feature_server.py @@ -21,7 +21,7 @@ @pytest.mark.integration @pytest.mark.universal_online_stores -def test_get_online_features(python_fs_client): +async def test_get_online_features(python_fs_client): request_data_dict = { "features": [ "driver_stats:conv_rate", @@ -59,7 +59,6 @@ def test_get_online_features(python_fs_client): @pytest.mark.integration @pytest.mark.universal_online_stores -@pytest.mark.asyncio async def test_push(python_fs_client): initial_temp = await _get_temperatures_from_feature_server( python_fs_client, location_ids=[1] @@ -91,9 +90,6 @@ async def test_push(python_fs_client): @pytest.mark.integration @pytest.mark.universal_online_stores def test_push_source_does_not_exist(python_fs_client): - initial_temp = _get_temperatures_from_feature_server( - python_fs_client, location_ids=[1] - )[0] with pytest.raises( PushSourceNotFoundException, match="Unable to find push source 'push_source_does_not_exist'", @@ -105,7 +101,7 @@ def test_push_source_does_not_exist(python_fs_client): "push_source_name": "push_source_does_not_exist", "df": { "location_id": [1], - "temperature": [initial_temp * 100], + "temperature": [100], "event_timestamp": [str(_utc_now())], "created": [str(_utc_now())], }, From ee2249a6546b3047ca7625bce8d3daf7f37f48fe Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 23 Oct 2024 22:05:50 -0400 Subject: [PATCH 12/18] not in a finally Signed-off-by: Rob Howley --- .../online_store/test_python_feature_server.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/sdk/python/tests/integration/online_store/test_python_feature_server.py b/sdk/python/tests/integration/online_store/test_python_feature_server.py index ee1d27d7e8a..5de8a6c7b55 100644 --- a/sdk/python/tests/integration/online_store/test_python_feature_server.py +++ b/sdk/python/tests/integration/online_store/test_python_feature_server.py @@ -136,10 +136,7 @@ def python_fs_client(environment, universal_data_sources, request): feast_objects.extend([driver(), customer(), location()]) fs.apply(feast_objects) fs.materialize(environment.start_date, environment.end_date) - loop = asyncio.get_event_loop() - loop.run_until_complete(fs.initialize()) + asyncio.get_event_loop().run_until_complete(fs.initialize()) client = TestClient(get_app(fs)) - try: - yield client - finally: - asyncio.get_event_loop().run_until_complete(fs.close()) + yield client + asyncio.get_event_loop().run_until_complete(fs.close()) From 5c69273b0f7d2ba9c5d9d5986f3d8585bd028c8d Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 23 Oct 2024 23:06:01 -0400 Subject: [PATCH 13/18] remove close Signed-off-by: Rob Howley --- .../tests/integration/online_store/test_python_feature_server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/python/tests/integration/online_store/test_python_feature_server.py b/sdk/python/tests/integration/online_store/test_python_feature_server.py index 5de8a6c7b55..411e7a6f5b2 100644 --- a/sdk/python/tests/integration/online_store/test_python_feature_server.py +++ b/sdk/python/tests/integration/online_store/test_python_feature_server.py @@ -139,4 +139,3 @@ def python_fs_client(environment, universal_data_sources, request): asyncio.get_event_loop().run_until_complete(fs.initialize()) client = TestClient(get_app(fs)) yield client - asyncio.get_event_loop().run_until_complete(fs.close()) From 385e3eabb16fe92caaae92f72f18160435470f96 Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 23 Oct 2024 23:37:10 -0400 Subject: [PATCH 14/18] test client is a lifespan aware context manager Signed-off-by: Rob Howley --- .../integration/online_store/test_python_feature_server.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sdk/python/tests/integration/online_store/test_python_feature_server.py b/sdk/python/tests/integration/online_store/test_python_feature_server.py index 411e7a6f5b2..8e69f719bc8 100644 --- a/sdk/python/tests/integration/online_store/test_python_feature_server.py +++ b/sdk/python/tests/integration/online_store/test_python_feature_server.py @@ -1,4 +1,3 @@ -import asyncio import json from typing import List @@ -136,6 +135,5 @@ def python_fs_client(environment, universal_data_sources, request): feast_objects.extend([driver(), customer(), location()]) fs.apply(feast_objects) fs.materialize(environment.start_date, environment.end_date) - asyncio.get_event_loop().run_until_complete(fs.initialize()) - client = TestClient(get_app(fs)) - yield client + with TestClient(get_app(fs)) as client: + yield client From 7991146830f72c5abd9f8767ab6fb21812668db6 Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Thu, 24 Oct 2024 09:19:41 -0400 Subject: [PATCH 15/18] add async writer for dynamo Signed-off-by: Rob Howley --- .../feast/infra/online_stores/dynamodb.py | 67 +++++++++++++++---- sdk/python/feast/infra/utils/aws_utils.py | 54 +++++++++++++++ .../test_push_features_to_online_store.py | 38 ++++++++--- 3 files changed, 138 insertions(+), 21 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/dynamodb.py b/sdk/python/feast/infra/online_stores/dynamodb.py index a97e81bc447..d5bc9f99990 100644 --- a/sdk/python/feast/infra/online_stores/dynamodb.py +++ b/sdk/python/feast/infra/online_stores/dynamodb.py @@ -26,6 +26,7 @@ from feast.infra.online_stores.helpers import compute_entity_id from feast.infra.online_stores.online_store import OnlineStore from feast.infra.supported_async_methods import SupportedAsyncMethods +from feast.infra.utils.aws_utils import dynamo_write_items_async from feast.protos.feast.core.DynamoDBTable_pb2 import ( DynamoDBTable as DynamoDBTableProto, ) @@ -103,7 +104,7 @@ async def close(self): @property def async_supported(self) -> SupportedAsyncMethods: - return SupportedAsyncMethods(read=True) + return SupportedAsyncMethods(read=True, write=True) def update( self, @@ -238,6 +239,42 @@ def online_write_batch( ) self._write_batch_non_duplicates(table_instance, data, progress, config) + async def online_write_batch_async( + self, + config: RepoConfig, + table: FeatureView, + data: List[ + Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] + ], + progress: Optional[Callable[[int], Any]], + ) -> None: + """ + Writes a batch of feature rows to the online store asynchronously. + + If a tz-naive timestamp is passed to this method, it is assumed to be UTC. + + Args: + config: The config for the current feature store. + table: Feature view to which these feature rows correspond. + data: A list of quadruplets containing feature data. Each quadruplet contains an entity + key, a dict containing feature values, an event timestamp for the row, and the created + timestamp for the row if it exists. + progress: Function to be called once a batch of rows is written to the online store, used + to show progress. + """ + online_config = config.online_store + assert isinstance(online_config, DynamoDBOnlineStoreConfig) + + table_name = _get_table_name(online_config, config, table) + items = [ + _to_write_item(config, entity_key, features, timestamp) + for entity_key, features, timestamp, _ in data + ] + client = _get_aiodynamodb_client( + online_config.region, config.online_store.max_pool_connections + ) + await dynamo_write_items_async(client, table_name, items) + def online_read( self, config: RepoConfig, @@ -419,19 +456,8 @@ def _write_batch_non_duplicates( """Deduplicate write batch request items on ``entity_id`` primary key.""" with table_instance.batch_writer(overwrite_by_pkeys=["entity_id"]) as batch: for entity_key, features, timestamp, created_ts in data: - entity_id = compute_entity_id( - entity_key, - entity_key_serialization_version=config.entity_key_serialization_version, - ) batch.put_item( - Item={ - "entity_id": entity_id, # PartitionKey - "event_ts": str(utils.make_tzaware(timestamp)), - "values": { - k: v.SerializeToString() - for k, v in features.items() # Serialized Features - }, - } + Item=_to_write_item(config, entity_key, features, timestamp) ) if progress: progress(1) @@ -675,3 +701,18 @@ def _get_dynamodb_resource(self, region: str, endpoint_url: Optional[str] = None region, endpoint_url ) return self._dynamodb_resource + + +def _to_write_item(config, entity_key, features, timestamp): + entity_id = compute_entity_id( + entity_key, + entity_key_serialization_version=config.entity_key_serialization_version, + ) + return { + "entity_id": entity_id, # PartitionKey + "event_ts": str(utils.make_tzaware(timestamp)), + "values": { + k: v.SerializeToString() + for k, v in features.items() # Serialized Features + }, + } diff --git a/sdk/python/feast/infra/utils/aws_utils.py b/sdk/python/feast/infra/utils/aws_utils.py index 8e1b182249a..6d522c191f8 100644 --- a/sdk/python/feast/infra/utils/aws_utils.py +++ b/sdk/python/feast/infra/utils/aws_utils.py @@ -1,4 +1,6 @@ +import asyncio import contextlib +import itertools import os import tempfile import uuid @@ -10,6 +12,7 @@ import pyarrow as pa import pyarrow.parquet as pq from tenacity import ( + AsyncRetrying, retry, retry_if_exception_type, stop_after_attempt, @@ -1076,3 +1079,54 @@ def upload_arrow_table_to_athena( # Clean up S3 temporary data # for file_path in uploaded_files: # s3_resource.Object(bucket, file_path).delete() + + +class DynamoUnprocessedWriteItems(Exception): + pass + + +async def dynamo_write_items_async( + dynamo_client, table_name: str, items: list[Any] +) -> None: + DYNAMO_MAX_WRITE_BATCH_SIZE = 25 + + async def _do_write(items): + item_iter = iter(items) + item_batches = [] + while True: + item_batch = [ + item + for item in itertools.islice(item_iter, DYNAMO_MAX_WRITE_BATCH_SIZE) + ] + if not item_batch: + break + + item_batches.append(item_batch) + + return await asyncio.gather( + *[ + dynamo_client.batch_write_item( + RequestItems={table_name: item_batch}, + ) + for item_batch in item_batches + ] + ) + + put_items = [{"PutRequest": {"Item": item}} for item in items] + + retries = AsyncRetrying( + retry=retry_if_exception_type(DynamoUnprocessedWriteItems), + wait=wait_exponential(multiplier=1, max=4), + reraise=True, + ) + + for attempt in retries: + with attempt: + response_batches = await _do_write(put_items) + + put_items = [] + for response in response_batches: + put_items.extend(response["UnprocessedItems"]) + + if put_items: + raise DynamoUnprocessedWriteItems() diff --git a/sdk/python/tests/integration/online_store/test_push_features_to_online_store.py b/sdk/python/tests/integration/online_store/test_push_features_to_online_store.py index 98fe3ab1ec0..8986e21c57d 100644 --- a/sdk/python/tests/integration/online_store/test_push_features_to_online_store.py +++ b/sdk/python/tests/integration/online_store/test_push_features_to_online_store.py @@ -8,29 +8,51 @@ from tests.integration.feature_repos.universal.entities import location -@pytest.mark.integration -@pytest.mark.universal_online_stores -def test_push_features_and_read(environment, universal_data_sources): +@pytest.fixture +def store(environment, universal_data_sources): store = environment.feature_store _, _, data_sources = universal_data_sources feature_views = construct_universal_feature_views(data_sources) location_fv = feature_views.pushed_locations store.apply([location(), location_fv]) + return store + +def _ingest_df(): data = { "location_id": [1], "temperature": [4], "event_timestamp": [pd.Timestamp(_utc_now()).round("ms")], "created": [pd.Timestamp(_utc_now()).round("ms")], } - df_ingest = pd.DataFrame(data) + return pd.DataFrame(data) - store.push("location_stats_push_source", df_ingest) + +def assert_response(online_resp): + online_resp_dict = online_resp.to_dict() + assert online_resp_dict["location_id"] == [1] + assert online_resp_dict["temperature"] == [4] + + +@pytest.mark.integration +@pytest.mark.universal_online_stores +def test_push_features_and_read(store): + store.push("location_stats_push_source", _ingest_df()) online_resp = store.get_online_features( features=["pushable_location_stats:temperature"], entity_rows=[{"location_id": 1}], ) - online_resp_dict = online_resp.to_dict() - assert online_resp_dict["location_id"] == [1] - assert online_resp_dict["temperature"] == [4] + assert_response(online_resp) + + +@pytest.mark.integration +@pytest.mark.universal_online_stores(only=["dynamodb"]) +async def test_push_features_and_read_async(store): + await store.push_async("location_stats_push_source", _ingest_df()) + + online_resp = await store.get_online_features_async( + features=["pushable_location_stats:temperature"], + entity_rows=[{"location_id": 1}], + ) + assert_response(online_resp) From b381892d7a0a0e06af99bef6e3172b182cbeb199 Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Thu, 24 Oct 2024 09:38:30 -0400 Subject: [PATCH 16/18] fix dynamo client put item format Signed-off-by: Rob Howley --- .../feast/infra/online_stores/dynamodb.py | 27 ++++++++++++++++--- sdk/python/feast/infra/utils/aws_utils.py | 13 +++++++-- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/dynamodb.py b/sdk/python/feast/infra/online_stores/dynamodb.py index d5bc9f99990..cfdee60c023 100644 --- a/sdk/python/feast/infra/online_stores/dynamodb.py +++ b/sdk/python/feast/infra/online_stores/dynamodb.py @@ -267,10 +267,10 @@ async def online_write_batch_async( table_name = _get_table_name(online_config, config, table) items = [ - _to_write_item(config, entity_key, features, timestamp) + _to_client_write_item(config, entity_key, features, timestamp) for entity_key, features, timestamp, _ in data ] - client = _get_aiodynamodb_client( + client = await _get_aiodynamodb_client( online_config.region, config.online_store.max_pool_connections ) await dynamo_write_items_async(client, table_name, items) @@ -457,7 +457,9 @@ def _write_batch_non_duplicates( with table_instance.batch_writer(overwrite_by_pkeys=["entity_id"]) as batch: for entity_key, features, timestamp, created_ts in data: batch.put_item( - Item=_to_write_item(config, entity_key, features, timestamp) + Item=_to_resource_write_item( + config, entity_key, features, timestamp + ) ) if progress: progress(1) @@ -703,7 +705,7 @@ def _get_dynamodb_resource(self, region: str, endpoint_url: Optional[str] = None return self._dynamodb_resource -def _to_write_item(config, entity_key, features, timestamp): +def _to_resource_write_item(config, entity_key, features, timestamp): entity_id = compute_entity_id( entity_key, entity_key_serialization_version=config.entity_key_serialization_version, @@ -716,3 +718,20 @@ def _to_write_item(config, entity_key, features, timestamp): for k, v in features.items() # Serialized Features }, } + + +def _to_client_write_item(config, entity_key, features, timestamp): + entity_id = compute_entity_id( + entity_key, + entity_key_serialization_version=config.entity_key_serialization_version, + ) + return { + "entity_id": {"S": entity_id}, # PartitionKey + "event_ts": {"S": str(utils.make_tzaware(timestamp))}, + "values": { + "M": { + k: {"B": v.SerializeToString()} + for k, v in features.items() # Serialized Features + } + }, + } diff --git a/sdk/python/feast/infra/utils/aws_utils.py b/sdk/python/feast/infra/utils/aws_utils.py index 6d522c191f8..8b4417d3cae 100644 --- a/sdk/python/feast/infra/utils/aws_utils.py +++ b/sdk/python/feast/infra/utils/aws_utils.py @@ -1086,8 +1086,17 @@ class DynamoUnprocessedWriteItems(Exception): async def dynamo_write_items_async( - dynamo_client, table_name: str, items: list[Any] + dynamo_client, table_name: str, items: list[dict] ) -> None: + """ + Writes in batches to a dynamo table asynchronously. Max batch size is 25. + Raises DynamoUnprocessedWriteItems if not all items can be written. + + Args: + dynamo_client: async dynamodb client + table_name: name of table being written to + items: list of items to be written. see boto3 docs on format of the items. + """ DYNAMO_MAX_WRITE_BATCH_SIZE = 25 async def _do_write(items): @@ -1120,7 +1129,7 @@ async def _do_write(items): reraise=True, ) - for attempt in retries: + async for attempt in retries: with attempt: response_batches = await _do_write(put_items) From 4b0378e001fe2a7bb78066bdc3cb93897b2897ec Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Thu, 24 Oct 2024 09:40:20 -0400 Subject: [PATCH 17/18] clarify documentation Signed-off-by: Rob Howley --- sdk/python/feast/infra/utils/aws_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/utils/aws_utils.py b/sdk/python/feast/infra/utils/aws_utils.py index 8b4417d3cae..0526cf8b65c 100644 --- a/sdk/python/feast/infra/utils/aws_utils.py +++ b/sdk/python/feast/infra/utils/aws_utils.py @@ -1089,7 +1089,8 @@ async def dynamo_write_items_async( dynamo_client, table_name: str, items: list[dict] ) -> None: """ - Writes in batches to a dynamo table asynchronously. Max batch size is 25. + Writes in batches to a dynamo table asynchronously. Max size of each + attempted batch is 25. Raises DynamoUnprocessedWriteItems if not all items can be written. Args: From 24d203d18ff8deefa25edc851d2141f401b09097 Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Thu, 24 Oct 2024 12:53:31 -0400 Subject: [PATCH 18/18] add deduplication to async dynamo write Signed-off-by: Rob Howley --- .../feast/infra/online_stores/dynamodb.py | 13 ++++++++++- .../test_dynamodb_online_store.py | 23 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/online_stores/dynamodb.py b/sdk/python/feast/infra/online_stores/dynamodb.py index cfdee60c023..15e8357754b 100644 --- a/sdk/python/feast/infra/online_stores/dynamodb.py +++ b/sdk/python/feast/infra/online_stores/dynamodb.py @@ -15,6 +15,7 @@ import contextlib import itertools import logging +from collections import OrderedDict from datetime import datetime from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union @@ -268,7 +269,7 @@ async def online_write_batch_async( table_name = _get_table_name(online_config, config, table) items = [ _to_client_write_item(config, entity_key, features, timestamp) - for entity_key, features, timestamp, _ in data + for entity_key, features, timestamp, _ in _latest_data_to_write(data) ] client = await _get_aiodynamodb_client( online_config.region, config.online_store.max_pool_connections @@ -735,3 +736,13 @@ def _to_client_write_item(config, entity_key, features, timestamp): } }, } + + +def _latest_data_to_write( + data: List[ + Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] + ], +): + as_hashable = ((d[0].SerializeToString(), d) for d in data) + sorted_data = sorted(as_hashable, key=lambda ah: (ah[0], ah[1][2])) + return (v for _, v in OrderedDict((ah[0], ah[1]) for ah in sorted_data).items()) diff --git a/sdk/python/tests/unit/infra/online_store/test_dynamodb_online_store.py b/sdk/python/tests/unit/infra/online_store/test_dynamodb_online_store.py index 6ff7b3c3605..cb1c15ee6e4 100644 --- a/sdk/python/tests/unit/infra/online_store/test_dynamodb_online_store.py +++ b/sdk/python/tests/unit/infra/online_store/test_dynamodb_online_store.py @@ -1,5 +1,6 @@ from copy import deepcopy from dataclasses import dataclass +from datetime import datetime import boto3 import pytest @@ -10,6 +11,7 @@ DynamoDBOnlineStore, DynamoDBOnlineStoreConfig, DynamoDBTable, + _latest_data_to_write, ) from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto @@ -358,3 +360,24 @@ def test_dynamodb_online_store_online_read_unknown_entity_end_of_batch( # ensure the entity is not dropped assert len(returned_items) == len(entity_keys) assert returned_items[-1] == (None, None) + + +def test_batch_write_deduplication(): + def to_ek_proto(val): + return EntityKeyProto( + join_keys=["customer"], entity_values=[ValueProto(string_val=val)] + ) + + # is out of order and has duplicate keys + data = [ + (to_ek_proto("key-1"), {}, datetime(2024, 1, 1), None), + (to_ek_proto("key-2"), {}, datetime(2024, 1, 1), None), + (to_ek_proto("key-1"), {}, datetime(2024, 1, 3), None), + (to_ek_proto("key-1"), {}, datetime(2024, 1, 2), None), + (to_ek_proto("key-3"), {}, datetime(2024, 1, 2), None), + ] + + # assert we only keep the most recent record per key + actual = list(_latest_data_to_write(data)) + expected = [data[2], data[1], data[4]] + assert expected == actual