From 6b9c2beab508b9fe614fa519ca88eceeefcde814 Mon Sep 17 00:00:00 2001 From: Bashar Eid Date: Tue, 22 Aug 2023 00:57:19 -0400 Subject: [PATCH 1/9] allow users to specify whether or not to include binary data when requesting binary files --- src/viam/app/data_client.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/viam/app/data_client.py b/src/viam/app/data_client.py index 71840ac68..9bd010baf 100644 --- a/src/viam/app/data_client.py +++ b/src/viam/app/data_client.py @@ -177,12 +177,15 @@ async def binary_data_by_filter( self, filter: Optional[Filter] = None, dest: Optional[str] = None, + include_binary: bool = False ) -> List[BinaryData]: """Filter and download binary data. Args: filter (viam.proto.app.data.Filter): Optional `Filter` specifying binary data to retrieve. No `Filter` implies all binary data. dest (str): Optional filepath for writing retrieved data. + include_binary (bool): Boolean specifying whether to actually include theh binary file data with each retrieved file. Defaults + to false (i.e., only the files' metadata is returned). Returns: List[bytes]: The binary data. @@ -194,8 +197,8 @@ async def binary_data_by_filter( # `DataRequest`s are limited to 100 pieces of data, so we loop through calls until # we are certain we've received everything. while True: - data_request = DataRequest(filter=filter, limit=100, last=last) - request = BinaryDataByFilterRequest(data_request=data_request, count_only=False) + data_request = DataRequest(filter=filter, limit=1 if include_binary else 100, last=last) + request = BinaryDataByFilterRequest(data_request=data_request, count_only=False, include_binary=include_binary) response: BinaryDataByFilterResponse = await self._data_client.BinaryDataByFilter(request, metadata=self._metadata) if not response.data or len(response.data) == 0: break From 536130659ce48b21a44db297d4548d94081343fa Mon Sep 17 00:00:00 2001 From: Bashar Eid Date: Tue, 22 Aug 2023 01:03:22 -0400 Subject: [PATCH 2/9] update tests --- src/viam/app/data_client.py | 5 +---- tests/mocks/services.py | 1 + tests/test_data_client.py | 4 +++- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/viam/app/data_client.py b/src/viam/app/data_client.py index 9bd010baf..f951b1c6e 100644 --- a/src/viam/app/data_client.py +++ b/src/viam/app/data_client.py @@ -174,10 +174,7 @@ async def tabular_data_by_filter( return data async def binary_data_by_filter( - self, - filter: Optional[Filter] = None, - dest: Optional[str] = None, - include_binary: bool = False + self, filter: Optional[Filter] = None, dest: Optional[str] = None, include_binary: bool = False ) -> List[BinaryData]: """Filter and download binary data. diff --git a/tests/mocks/services.py b/tests/mocks/services.py index 95dc2b12c..4dab00b8e 100644 --- a/tests/mocks/services.py +++ b/tests/mocks/services.py @@ -528,6 +528,7 @@ async def BinaryDataByFilter(self, stream: Stream[BinaryDataByFilterRequest, Bin await stream.send_message(BinaryDataByFilterResponse()) return self.filter = request.data_request.filter + self.include_binary = request.include_binary await stream.send_message( BinaryDataByFilterResponse(data=[BinaryData(binary=data.data, metadata=data.metadata) for data in self.binary_response]) ) diff --git a/tests/test_data_client.py b/tests/test_data_client.py index 8575d6b43..0a9c105d0 100644 --- a/tests/test_data_client.py +++ b/tests/test_data_client.py @@ -16,6 +16,7 @@ from .mocks.services import MockData +INCLUDE_BINARY = True COMPONENT_NAME = "component_name" COMPONENT_TYPE = "component_type" METHOD = "method" @@ -136,7 +137,8 @@ async def test_tabular_data_by_filter(self, service: MockData): async def test_binary_data_by_filter(self, service: MockData): async with ChannelFor([service]) as channel: client = DataClient(channel, DATA_SERVICE_METADATA) - binary_data = await client.binary_data_by_filter(filter=FILTER) + binary_data = await client.binary_data_by_filter(filter=FILTER, include_binary=INCLUDE_BINARY) + assert service.include_binary == INCLUDE_BINARY assert binary_data == BINARY_RESPONSE self.assert_filter(filter=service.filter) From e7e269b13f301af91e624b645b5f07f96f569f85 Mon Sep 17 00:00:00 2001 From: Bashar Eid Date: Tue, 22 Aug 2023 10:20:35 -0400 Subject: [PATCH 3/9] fix spelling error and rename input parameter --- src/viam/app/data_client.py | 8 ++++---- tests/test_data_client.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/viam/app/data_client.py b/src/viam/app/data_client.py index f951b1c6e..c383316cb 100644 --- a/src/viam/app/data_client.py +++ b/src/viam/app/data_client.py @@ -174,14 +174,14 @@ async def tabular_data_by_filter( return data async def binary_data_by_filter( - self, filter: Optional[Filter] = None, dest: Optional[str] = None, include_binary: bool = False + self, filter: Optional[Filter] = None, dest: Optional[str] = None, include_file_data: bool = False ) -> List[BinaryData]: """Filter and download binary data. Args: filter (viam.proto.app.data.Filter): Optional `Filter` specifying binary data to retrieve. No `Filter` implies all binary data. dest (str): Optional filepath for writing retrieved data. - include_binary (bool): Boolean specifying whether to actually include theh binary file data with each retrieved file. Defaults + include_file_data (bool): Boolean specifying whether to actually include the binary file data with each retrieved file. Defaults to false (i.e., only the files' metadata is returned). Returns: @@ -194,8 +194,8 @@ async def binary_data_by_filter( # `DataRequest`s are limited to 100 pieces of data, so we loop through calls until # we are certain we've received everything. while True: - data_request = DataRequest(filter=filter, limit=1 if include_binary else 100, last=last) - request = BinaryDataByFilterRequest(data_request=data_request, count_only=False, include_binary=include_binary) + data_request = DataRequest(filter=filter, limit=1 if include_file_data else 100, last=last) + request = BinaryDataByFilterRequest(data_request=data_request, count_only=False, include_binary=include_file_data) response: BinaryDataByFilterResponse = await self._data_client.BinaryDataByFilter(request, metadata=self._metadata) if not response.data or len(response.data) == 0: break diff --git a/tests/test_data_client.py b/tests/test_data_client.py index 0a9c105d0..55f817cb1 100644 --- a/tests/test_data_client.py +++ b/tests/test_data_client.py @@ -137,7 +137,7 @@ async def test_tabular_data_by_filter(self, service: MockData): async def test_binary_data_by_filter(self, service: MockData): async with ChannelFor([service]) as channel: client = DataClient(channel, DATA_SERVICE_METADATA) - binary_data = await client.binary_data_by_filter(filter=FILTER, include_binary=INCLUDE_BINARY) + binary_data = await client.binary_data_by_filter(filter=FILTER, include_file_data=INCLUDE_BINARY) assert service.include_binary == INCLUDE_BINARY assert binary_data == BINARY_RESPONSE self.assert_filter(filter=service.filter) From d8aa01746addc2456d214c6be5082f488301ebde Mon Sep 17 00:00:00 2001 From: Bashar Eid Date: Tue, 22 Aug 2023 14:20:04 -0400 Subject: [PATCH 4/9] implement limit on binary data retrieval --- src/viam/app/data_client.py | 43 ++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/src/viam/app/data_client.py b/src/viam/app/data_client.py index c383316cb..bed6909c9 100644 --- a/src/viam/app/data_client.py +++ b/src/viam/app/data_client.py @@ -174,33 +174,48 @@ async def tabular_data_by_filter( return data async def binary_data_by_filter( - self, filter: Optional[Filter] = None, dest: Optional[str] = None, include_file_data: bool = False + self, filter: Optional[Filter] = None, dest: Optional[str] = None, include_file_data: bool = False, num_files: Optional[int] = None ) -> List[BinaryData]: """Filter and download binary data. Args: - filter (viam.proto.app.data.Filter): Optional `Filter` specifying binary data to retrieve. No `Filter` implies all binary data. - dest (str): Optional filepath for writing retrieved data. + filter (Optional[viam.proto.app.data.Filter]): Optional `Filter` specifying binary data to retrieve. No `Filter` implies al + binary data. + dest (Optional[str]): Optional filepath for writing retrieved data. include_file_data (bool): Boolean specifying whether to actually include the binary file data with each retrieved file. Defaults to false (i.e., only the files' metadata is returned). + num_files (Optional[str]): Number of binary data to return. Passing 0 returns all binary data matching the filter no matter. + Defaults to 100 if no binary data is requested, otherwise 10. All binary data or the first `num_binary_data` will be + returned, whichever comes first. + + Raises: + ValueError: If `num_files` is less than 0. Returns: List[bytes]: The binary data. """ + num_files = num_files if num_files else 10 if include_file_data else 100 + if num_files < 0: + raise ValueError("num_files must be at least 0.") filter = filter if filter else Filter() + limit = 1 if include_file_data else 100 last = "" data = [] - # `DataRequest`s are limited to 100 pieces of data, so we loop through calls until + # `DataRequest`s are limited in pieces of data, so we loop through calls until # we are certain we've received everything. while True: - data_request = DataRequest(filter=filter, limit=1 if include_file_data else 100, last=last) - request = BinaryDataByFilterRequest(data_request=data_request, count_only=False, include_binary=include_file_data) - response: BinaryDataByFilterResponse = await self._data_client.BinaryDataByFilter(request, metadata=self._metadata) - if not response.data or len(response.data) == 0: + new_data, last = await self._binary_data_by_filter(filter=filter, limit=limit, include_binary=include_file_data, last=last) + if not new_data or len(new_data) == 0: break - data += [DataClient.BinaryData(data.binary, data.metadata) for data in response.data] - last = response.last + elif num_files != 0 and len(new_data) > num_files: + data += new_data[0:num_files] + break + else: + data += new_data + num_files -= len(new_data) + if num_files == 0: + break if dest: try: @@ -211,6 +226,14 @@ async def binary_data_by_filter( return data + async def _binary_data_by_filter( + self, filter: Filter, limit: int, include_binary: bool, last: str + ) -> Tuple[List[BinaryData], str]: + data_request = DataRequest(filter=filter, limit=limit, last=last) + request = BinaryDataByFilterRequest(data_request=data_request, count_only=False, include_binary=include_binary) + response: BinaryDataByFilterResponse = await self._data_client.BinaryDataByFilter(request, metadata=self._metadata) + return [DataClient.BinaryData(data.binary, data.metadata) for data in response.data], response.last + async def binary_data_by_ids( self, binary_ids: List[BinaryID], From a107192c08e3eebbc0febc30468852837fe1870e Mon Sep 17 00:00:00 2001 From: Bashar Eid Date: Tue, 22 Aug 2023 14:21:35 -0400 Subject: [PATCH 5/9] format --- src/viam/app/data_client.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/viam/app/data_client.py b/src/viam/app/data_client.py index bed6909c9..83ef5c24a 100644 --- a/src/viam/app/data_client.py +++ b/src/viam/app/data_client.py @@ -226,9 +226,7 @@ async def binary_data_by_filter( return data - async def _binary_data_by_filter( - self, filter: Filter, limit: int, include_binary: bool, last: str - ) -> Tuple[List[BinaryData], str]: + async def _binary_data_by_filter(self, filter: Filter, limit: int, include_binary: bool, last: str) -> Tuple[List[BinaryData], str]: data_request = DataRequest(filter=filter, limit=limit, last=last) request = BinaryDataByFilterRequest(data_request=data_request, count_only=False, include_binary=include_binary) response: BinaryDataByFilterResponse = await self._data_client.BinaryDataByFilter(request, metadata=self._metadata) From bca7429aeb9a7095d88370224f09bcd40f6c8fc1 Mon Sep 17 00:00:00 2001 From: Bashar Eid Date: Tue, 22 Aug 2023 14:26:53 -0400 Subject: [PATCH 6/9] change default value --- src/viam/app/data_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/viam/app/data_client.py b/src/viam/app/data_client.py index 83ef5c24a..596c0b78e 100644 --- a/src/viam/app/data_client.py +++ b/src/viam/app/data_client.py @@ -174,7 +174,7 @@ async def tabular_data_by_filter( return data async def binary_data_by_filter( - self, filter: Optional[Filter] = None, dest: Optional[str] = None, include_file_data: bool = False, num_files: Optional[int] = None + self, filter: Optional[Filter] = None, dest: Optional[str] = None, include_file_data: bool = True, num_files: Optional[int] = None ) -> List[BinaryData]: """Filter and download binary data. From 2ab72fffe0211b50e3c587b698cbbfbba7cf1078 Mon Sep 17 00:00:00 2001 From: Bashar Eid Date: Thu, 24 Aug 2023 12:37:34 -0400 Subject: [PATCH 7/9] fix docstring --- src/viam/app/data_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/viam/app/data_client.py b/src/viam/app/data_client.py index 596c0b78e..696af209e 100644 --- a/src/viam/app/data_client.py +++ b/src/viam/app/data_client.py @@ -185,8 +185,8 @@ async def binary_data_by_filter( include_file_data (bool): Boolean specifying whether to actually include the binary file data with each retrieved file. Defaults to false (i.e., only the files' metadata is returned). num_files (Optional[str]): Number of binary data to return. Passing 0 returns all binary data matching the filter no matter. - Defaults to 100 if no binary data is requested, otherwise 10. All binary data or the first `num_binary_data` will be - returned, whichever comes first. + Defaults to 100 if no binary data is requested, otherwise 10. All binary data or the first `num_files` will be returned, + whichever comes first. Raises: ValueError: If `num_files` is less than 0. From 83fe346433d6fdbe6285faafc5c8e7aef1ff0adb Mon Sep 17 00:00:00 2001 From: Ethan Date: Mon, 28 Aug 2023 14:14:57 -0400 Subject: [PATCH 8/9] Update src/viam/app/data_client.py --- src/viam/app/data_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/viam/app/data_client.py b/src/viam/app/data_client.py index 696af209e..f76bd5f2a 100644 --- a/src/viam/app/data_client.py +++ b/src/viam/app/data_client.py @@ -179,7 +179,7 @@ async def binary_data_by_filter( """Filter and download binary data. Args: - filter (Optional[viam.proto.app.data.Filter]): Optional `Filter` specifying binary data to retrieve. No `Filter` implies al + filter (Optional[viam.proto.app.data.Filter]): Optional `Filter` specifying binary data to retrieve. No `Filter` implies all binary data. dest (Optional[str]): Optional filepath for writing retrieved data. include_file_data (bool): Boolean specifying whether to actually include the binary file data with each retrieved file. Defaults From 52cef20451b9d15ca130ada3e142a9da37ed46ae Mon Sep 17 00:00:00 2001 From: Ethan Date: Mon, 28 Aug 2023 14:16:24 -0400 Subject: [PATCH 9/9] Update src/viam/app/data_client.py --- src/viam/app/data_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/viam/app/data_client.py b/src/viam/app/data_client.py index f76bd5f2a..e35dd1fe7 100644 --- a/src/viam/app/data_client.py +++ b/src/viam/app/data_client.py @@ -183,7 +183,7 @@ async def binary_data_by_filter( binary data. dest (Optional[str]): Optional filepath for writing retrieved data. include_file_data (bool): Boolean specifying whether to actually include the binary file data with each retrieved file. Defaults - to false (i.e., only the files' metadata is returned). + to true (i.e., both the files' data and metadata are returned). num_files (Optional[str]): Number of binary data to return. Passing 0 returns all binary data matching the filter no matter. Defaults to 100 if no binary data is requested, otherwise 10. All binary data or the first `num_files` will be returned, whichever comes first.