-
Notifications
You must be signed in to change notification settings - Fork 2.7k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Added get file range with the prevsharesnapshot parameter #13507
Changes from 19 commits
3b36965
cea6275
8a11235
379288a
55708ea
e4b6100
65073b0
61d9081
50e6a7f
7f288f7
85f4198
7af74c6
e906ffb
d3ffaa6
bb108f0
de893c4
caccc44
1bfe833
00e5e55
bbd3d40
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,12 +3,12 @@ | |
# Licensed under the MIT License. See License.txt in the project root for | ||
# license information. | ||
# -------------------------------------------------------------------------- | ||
# pylint: disable=too-many-lines | ||
# pylint: disable=too-many-lines, too-many-public-methods | ||
import functools | ||
import time | ||
from io import BytesIO | ||
from typing import ( # pylint: disable=unused-import | ||
Optional, Union, IO, List, Dict, Any, Iterable, | ||
Optional, Union, IO, List, Dict, Any, Iterable, Tuple, | ||
TYPE_CHECKING | ||
) | ||
|
||
|
@@ -33,7 +33,7 @@ | |
from ._parser import _get_file_permission, _datetime_to_str | ||
from ._lease import ShareLeaseClient | ||
from ._serialize import get_source_conditions, get_access_conditions, get_smb_properties, get_api_version | ||
from ._deserialize import deserialize_file_properties, deserialize_file_stream | ||
from ._deserialize import deserialize_file_properties, deserialize_file_stream, get_file_ranges_result | ||
from ._models import HandlesPaged, NTFSAttributes # pylint: disable=unused-import | ||
from ._download import StorageStreamDownloader | ||
|
||
|
@@ -1093,14 +1093,49 @@ def upload_range_from_url(self, source_url, | |
except StorageErrorException as error: | ||
process_storage_error(error) | ||
|
||
def _get_ranges_options( # type: ignore | ||
self, offset=None, # type: Optional[int] | ||
length=None, # type: Optional[int] | ||
previous_sharesnapshot=None, # type: Optional[Union[str, Dict[str, Any]]] | ||
**kwargs | ||
): | ||
# type: (...) -> Dict[str, Any] | ||
if self.require_encryption or (self.key_encryption_key is not None): | ||
raise ValueError("Unsupported method for encryption.") | ||
access_conditions = get_access_conditions(kwargs.pop('lease', None)) | ||
|
||
content_range = None | ||
if offset is not None: | ||
if length is not None: | ||
end_range = offset + length - 1 # Reformat to an inclusive range index | ||
content_range = 'bytes={0}-{1}'.format(offset, end_range) | ||
else: | ||
content_range = 'bytes={0}-'.format(offset) | ||
options = { | ||
'sharesnapshot': self.snapshot, | ||
'lease_access_conditions': access_conditions, | ||
'timeout': kwargs.pop('timeout', None), | ||
'range': content_range} | ||
if previous_sharesnapshot: | ||
try: | ||
options['prevsharesnapshot'] = previous_sharesnapshot.snapshot # type: ignore | ||
except AttributeError: | ||
try: | ||
options['prevsharesnapshot'] = previous_sharesnapshot['snapshot'] # type: ignore | ||
except TypeError: | ||
options['prevsharesnapshot'] = previous_sharesnapshot | ||
options.update(kwargs) | ||
return options | ||
|
||
@distributed_trace | ||
def get_ranges( # type: ignore | ||
self, offset=None, # type: Optional[int] | ||
length=None, # type: Optional[int] | ||
**kwargs # type: Any | ||
): | ||
# type: (...) -> List[Dict[str, int]] | ||
"""Returns the list of valid ranges of a file. | ||
"""Returns the list of valid page ranges for a file or snapshot | ||
of a file. | ||
|
||
:param int offset: | ||
Specifies the start offset of bytes over which to get ranges. | ||
|
@@ -1115,31 +1150,62 @@ def get_ranges( # type: ignore | |
:paramtype lease: ~azure.storage.fileshare.ShareLeaseClient or str | ||
:keyword int timeout: | ||
The timeout parameter is expressed in seconds. | ||
:returns: A list of valid ranges. | ||
:returns: | ||
A list of valid ranges. | ||
:rtype: List[dict[str, int]] | ||
""" | ||
timeout = kwargs.pop('timeout', None) | ||
if self.require_encryption or (self.key_encryption_key is not None): | ||
raise ValueError("Unsupported method for encryption.") | ||
access_conditions = get_access_conditions(kwargs.pop('lease', None)) | ||
options = self._get_ranges_options( | ||
offset=offset, | ||
length=length, | ||
**kwargs) | ||
try: | ||
ranges = self._client.file.get_range_list(**options) | ||
except StorageErrorException as error: | ||
process_storage_error(error) | ||
return [{'start': file_range.start, 'end': file_range.end} for file_range in ranges.ranges] | ||
|
||
content_range = None | ||
if offset is not None: | ||
if length is not None: | ||
end_range = offset + length - 1 # Reformat to an inclusive range index | ||
content_range = 'bytes={0}-{1}'.format(offset, end_range) | ||
else: | ||
content_range = 'bytes={0}-'.format(offset) | ||
def get_ranges_diff( # type: ignore | ||
self, | ||
previous_sharesnapshot, # type: Union[str, Dict[str, Any]] | ||
offset=None, # type: Optional[int] | ||
length=None, # type: Optional[int] | ||
**kwargs # type: Any | ||
): | ||
# type: (...) -> Tuple[List[Dict[str, int]], List[Dict[str, int]]] | ||
"""Returns the list of valid page ranges for a file or snapshot | ||
of a file. | ||
|
||
.. versionadded:: 12.6.0 | ||
|
||
:param int offset: | ||
Specifies the start offset of bytes over which to get ranges. | ||
:param int length: | ||
Number of bytes to use over which to get ranges. | ||
:param str previous_sharesnapshot: | ||
The snapshot diff parameter that contains an opaque DateTime value that | ||
specifies a previous file snapshot to be compared | ||
against a more recent snapshot or the current file. | ||
:keyword lease: | ||
Required if the file has an active lease. Value can be a ShareLeaseClient object | ||
or the lease ID as a string. | ||
:paramtype lease: ~azure.storage.fileshare.ShareLeaseClient or str | ||
:keyword int timeout: | ||
The timeout parameter is expressed in seconds. | ||
:returns: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The return type needs to be changed, it's not a tuple now, I'm thinking if we want to create a new API to get range diff, sharing the same API with get_ranges could cause confusion I feel. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks @xiafu-msft - yes we shouldn't have mixed return types. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, that is correct @annatisch |
||
A tuple of two lists of file ranges as dictionaries with 'start' and 'end' keys. | ||
The first element are filled file ranges, the 2nd element is cleared file ranges. | ||
:rtype: tuple(list(dict(str, str), list(dict(str, str)) | ||
""" | ||
options = self._get_ranges_options( | ||
offset=offset, | ||
length=length, | ||
previous_sharesnapshot=previous_sharesnapshot, | ||
**kwargs) | ||
try: | ||
ranges = self._client.file.get_range_list( | ||
range=content_range, | ||
sharesnapshot=self.snapshot, | ||
lease_access_conditions=access_conditions, | ||
timeout=timeout, | ||
**kwargs) | ||
ranges = self._client.file.get_range_list(**options) | ||
except StorageErrorException as error: | ||
process_storage_error(error) | ||
return [{'start': b.start, 'end': b.end} for b in ranges] | ||
return get_file_ranges_result(ranges) | ||
tasherif-msft marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
@distributed_trace | ||
def clear_range( # type: ignore | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
this line could be removed, weird that pylint didn't get mad before!
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It is actually being used here
azure-sdk-for-python/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_models.py
Line 302 in 00e5e55
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
it seems on stg74 branch it's not used, while I didn't see pylint error when merging previous prs.
I just resolved conflicts on this pr and removed this line, let's see if CI is happy