Skip to content
Permalink
Browse files

Merge pull request #624 from Azure/dev

2.1.0 Release for Blob/File/Queue/Common
  • Loading branch information
zezha-msft committed Aug 2, 2019
2 parents 85e1de3 + e355944 commit 4e03ff8c9d6c76a38666a8ceb8937593e93fb7eb
Showing with 28,980 additions and 445 deletions.
  1. +9 −0 azure-storage-blob/ChangeLog.md
  2. +6 −0 azure-storage-blob/azure/storage/blob/__init__.py
  3. +2 −2 azure-storage-blob/azure/storage/blob/_constants.py
  4. +111 −17 azure-storage-blob/azure/storage/blob/_deserialization.py
  5. +8 −5 azure-storage-blob/azure/storage/blob/_download_chunking.py
  6. +151 −0 azure-storage-blob/azure/storage/blob/_serialization.py
  7. +19 −13 azure-storage-blob/azure/storage/blob/_upload_chunking.py
  8. +54 −12 azure-storage-blob/azure/storage/blob/appendblobservice.py
  9. +197 −26 azure-storage-blob/azure/storage/blob/baseblobservice.py
  10. +258 −105 azure-storage-blob/azure/storage/blob/blockblobservice.py
  11. +152 −0 azure-storage-blob/azure/storage/blob/models.py
  12. +59 −13 azure-storage-blob/azure/storage/blob/pageblobservice.py
  13. +2 −2 azure-storage-blob/setup.py
  14. +7 −0 azure-storage-common/ChangeLog.md
  15. +0 −2 azure-storage-common/azure/storage/common/_auth.py
  16. +3 −2 azure-storage-common/azure/storage/common/_constants.py
  17. +13 −1 azure-storage-common/azure/storage/common/_deserialization.py
  18. +2 −1 azure-storage-common/azure/storage/common/_serialization.py
  19. +17 −1 azure-storage-common/azure/storage/common/storageclient.py
  20. +1 −1 azure-storage-common/setup.py
  21. +8 −0 azure-storage-file/ChangeLog.md
  22. +3 −0 azure-storage-file/azure/storage/file/__init__.py
  23. +2 −2 azure-storage-file/azure/storage/file/_constants.py
  24. +18 −0 azure-storage-file/azure/storage/file/_deserialization.py
  25. +32 −3 azure-storage-file/azure/storage/file/_serialization.py
  26. +260 −42 azure-storage-file/azure/storage/file/fileservice.py
  27. +149 −1 azure-storage-file/azure/storage/file/models.py
  28. +2 −2 azure-storage-file/setup.py
  29. +4 −0 azure-storage-queue/ChangeLog.md
  30. +1 −0 azure-storage-queue/azure/storage/queue/__init__.py
  31. +2 −2 azure-storage-queue/azure/storage/queue/_constants.py
  32. +2 −2 azure-storage-queue/setup.py
  33. +3 −6 samples/blob/block_blob_usage.py
  34. +3 −6 samples/blob/page_blob_usage.py
  35. +2 −4 tests/blob/blob_performance.py
  36. +3 −0 tests/blob/test_append_blob.py
  37. +8 −24 tests/blob/test_blob_access_conditions.py
  38. +3 −4 tests/blob/test_blob_encryption.py
  39. +6 −2 tests/blob/test_blob_sas.py
  40. +177 −1 tests/blob/test_blob_storage_account.py
  41. +172 −19 tests/blob/test_block_blob.py
  42. +184 −20 tests/blob/test_common_blob.py
  43. +24 −30 tests/blob/test_container.py
  44. +493 −0 tests/blob/test_cpk.py
  45. +2 −1 tests/blob/test_get_blob.py
  46. +4 −6 tests/blob/test_large_block_blob.py
  47. +11 −12 tests/blob/test_page_blob.py
  48. +31 −1 tests/common/test_client.py
  49. +48 −0 tests/file/test_directory.py
  50. +179 −2 tests/file/test_file.py
  51. +19 −0 tests/file/test_share.py
  52. +1 −1 tests/queues/test_queue.py
  53. +2 −2 tests/recordings/test_blob_sas.test_get_user_delegation_key.yaml
  54. +370 −0 tests/recordings/test_blob_storage_account.test_batch_set_nine_standard_blob_tier.yaml
  55. +210 −0 tests/recordings/test_blob_storage_account.test_batch_set_non_existing_blob_tier.yaml
  56. +465 −0 tests/recordings/test_blob_storage_account.test_batch_set_standard_blob_tier_api.yaml
  57. +466 −0 ...ngs/test_blob_storage_account.test_batch_set_standard_blob_tier_api_with_non_askii_blob_name.yaml
  58. +316 −0 tests/recordings/test_blob_storage_account.test_batch_set_standard_blob_tier_for_one_blob.yaml
  59. +812 −0 tests/recordings/test_blob_storage_account.test_batch_set_three_blob_tier.yaml
  60. +219 −0 tests/recordings/test_blob_storage_account.test_set_standard_blob_tier_with_rehydrate_priority.yaml
  61. +106 −0 ...cordings/test_block_blob.test_createBlobFromStream_when_short_read_designated_size_of_stream.yaml
  62. +459 −0 tests/recordings/test_block_blob.test_createBlobFromStream_when_short_read_the_whole_stream.yaml
  63. +106 −0 tests/recordings/test_block_blob.test_create_blob_from_bytes_with_blob_tier_specified.yaml
  64. +108 −0 .../recordings/test_block_blob.test_create_blob_from_path_non_parallel_with_blob_tier_specified.yaml
  65. +124 −0 ...ordings/test_block_blob.test_create_blob_from_stream_for_small_blob_with_blob_tier_specified.yaml
  66. +210 −0 tests/recordings/test_block_blob.test_create_blob_from_text_with_blob_tier_specified.yaml
  67. +132 −0 tests/recordings/test_block_blob.test_put_block_list_with_blob_tier_specified.yaml
  68. +74 −0 tests/recordings/test_client.test_client_request_id_echo.yaml
  69. +109 −0 tests/recordings/test_common_blob.test_batch_delete_one_existing_blob.yaml
  70. +109 −0 tests/recordings/test_common_blob.test_batch_delete_one_existing_blob_with_non_askii_name.yaml
  71. +586 −0 tests/recordings/test_common_blob.test_batch_delete_ten_existing_blob_and_their_snapshot.yaml
  72. +301 −0 tests/recordings/test_common_blob.test_batch_delete_two_existing_blob_and_their_snapshot.yaml
  73. +58 −0 tests/recordings/test_common_blob.test_batch_delete_two_non_existing_blobs.yaml
  74. +178 −0 tests/recordings/test_common_blob.test_copy_blob_with_blob_tier_specified.yaml
  75. +290 −0 tests/recordings/test_common_blob.test_copy_blob_with_rehydrate_priority.yaml
  76. +3 −3 tests/recordings/test_common_blob.test_token_credential.yaml
  77. +187 −0 tests/recordings/test_cpk.test_append_block.yaml
  78. +200 −0 tests/recordings/test_cpk.test_append_block_from_url.yaml
  79. +4,444 −0 tests/recordings/test_cpk.test_create_append_blob_with_chunks.yaml
  80. +4,036 −0 tests/recordings/test_cpk.test_create_block_blob_with_chunks.yaml
  81. +96 −0 tests/recordings/test_cpk.test_create_block_blob_with_single_chunk.yaml
  82. +4,396 −0 tests/recordings/test_cpk.test_create_page_blob_with_chunks.yaml
  83. +146 −0 tests/recordings/test_cpk.test_get_set_blob_metadata.yaml
  84. +157 −0 tests/recordings/test_cpk.test_get_set_blob_properties.yaml
  85. +176 −0 tests/recordings/test_cpk.test_put_block_and_put_block_list.yaml
  86. +322 −0 tests/recordings/test_cpk.test_put_block_from_url_and_commit.yaml
  87. +87 −0 tests/recordings/test_cpk.test_snapshot_blob.yaml
  88. +2,428 −0 tests/recordings/test_cpk.test_update_page.yaml
  89. +1,280 −0 tests/recordings/test_cpk.test_update_page_from_url.yaml
  90. +242 −0 tests/recordings/test_directory.test_set_directory_properties_with_empty_smb_properties.yaml
  91. +291 −0 tests/recordings/test_directory.test_set_directory_properties_with_file_permission_key.yaml
  92. +164 −35 tests/recordings/test_file.test_create_file.yaml
  93. +180 −0 tests/recordings/test_file.test_create_file_will_set_all_smb_properties.yaml
  94. +55 −0 tests/recordings/test_file.test_create_file_with_invalid_file_permission.yaml
  95. +398 −0 tests/recordings/test_file.test_set_file_properties_with_file_permission.yaml
  96. +191 −0 tests/recordings/test_file.test_update_big_range_from_file_url.yaml
  97. +191 −0 tests/recordings/test_file.test_update_range_from_file_url.yaml
  98. +133 −0 ...rdings/test_file.test_update_range_from_file_url_when_source_file_does_not_have_enough_bytes.yaml
  99. +5 −5 tests/recordings/test_queue.test_token_credential.yaml
  100. +156 −0 tests/recordings/test_share.test_create_permission_for_share.yaml
  101. +49 −0 tests/recordings_backup/test_handle.test_close_all_handle.yaml
  102. +50 −0 tests/recordings_backup/test_handle.test_close_single_handle.yaml
  103. +46 −0 tests/recordings_backup/test_handle.test_list_handles_on_directory.yaml
  104. +23 −0 tests/recordings_backup/test_handle.test_list_handles_on_file.yaml
  105. +30 −0 tests/recordings_backup/test_handle.test_list_handles_on_share.yaml
  106. +25 −0 tests/recordings_backup/test_handle.test_list_handles_on_share_snapshot.yaml
  107. +51 −0 tests/recordings_backup/test_handle.test_list_handles_with_marker.yaml
  108. +0 −2 tests/settings_fake.py
  109. +5 −2 tests/testcase.py
@@ -2,7 +2,16 @@

> See [BreakingChanges](BreakingChanges.md) for a detailed list of API breaks.
## Version 2.1.0:

- Support for 2019-02-02 REST version. Please see our REST API documentation and blog for information about the related added features.
- Added Batch Delete Blob API.
- Added Batch Set Standard Blob Tier API(for BlockBlob).
- Added Blob Tier support for PutBlob/PutBlockList/CopyBlob APIs.
- Added support for client provided encryption key to numerous APIs.

## Version 2.0.1:

- Updated dependency on azure-storage-common.

## Version 2.0.0:
@@ -27,5 +27,11 @@
PublicAccess,
BlobPrefix,
DeleteSnapshot,
BatchDeleteSubRequest,
BatchSetBlobTierSubRequest,
BatchSubResponse,
CustomerProvidedEncryptionKey,
RehydratePriority,
)
from .pageblobservice import PageBlobService
from ._constants import __version__
@@ -5,10 +5,10 @@
# --------------------------------------------------------------------------

__author__ = 'Microsoft Corp. <ptvshelp@microsoft.com>'
__version__ = '2.0.1'
__version__ = '2.1.0'

# x-ms-version for storage service.
X_MS_VERSION = '2018-11-09'
X_MS_VERSION = '2019-02-02'

# internal configurations, should not be changed
_LARGE_BLOB_UPLOAD_MAX_READ_BUFFER_SIZE = 4 * 1024 * 1024
@@ -6,6 +6,8 @@
from azure.common import AzureException
from dateutil import parser

from azure.storage.common._http import HTTPResponse

try:
from xml.etree import cElementTree as ETree
except ImportError:
@@ -36,14 +38,23 @@
ResourceProperties,
BlobPrefix,
AccountInformation,
UserDelegationKey,
)
UserDelegationKey, BatchSubResponse)
from ._encryption import _decrypt_blob
from azure.storage.common.models import _list
from azure.storage.common._error import (
_validate_content_match,
_ERROR_DECRYPTION_FAILURE,
)
from io import BytesIO

_HTTP_LINE_ENDING = "\r\n"

def _parse_cpk_headers(response, properties):
server_encrypted = response.headers.get('x-ms-request-server-encrypted')
if server_encrypted is not None:
properties.request_server_encrypted = _bool(server_encrypted)

properties.encryption_key_sha256 = response.headers.get('x-ms-encryption-key-sha256')


def _parse_base_properties(response):
@@ -53,6 +64,7 @@ def _parse_base_properties(response):
resource_properties = ResourceProperties()
resource_properties.last_modified = parser.parse(response.headers.get('last-modified'))
resource_properties.etag = response.headers.get('etag')
_parse_cpk_headers(response, resource_properties)

return resource_properties

@@ -65,6 +77,7 @@ def _parse_page_properties(response):
put_page.last_modified = parser.parse(response.headers.get('last-modified'))
put_page.etag = response.headers.get('etag')
put_page.sequence_number = _to_int(response.headers.get('x-ms-blob-sequence-number'))
_parse_cpk_headers(response, put_page)

return put_page

@@ -78,6 +91,7 @@ def _parse_append_block(response):
append_block.etag = response.headers.get('etag')
append_block.append_offset = _to_int(response.headers.get('x-ms-blob-append-offset'))
append_block.committed_block_count = _to_int(response.headers.get('x-ms-blob-committed-block-count'))
_parse_cpk_headers(response, append_block)

return append_block

@@ -290,7 +304,7 @@ def _convert_xml_to_blob_list(response):
<RemainingRetentionDays>int</RemainingRetentionDays>
<Creation-Time>date-time-value</Creation-Time>
</Properties>
<Metadata>
<Metadata>
<Name>value</Name>
</Metadata>
</Blob>
@@ -392,7 +406,7 @@ def _convert_xml_to_blob_name_list(response):
<RemainingRetentionDays>int</RemainingRetentionDays>
<Creation-Time>date-time-value</Creation-Time>
</Properties>
<Metadata>
<Metadata>
<Name>value</Name>
</Metadata>
</Blob>
@@ -475,19 +489,19 @@ def _convert_xml_to_page_ranges(response):
'''
<?xml version="1.0" encoding="utf-8"?>
<PageList>
<PageRange>
<Start>Start Byte</Start>
<End>End Byte</End>
</PageRange>
<ClearRange>
<Start>Start Byte</Start>
<End>End Byte</End>
</ClearRange>
<PageRange>
<Start>Start Byte</Start>
<End>End Byte</End>
</PageRange>
</PageList>
<PageRange>
<Start>Start Byte</Start>
<End>End Byte</End>
</PageRange>
<ClearRange>
<Start>Start Byte</Start>
<End>End Byte</End>
</ClearRange>
<PageRange>
<Start>Start Byte</Start>
<End>End Byte</End>
</PageRange>
</PageList>
'''
if response is None or response.body is None:
return None
@@ -554,3 +568,83 @@ def _convert_xml_to_user_delegation_key(response):
delegation_key.value = key_element.findtext('Value')

return delegation_key


def _ingest_batch_response(batch_response, batch_sub_requests):
"""
Takes the response to a batch request and parses the response into the separate responses.
:param :class:`~azure.storage.common._http.HTTPResponse` batch_response:
batchResponse The response of the HTTP batch request generated by this object.
:return: sub-responses parsed from batch HTTP response
:rtype: list of :class:`~azure.storage.common._http.HTTPResponse`
"""
parsed_batch_sub_response_list = []

# header value format: `multipart/mixed; boundary=<delimiter>`
response_delimiter = batch_response.headers.get('content-type').split("=")[1]

response_body = batch_response.body.decode('utf-8')

# split byte[] on the "substring" "--<delim>\r\n"
sub_response_list = response_body.split("--" + response_delimiter + _HTTP_LINE_ENDING)

# strip final, slightly different delim "\r\n--<delim>--" off last entry
sub_response_list[len(sub_response_list) - 1] = \
sub_response_list[len(sub_response_list) - 1].split(_HTTP_LINE_ENDING + "--" + response_delimiter + "--")[0]

for sub_response in sub_response_list:
if len(sub_response) != 0:
http_response = _parse_sub_response_to_http_response(sub_response)
is_successful = 200 <= http_response.status < 300
index_of_sub_request = _to_int(http_response.headers.get('Content-ID'))
batch_sub_request = batch_sub_requests[index_of_sub_request]

parsed_batch_sub_response_list.append(BatchSubResponse(is_successful, http_response, batch_sub_request))

return parsed_batch_sub_response_list


def _parse_sub_response_to_http_response(sub_response):
"""
Header: Value (1 or more times)
HTTP/<version> <statusCode> <statusName>
Header: Value (1 or more times)
body (if any)
:param sub_response:
The raw bytes of this sub-response.
:return: An HttpResponse object.
"""

empty_line = _HTTP_LINE_ENDING.encode('utf-8')
num_empty_lines = 0
batch_http_sub_response = HTTPResponse(None, '', dict(), b'')
try:
body_stream = BytesIO()
body_stream.write(sub_response.encode('utf-8'))
body_stream.seek(0)

while True:
line = body_stream.readline()
if line == b'':
return batch_http_sub_response

if line.startswith("HTTP".encode('utf-8')):
batch_http_sub_response.status = _to_int(line.decode('utf-8').split(" ")[1])
elif line == empty_line:
num_empty_lines += 1
elif line.startswith("x-ms-error-code".encode('utf-8')):
batch_http_sub_response.message = line.decode('utf-8').split(": ")[1].rstrip()
elif num_empty_lines is 2:
batch_http_sub_response.body += line
else:
header = line.decode('utf-8').split(": ")[0]
value = line.decode('utf-8').split(": ")[1].rstrip()
batch_http_sub_response.headers[header] = value
finally:
body_stream.close()

return batch_http_sub_response
@@ -10,7 +10,7 @@ def _download_blob_chunks(blob_service, container_name, blob_name, snapshot,
download_size, block_size, progress, start_range, end_range,
stream, max_connections, progress_callback, validate_content,
lease_id, if_modified_since, if_unmodified_since, if_match,
if_none_match, timeout, operation_context):
if_none_match, timeout, operation_context, cpk):

downloader_class = _ParallelBlobChunkDownloader if max_connections > 1 else _SequentialBlobChunkDownloader

@@ -34,6 +34,7 @@ def _download_blob_chunks(blob_service, container_name, blob_name, snapshot,
if_none_match,
timeout,
operation_context,
cpk,
)

if max_connections > 1:
@@ -49,7 +50,7 @@ class _BlobChunkDownloader(object):
def __init__(self, blob_service, container_name, blob_name, snapshot, download_size,
chunk_size, progress, start_range, end_range, stream,
progress_callback, validate_content, lease_id, if_modified_since,
if_unmodified_since, if_match, if_none_match, timeout, operation_context):
if_unmodified_since, if_match, if_none_match, timeout, operation_context, cpk):
# identifiers for the blob
self.blob_service = blob_service
self.container_name = container_name
@@ -78,6 +79,7 @@ def __init__(self, blob_service, container_name, blob_name, snapshot, download_s
self.if_unmodified_since = if_unmodified_since
self.if_match = if_match
self.if_none_match = if_none_match
self.cpk = cpk

def get_chunk_offsets(self):
index = self.start_index
@@ -119,7 +121,8 @@ def _download_chunk(self, chunk_start, chunk_end):
if_match=self.if_match,
if_none_match=self.if_none_match,
timeout=self.timeout,
_context=self.operation_context
_context=self.operation_context,
cpk=self.cpk,
)

# This makes sure that if_match is set so that we can validate
@@ -132,15 +135,15 @@ class _ParallelBlobChunkDownloader(_BlobChunkDownloader):
def __init__(self, blob_service, container_name, blob_name, snapshot, download_size,
chunk_size, progress, start_range, end_range, stream,
progress_callback, validate_content, lease_id, if_modified_since,
if_unmodified_since, if_match, if_none_match, timeout, operation_context):
if_unmodified_since, if_match, if_none_match, timeout, operation_context, cpk):

super(_ParallelBlobChunkDownloader, self).__init__(blob_service, container_name, blob_name, snapshot,
download_size,
chunk_size, progress, start_range, end_range, stream,
progress_callback, validate_content, lease_id,
if_modified_since,
if_unmodified_since, if_match, if_none_match, timeout,
operation_context)
operation_context, cpk)

# for a parallel download, the stream is always seekable, so we note down the current position
# in order to seek to the right place when out-of-order chunks come in

0 comments on commit 4e03ff8

Please sign in to comment.
You can’t perform that action at this time.