Feature/storage stg74

eng/ci_tools.txt

@@ -1,4 +1,5 @@
 # requirements leveraged by ci tools
+cryptography==3.1
 setuptools==44.1.0; python_version == '2.7'
 setuptools==46.4.0; python_version >= '3.5'
 virtualenv==20.0.23

sdk/storage/azure-storage-blob/azure/storage/blob/__init__.py

@@ -54,6 +54,8 @@
     BlobQueryError,
     DelimitedJsonDialect,
     DelimitedTextDialect,
+    ArrowDialect,
+    ArrowType,
     ObjectReplicationPolicy,
     ObjectReplicationRule
 )
@@ -219,6 +221,8 @@ def download_blob_from_url(
     'BlobQueryError',
     'DelimitedJsonDialect',
     'DelimitedTextDialect',
+    'ArrowDialect',
+    'ArrowType',
     'BlobQueryReader',
     'ObjectReplicationPolicy',
     'ObjectReplicationRule'

sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py

@@ -476,7 +476,7 @@ def upload_blob(  # pylint: disable=too-many-locals
             and act according to the condition specified by the `match_condition` parameter.
         :keyword ~azure.core.MatchConditions match_condition:
             The match condition to use upon the etag.
-        :keyword str if_tags_match_condition
+        :keyword str if_tags_match_condition:
             Specify a SQL where clause on blob tags to operate only on blob with a matching value.
             eg. "\"tagname\"='my tag'"
 
@@ -576,7 +576,7 @@ def _download_blob_options(self, offset=None, length=None, **kwargs):
             'lease_access_conditions': access_conditions,
             'modified_access_conditions': mod_conditions,
             'cpk_info': cpk_info,
-            'cls': deserialize_blob_stream,
+            'cls': kwargs.pop('cls', None) or deserialize_blob_stream,
             'max_concurrency':kwargs.pop('max_concurrency', 1),
             'encoding': kwargs.pop('encoding', None),
             'timeout': kwargs.pop('timeout', None),
@@ -636,7 +636,7 @@ def download_blob(self, offset=None, length=None, **kwargs):
             and act according to the condition specified by the `match_condition` parameter.
         :keyword ~azure.core.MatchConditions match_condition:
             The match condition to use upon the etag.
-        :keyword str if_tags_match_condition
+        :keyword str if_tags_match_condition:
             Specify a SQL where clause on blob tags to operate only on blob with a matching value.
             eg. "\"tagname\"='my tag'"
 
@@ -682,13 +682,19 @@ def _quick_query_options(self, query_expression,
             try:
                 delimiter = input_format.lineterminator
             except AttributeError:
-                delimiter = input_format.delimiter
+                try:
+                    delimiter = input_format.delimiter
+                except AttributeError:
+                    raise ValueError("The Type of blob_format can only be DelimitedTextDialect or DelimitedJsonDialect")
         output_format = kwargs.pop('output_format', None)
         if output_format:
             try:
                 delimiter = output_format.lineterminator
             except AttributeError:
-                delimiter = output_format.delimiter
+                try:
+                    delimiter = output_format.delimiter
+                except AttributeError:
+                    pass
         else:
             output_format = input_format
         query_request = QueryRequest(
@@ -729,7 +735,7 @@ def query_blob(self, query_expression, **kwargs):
 
         :param str query_expression:
             Required. a query statement.
-        :keyword Callable[Exception] on_error:
+        :keyword Callable[~azure.storage.blob.BlobQueryError] on_error:
             A function to be called on any processing errors returned by the service.
         :keyword blob_format:
             Optional. Defines the serialization of the data currently stored in the blob. The default is to
@@ -740,7 +746,8 @@ def query_blob(self, query_expression, **kwargs):
             Optional. Defines the output serialization for the data stream. By default the data will be returned
             as it is represented in the blob. By providing an output format, the blob data will be reformatted
             according to that profile. This value can be a DelimitedTextDialect or a DelimitedJsonDialect.
-        :paramtype output_format: ~azure.storage.blob.DelimitedTextDialect or ~azure.storage.blob.DelimitedJsonDialect
+        :paramtype output_format: ~azure.storage.blob.DelimitedTextDialect, ~azure.storage.blob.DelimitedJsonDialect
+            or list[~azure.storage.blob.ArrowDialect]
         :keyword lease:
             Required if the blob has an active lease. Value can be a BlobLeaseClient object
             or the lease ID as a string.
@@ -762,7 +769,7 @@ def query_blob(self, query_expression, **kwargs):
             and act according to the condition specified by the `match_condition` parameter.
         :keyword ~azure.core.MatchConditions match_condition:
             The match condition to use upon the etag.
-        :keyword str if_tags_match_condition
+        :keyword str if_tags_match_condition:
             Specify a SQL where clause on blob tags to operate only on blob with a matching value.
             eg. "\"tagname\"='my tag'"
 
@@ -879,7 +886,7 @@ def delete_blob(self, delete_snapshots=False, **kwargs):
             and act according to the condition specified by the `match_condition` parameter.
         :keyword ~azure.core.MatchConditions match_condition:
             The match condition to use upon the etag.
-        :keyword str if_tags_match_condition
+        :keyword str if_tags_match_condition:
             Specify a SQL where clause on blob tags to operate only on blob with a matching value.
             eg. "\"tagname\"='my tag'"
 
@@ -989,7 +996,7 @@ def get_blob_properties(self, **kwargs):
             and act according to the condition specified by the `match_condition` parameter.
         :keyword ~azure.core.MatchConditions match_condition:
             The match condition to use upon the etag.
-        :keyword str if_tags_match_condition
+        :keyword str if_tags_match_condition:
             Specify a SQL where clause on blob tags to operate only on blob with a matching value.
             eg. "\"tagname\"='my tag'"
 
@@ -1031,14 +1038,15 @@ def get_blob_properties(self, **kwargs):
                 snapshot=self.snapshot,
                 lease_access_conditions=access_conditions,
                 modified_access_conditions=mod_conditions,
-                cls=deserialize_blob_properties,
+                cls=kwargs.pop('cls', None) or deserialize_blob_properties,
                 cpk_info=cpk_info,
                 **kwargs)
         except StorageErrorException as error:
             process_storage_error(error)
         blob_props.name = self.blob_name
-        blob_props.snapshot = self.snapshot
-        blob_props.container = self.container_name
+        if isinstance(blob_props, BlobProperties):
+            blob_props.container = self.container_name
+            blob_props.snapshot = self.snapshot
         return blob_props # type: ignore
 
     def _set_http_headers_options(self, content_settings=None, **kwargs):
@@ -1095,7 +1103,7 @@ def set_http_headers(self, content_settings=None, **kwargs):
             and act according to the condition specified by the `match_condition` parameter.
         :keyword ~azure.core.MatchConditions match_condition:
             The match condition to use upon the etag.
-        :keyword str if_tags_match_condition
+        :keyword str if_tags_match_condition:
             Specify a SQL where clause on blob tags to operate only on blob with a matching value.
             eg. "\"tagname\"='my tag'"
 
@@ -1169,7 +1177,7 @@ def set_blob_metadata(self, metadata=None, **kwargs):
             and act according to the condition specified by the `match_condition` parameter.
         :keyword ~azure.core.MatchConditions match_condition:
             The match condition to use upon the etag.
-        :keyword str if_tags_match_condition
+        :keyword str if_tags_match_condition:
             Specify a SQL where clause on blob tags to operate only on blob with a matching value.
             eg. "\"tagname\"='my tag'"
 
@@ -1516,7 +1524,7 @@ def create_snapshot(self, metadata=None, **kwargs):
             and act according to the condition specified by the `match_condition` parameter.
         :keyword ~azure.core.MatchConditions match_condition:
             The match condition to use upon the etag.
-        :keyword str if_tags_match_condition
+        :keyword str if_tags_match_condition:
             Specify a SQL where clause on blob tags to operate only on destination blob with a matching value.
 
             .. versionadded:: 12.4.0
@@ -1834,7 +1842,7 @@ def acquire_lease(self, lease_duration=-1, lease_id=None, **kwargs):
             and act according to the condition specified by the `match_condition` parameter.
         :keyword ~azure.core.MatchConditions match_condition:
             The match condition to use upon the etag.
-        :keyword str if_tags_match_condition
+        :keyword str if_tags_match_condition:
             Specify a SQL where clause on blob tags to operate only on blob with a matching value.
             eg. "\"tagname\"='my tag'"
 
@@ -1882,7 +1890,7 @@ def set_standard_blob_tier(self, standard_blob_tier, **kwargs):
 
             .. versionadded:: 12.4.0
             This keyword argument was introduced in API version '2019-12-12'.
-        :keyword str if_tags_match_condition
+        :keyword str if_tags_match_condition:
             Specify a SQL where clause on blob tags to operate only on blob with a matching value.
             eg. "\"tagname\"='my tag'"
 
@@ -2141,7 +2149,7 @@ def get_block_list(self, block_list_type="committed", **kwargs):
             Required if the blob has an active lease. Value can be a BlobLeaseClient object
             or the lease ID as a string.
         :paramtype lease: ~azure.storage.blob.BlobLeaseClient or str
-        :keyword str if_tags_match_condition
+        :keyword str if_tags_match_condition:
             Specify a SQL where clause on blob tags to operate only on destination blob with a matching value.
 
             .. versionadded:: 12.4.0
@@ -2287,7 +2295,7 @@ def commit_block_list( # type: ignore
             and act according to the condition specified by the `match_condition` parameter.
         :keyword ~azure.core.MatchConditions match_condition:
             The match condition to use upon the etag.
-        :keyword str if_tags_match_condition
+        :keyword str if_tags_match_condition:
             Specify a SQL where clause on blob tags to operate only on destination blob with a matching value.
 
             .. versionadded:: 12.4.0
@@ -2333,7 +2341,7 @@ def set_premium_page_blob_tier(self, premium_page_blob_tier, **kwargs):
             blob and number of allowed IOPS. This is only applicable to page blobs on
             premium storage accounts.
         :type premium_page_blob_tier: ~azure.storage.blob.PremiumPageBlobTier
-        :keyword str if_tags_match_condition
+        :keyword str if_tags_match_condition:
             Specify a SQL where clause on blob tags to operate only on blob with a matching value.
             eg. "\"tagname\"='my tag'"
 
@@ -2402,7 +2410,7 @@ def set_blob_tags(self, tags=None, **kwargs):
             bitflips on the wire if using http instead of https, as https (the default),
             will already validate. Note that this MD5 hash is not stored with the
             blob.
-        :keyword str if_tags_match_condition
+        :keyword str if_tags_match_condition:
             Specify a SQL where clause on blob tags to operate only on destination blob with a matching value.
         :keyword int timeout:
             The timeout parameter is expressed in seconds.
@@ -2438,7 +2446,7 @@ def get_blob_tags(self, **kwargs):
         :keyword str version_id:
             The version id parameter is an opaque DateTime
             value that, when present, specifies the version of the blob to add tags to.
-        :keyword str if_tags_match_condition
+        :keyword str if_tags_match_condition:
             Specify a SQL where clause on blob tags to operate only on destination blob with a matching value.
         :keyword int timeout:
             The timeout parameter is expressed in seconds.
@@ -2535,7 +2543,7 @@ def get_page_ranges( # type: ignore
             and act according to the condition specified by the `match_condition` parameter.
         :keyword ~azure.core.MatchConditions match_condition:
             The match condition to use upon the etag.
-        :keyword str if_tags_match_condition
+        :keyword str if_tags_match_condition:
             Specify a SQL where clause on blob tags to operate only on blob with a matching value.
             eg. "\"tagname\"='my tag'"
 
@@ -2684,7 +2692,7 @@ def set_sequence_number(self, sequence_number_action, sequence_number=None, **kw
             and act according to the condition specified by the `match_condition` parameter.
         :keyword ~azure.core.MatchConditions match_condition:
             The match condition to use upon the etag.
-        :keyword str if_tags_match_condition
+        :keyword str if_tags_match_condition:
             Specify a SQL where clause on blob tags to operate only on blob with a matching value.
             eg. "\"tagname\"='my tag'"
 
@@ -2758,7 +2766,7 @@ def resize_blob(self, size, **kwargs):
             and act according to the condition specified by the `match_condition` parameter.
         :keyword ~azure.core.MatchConditions match_condition:
             The match condition to use upon the etag.
-        :keyword str if_tags_match_condition
+        :keyword str if_tags_match_condition:
             Specify a SQL where clause on blob tags to operate only on blob with a matching value.
             eg. "\"tagname\"='my tag'"
 
@@ -2888,7 +2896,7 @@ def upload_page( # type: ignore
             and act according to the condition specified by the `match_condition` parameter.
         :keyword ~azure.core.MatchConditions match_condition:
             The match condition to use upon the etag.
-        :keyword str if_tags_match_condition
+        :keyword str if_tags_match_condition:
             Specify a SQL where clause on blob tags to operate only on blob with a matching value.
             eg. "\"tagname\"='my tag'"
 
@@ -3060,7 +3068,7 @@ def upload_pages_from_url(self, source_url,  # type: str
             and act according to the condition specified by the `match_condition` parameter.
         :keyword ~azure.core.MatchConditions match_condition:
             The destination match condition to use upon the etag.
-        :keyword str if_tags_match_condition
+        :keyword str if_tags_match_condition:
             Specify a SQL where clause on blob tags to operate only on blob with a matching value.
             eg. "\"tagname\"='my tag'"
 
@@ -3177,7 +3185,7 @@ def clear_page(self, offset, length, **kwargs):
             and act according to the condition specified by the `match_condition` parameter.
         :keyword ~azure.core.MatchConditions match_condition:
             The match condition to use upon the etag.
-        :keyword str if_tags_match_condition
+        :keyword str if_tags_match_condition:
             Specify a SQL where clause on blob tags to operate only on blob with a matching value.
             eg. "\"tagname\"='my tag'"
 
@@ -3307,7 +3315,7 @@ def append_block( # type: ignore
             and act according to the condition specified by the `match_condition` parameter.
         :keyword ~azure.core.MatchConditions match_condition:
             The match condition to use upon the etag.
-        :keyword str if_tags_match_condition
+        :keyword str if_tags_match_condition:
             Specify a SQL where clause on blob tags to operate only on blob with a matching value.
             eg. "\"tagname\"='my tag'"
 
@@ -3454,7 +3462,7 @@ def append_block_from_url(self, copy_source_url,  # type: str
             and act according to the condition specified by the `match_condition` parameter.
         :keyword ~azure.core.MatchConditions match_condition:
             The destination match condition to use upon the etag.
-        :keyword str if_tags_match_condition
+        :keyword str if_tags_match_condition:
             Specify a SQL where clause on blob tags to operate only on blob with a matching value.
             eg. "\"tagname\"='my tag'"
 

sdk/storage/azure-storage-blob/azure/storage/blob/_blob_service_client.py

@@ -383,6 +383,10 @@ def list_containers(
         :param bool include_metadata:
             Specifies that container metadata to be returned in the response.
             The default value is `False`.
+        :keyword bool include_deleted:
+            Specifies that deleted containers to be returned in the response. This is for container restore enabled
+            account. The default value is `False`.
+            .. versionadded:: 12.4.0
         :keyword int results_per_page:
             The maximum number of container names to retrieve per API
             call. If the request does not specify the server will return up to 5,000 items.
@@ -401,6 +405,9 @@ def list_containers(
                 :caption: Listing the containers in the blob service.
         """
         include = ['metadata'] if include_metadata else []
+        include_deleted = kwargs.pop('include_deleted', None)
+        if include_deleted:
+            include.append("deleted")
 
         timeout = kwargs.pop('timeout', None)
         results_per_page = kwargs.pop('results_per_page', None)
@@ -557,7 +564,7 @@ def delete_container(
             **kwargs)
 
     @distributed_trace
-    def _undelete_container(self, deleted_container_name, deleted_container_version, new_name=None, **kwargs):
+    def undelete_container(self, deleted_container_name, deleted_container_version, **kwargs):
         # type: (str, str, str, **Any) -> ContainerClient
         """Restores soft-deleted container.
 
@@ -571,12 +578,14 @@ def _undelete_container(self, deleted_container_name, deleted_container_version,
             Specifies the name of the deleted container to restore.
         :param str deleted_container_version:
             Specifies the version of the deleted container to restore.
-        :param str new_name:
+        :keyword str new_name:
             The new name for the deleted container to be restored to.
+            If not specified deleted_container_name will be used as the restored container name.
         :keyword int timeout:
             The timeout parameter is expressed in seconds.
         :rtype: ~azure.storage.blob.ContainerClient
         """
+        new_name = kwargs.pop('new_name', None)
         container = self.get_container_client(new_name or deleted_container_name)
         try:
             container._client.container.restore(deleted_container_name=deleted_container_name,  # pylint: disable = protected-access