Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[ADLS][Samples]Add Samples for sync and async #10123

Merged
merged 5 commits into from Mar 9, 2020
Merged

[ADLS][Samples]Add Samples for sync and async #10123

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0e61722
[ADLS][Samples]Add Samples for sync and async
xiafu-msft Mar 5, 2020
dd500a9
pylint
xiafu-msft Mar 6, 2020
cf2c697
[ADLS]Fix Code Snippet Indentation
xiafu-msft Mar 7, 2020
88798f0
Merge branch 'master' into datalake-samples
xiafu-msft Mar 9, 2020
0dda07c
fix pylint
xiafu-msft Mar 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 2 additions & 1 deletion sdk/storage/azure-storage-blob/samples/blob_samples_containers.py
View file
Open in desktop
Expand Up @@ -20,6 +20,7 @@
import os
from datetime import datetime, timedelta

from azure.core.exceptions import ResourceExistsError

SOURCE_FILE = 'SampleSource.txt'

Expand All @@ -44,7 +45,7 @@ def container_sample(self):
# [START create_container_client_sasurl]
from azure.storage.blob import ContainerClient

sas_url = sas_url = "https://account.blob.core.windows.net/mycontainer?sv=2015-04-05&st=2015-04-29T22%3A18%3A26Z&se=2015-04-30T02%3A23%3A26Z&sr=b&sp=rw&sip=168.1.5.60-168.1.5.70&spr=https&sig=Z%2FRHIX5Xcg0Mq2rqI3OlWTjEg2tYkboXr1P9ZUXDtkk%3D"
sas_url = "https://account.blob.core.windows.net/mycontainer?sv=2015-04-05&st=2015-04-29T22%3A18%3A26Z&se=2015-04-30T02%3A23%3A26Z&sr=b&sp=rw&sip=168.1.5.60-168.1.5.70&spr=https&sig=Z%2FRHIX5Xcg0Mq2rqI3OlWTjEg2tYkboXr1P9ZUXDtkk%3D"
rakshith91 marked this conversation as resolved.
Show resolved Hide resolved
container = ContainerClient.from_container_url(sas_url)
# [END create_container_client_sasurl]

Expand Down
86 changes: 43 additions & 43 deletions ...age/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_directory_client.py
View file
Open in desktop
Expand Up @@ -37,19 +37,12 @@ class DataLakeDirectoryClient(PathClient):

.. admonition:: Example:

.. literalinclude:: ../samples/test_datalake_authentication_samples.py
:start-after: [START create_datalake_service_client]
:end-before: [END create_datalake_service_client]
.. literalinclude:: ../samples/datalake_samples_instantiate_client.py
:start-after: [START instantiate_directory_client_from_conn_str]
:end-before: [END instantiate_directory_client_from_conn_str]
:language: python
:dedent: 8
:caption: Creating the DataLakeServiceClient with account url and credential.

.. literalinclude:: ../samples/test_datalake_authentication_samples.py
:start-after: [START create_datalake_service_client_oauth]
:end-before: [END create_datalake_service_client_oauth]
:language: python
:dedent: 8
:caption: Creating the DataLakeServiceClient with Azure Identity credentials.
:dedent: 4
:caption: Creating the DataLakeServiceClient from connection string.
"""
def __init__(
self, account_url, # type: str
Expand Down Expand Up @@ -95,18 +88,17 @@ def from_connection_string(
account_url, file_system_name=file_system_name, directory_name=directory_name,
credential=credential, **kwargs)

def create_directory(self, content_settings=None, # type: Optional[ContentSettings]
metadata=None, # type: Optional[Dict[str, str]]
def create_directory(self, metadata=None, # type: Optional[Dict[str, str]]
**kwargs):
# type: (...) -> Dict[str, Union[str, datetime]]
"""
Create a new directory.

:param ~azure.storage.filedatalake.ContentSettings content_settings:
ContentSettings object used to set path properties.
:param metadata:
Name-value pairs associated with the file as metadata.
:type metadata: dict(str, str)
:keyword ~azure.storage.filedatalake.ContentSettings content_settings:
ContentSettings object used to set path properties.
:keyword lease:
Required if the file has an active lease. Value can be a DataLakeLeaseClient object
or the lease ID as a string.
Expand Down Expand Up @@ -146,8 +138,17 @@ def create_directory(self, content_settings=None, # type: Optional[ContentSetti
:keyword int timeout:
The timeout parameter is expressed in seconds.
:return: response dict (Etag and last modified).

.. admonition:: Example:

.. literalinclude:: ../samples/datalake_samples_directory.py
:start-after: [START create_directory]
:end-before: [END create_directory]
:language: python
:dedent: 8
:caption: Create directory.
"""
return self._create('directory', content_settings=content_settings, metadata=metadata, **kwargs)
return self._create('directory', metadata=metadata, **kwargs)

def delete_directory(self, **kwargs):
# type: (...) -> None
Expand Down Expand Up @@ -178,6 +179,15 @@ def delete_directory(self, **kwargs):
:keyword int timeout:
The timeout parameter is expressed in seconds.
:return: None

.. admonition:: Example:
rakshith91 marked this conversation as resolved.
Show resolved Hide resolved

.. literalinclude:: ../samples/datalake_samples_directory.py
:start-after: [START delete_directory]
:end-before: [END delete_directory]
:language: python
:dedent: 4
:caption: Delete directory.
"""
return self._delete(**kwargs)

Expand Down Expand Up @@ -213,11 +223,11 @@ def get_directory_properties(self, **kwargs):

.. admonition:: Example:

.. literalinclude:: ../tests/test_blob_samples_common.py
:start-after: [START get_blob_properties]
:end-before: [END get_blob_properties]
.. literalinclude:: ../samples/datalake_samples_directory.py
:start-after: [START get_directory_properties]
:end-before: [END get_directory_properties]
:language: python
:dedent: 8
:dedent: 4
:caption: Getting the properties for a file/directory.
"""
blob_properties = self._get_path_properties(**kwargs)
Expand Down Expand Up @@ -296,6 +306,15 @@ def rename_directory(self, new_name, # type: str
:keyword int timeout:
The timeout parameter is expressed in seconds.
:return: DataLakeDirectoryClient

.. admonition:: Example:

.. literalinclude:: ../samples/datalake_samples_directory.py
:start-after: [START rename_directory]
:end-before: [END rename_directory]
:language: python
:dedent: 4
:caption: Rename the source directory.
"""
new_name = new_name.strip('/')
new_file_system = new_name.split('/')[0]
Expand All @@ -312,7 +331,6 @@ def rename_directory(self, new_name, # type: str
return new_directory_client

def create_sub_directory(self, sub_directory, # type: Union[DirectoryProperties, str]
content_settings=None, # type: Optional[ContentSettings]
metadata=None, # type: Optional[Dict[str, str]]
**kwargs):
# type: (...) -> DataLakeDirectoryClient
Expand All @@ -323,11 +341,11 @@ def create_sub_directory(self, sub_directory, # type: Union[DirectoryProperties
The directory with which to interact. This can either be the name of the directory,
or an instance of DirectoryProperties.
:type sub_directory: str or ~azure.storage.filedatalake.DirectoryProperties
:param ~azure.storage.filedatalake.ContentSettings content_settings:
ContentSettings object used to set path properties.
:param metadata:
Name-value pairs associated with the file as metadata.
:type metadata: dict(str, str)
:keyword ~azure.storage.filedatalake.ContentSettings content_settings:
ContentSettings object used to set path properties.
:keyword lease:
Required if the file has an active lease. Value can be a DataLakeLeaseClient object
or the lease ID as a string.
Expand Down Expand Up @@ -369,7 +387,7 @@ def create_sub_directory(self, sub_directory, # type: Union[DirectoryProperties
:return: DataLakeDirectoryClient for the subdirectory.
"""
subdir = self.get_sub_directory_client(sub_directory)
subdir.create_directory(content_settings=content_settings, metadata=metadata, **kwargs)
subdir.create_directory(metadata=metadata, **kwargs)
return subdir

def delete_sub_directory(self, sub_directory, # type: Union[DirectoryProperties, str]
Expand Down Expand Up @@ -483,15 +501,6 @@ def get_file_client(self, file # type: Union[FileProperties, str]
:type file: str or ~azure.storage.filedatalake.FileProperties
:returns: A DataLakeFileClient.
:rtype: ~azure.storage.filedatalake..DataLakeFileClient

.. admonition:: Example:

.. literalinclude:: ../samples/test_datalake_service_samples.py
:start-after: [START bsc_get_file_client]
:end-before: [END bsc_get_file_client]
:language: python
:dedent: 12
:caption: Getting the file client to interact with a specific file.
"""
try:
file_path = file.name
Expand All @@ -518,15 +527,6 @@ def get_sub_directory_client(self, sub_directory # type: Union[DirectoryPropert
:type sub_directory: str or ~azure.storage.filedatalake.DirectoryProperties
:returns: A DataLakeDirectoryClient.
:rtype: ~azure.storage.filedatalake.DataLakeDirectoryClient

.. admonition:: Example:

.. literalinclude:: ../samples/test_datalake_service_samples.py
:start-after: [START bsc_get_directory_client]
:end-before: [END bsc_get_directory_client]
:language: python
:dedent: 12
:caption: Getting the directory client to interact with a specific directory.
"""
try:
subdir_path = sub_directory.name
Expand Down
85 changes: 62 additions & 23 deletions sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client.py
View file
Open in desktop
Expand Up @@ -44,19 +44,12 @@ class DataLakeFileClient(PathClient):

.. admonition:: Example:

.. literalinclude:: ../samples/test_datalake_authentication_samples.py
:start-after: [START create_datalake_service_client]
:end-before: [END create_datalake_service_client]
.. literalinclude:: ../samples/datalake_samples_instantiate_client.py
:start-after: [START instantiate_file_client_from_conn_str]
:end-before: [END instantiate_file_client_from_conn_str]
:language: python
:dedent: 8
:caption: Creating the DataLakeServiceClient with account url and credential.

.. literalinclude:: ../samples/test_datalake_authentication_samples.py
:start-after: [START create_datalake_service_client_oauth]
:end-before: [END create_datalake_service_client_oauth]
:language: python
:dedent: 8
:caption: Creating the DataLakeServiceClient with Azure Identity credentials.
:dedent: 4
:caption: Creating the DataLakeServiceClient from connection string.
"""
def __init__(
self, account_url, # type: str
Expand Down Expand Up @@ -153,6 +146,15 @@ def create_file(self, content_settings=None, # type: Optional[ContentSettings]
:keyword int timeout:
The timeout parameter is expressed in seconds.
:return: response dict (Etag and last modified).

.. admonition:: Example:

.. literalinclude:: ../samples/datalake_samples_upload_download.py
:start-after: [START create_file]
:end-before: [END create_file]
:language: python
:dedent: 4
:caption: Create file.
"""
return self._create('file', content_settings=content_settings, metadata=metadata, **kwargs)

Expand Down Expand Up @@ -185,6 +187,15 @@ def delete_file(self, **kwargs):
:keyword int timeout:
The timeout parameter is expressed in seconds.
:return: None

.. admonition:: Example:

.. literalinclude:: ../samples/datalake_samples_upload_download.py
:start-after: [START delete_file]
:end-before: [END delete_file]
:language: python
:dedent: 4
:caption: Delete file.
"""
return self._delete(**kwargs)

Expand Down Expand Up @@ -220,12 +231,12 @@ def get_file_properties(self, **kwargs):

.. admonition:: Example:

.. literalinclude:: ../tests/test_blob_samples_common.py
:start-after: [START get_blob_properties]
:end-before: [END get_blob_properties]
.. literalinclude:: ../samples/datalake_samples_upload_download.py
:start-after: [START get_file_properties]
:end-before: [END get_file_properties]
:language: python
:dedent: 8
:caption: Getting the properties for a file/directory.
:dedent: 4
:caption: Getting the properties for a file.
"""
blob_properties = self._get_path_properties(**kwargs)
return FileProperties._from_blob_properties(blob_properties) # pylint: disable=protected-access
Expand Down Expand Up @@ -382,6 +393,15 @@ def append_data(self, data, # type: Union[AnyStr, Iterable[AnyStr], IO[AnyStr]]
or the lease ID as a string.
:paramtype lease: ~azure.storage.filedatalake.DataLakeLeaseClient or str
:return: dict of the response header

.. admonition:: Example:

.. literalinclude:: ../samples/datalake_samples_upload_download.py
:start-after: [START append_data]
:end-before: [END append_data]
:language: python
:dedent: 4
:caption: Append data to the file.
"""
options = self._append_data_options(
data,
Expand Down Expand Up @@ -464,6 +484,15 @@ def flush_data(self, offset, # type: int
:keyword ~azure.core.MatchConditions match_condition:
The match condition to use upon the etag.
:return: response header in dict

.. admonition:: Example:

.. literalinclude:: ../samples/datalake_samples_file_system.py
:start-after: [START upload_file_to_file_system]
:end-before: [END upload_file_to_file_system]
:language: python
:dedent: 8
:caption: Commit the previous appended data.
"""
options = self._flush_data_options(
offset,
Expand Down Expand Up @@ -521,12 +550,12 @@ def read_file(self, offset=None, # type: Optional[int]

.. admonition:: Example:

.. literalinclude:: ../tests/test_blob_samples_hello_world.py
:start-after: [START download_a_blob]
:end-before: [END download_a_blob]
.. literalinclude:: ../samples/datalake_samples_upload_download.py
:start-after: [START read_file]
:end-before: [END read_file]
:language: python
:dedent: 12
:caption: Download a blob.
:dedent: 4
:caption: Return the downloaded data.
"""
downloader = self._blob_client.download_blob(offset=offset, length=length, **kwargs)
if stream:
Expand Down Expand Up @@ -603,7 +632,17 @@ def rename_file(self, new_name, # type: str
The source match condition to use upon the etag.
:keyword int timeout:
The timeout parameter is expressed in seconds.
:return:
:return: the renamed file client
:rtype: DataLakeFileClient

.. admonition:: Example:

.. literalinclude:: ../samples/datalake_samples_upload_download.py
:start-after: [START rename_file]
:end-before: [END rename_file]
:language: python
:dedent: 4
:caption: Rename the source file.
"""
new_name = new_name.strip('/')
new_file_system = new_name.split('/')[0]
Expand Down