-
Notifications
You must be signed in to change notification settings - Fork 150
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
chore: Amend Transfer Manager samples (#1113)
* chore: Amend Transfer Manager samples * tests * tests again * respond to feedback
- Loading branch information
Showing
5 changed files
with
111 additions
and
18 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
74 changes: 74 additions & 0 deletions
74
samples/snippets/storage_transfer_manager_download_bucket.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,74 @@ | ||
# Copyright 2022 Google LLC | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the 'License'); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
# [START storage_transfer_manager_download_bucket] | ||
def download_bucket_with_transfer_manager( | ||
bucket_name, destination_directory="", processes=8, max_results=1000 | ||
): | ||
"""Download all of the blobs in a bucket, concurrently in a process pool. | ||
The filename of each blob once downloaded is derived from the blob name and | ||
the `destination_directory `parameter. For complete control of the filename | ||
of each blob, use transfer_manager.download_many() instead. | ||
Directories will be created automatically as needed, for instance to | ||
accommodate blob names that include slashes. | ||
""" | ||
|
||
# The ID of your GCS bucket | ||
# bucket_name = "your-bucket-name" | ||
|
||
# The directory on your computer to which to download all of the files. This | ||
# string is prepended (with os.path.join()) to the name of each blob to form | ||
# the full path. Relative paths and absolute paths are both accepted. An | ||
# empty string means "the current working directory". Note that this | ||
# parameter allows accepts directory traversal ("../" etc.) and is not | ||
# intended for unsanitized end user input. | ||
# destination_directory = "" | ||
|
||
# The maximum number of processes to use for the operation. The performance | ||
# impact of this value depends on the use case, but smaller files usually | ||
# benefit from a higher number of processes. Each additional process occupies | ||
# some CPU and memory resources until finished. | ||
# processes=8 | ||
|
||
# The maximum number of results to fetch from bucket.list_blobs(). This | ||
# sample code fetches all of the blobs up to max_results and queues them all | ||
# for download at once. Though they will still be executed in batches up to | ||
# the processes limit, queueing them all at once can be taxing on system | ||
# memory if buckets are very large. Adjust max_results as needed for your | ||
# system environment, or set it to None if you are sure the bucket is not | ||
# too large to hold in memory easily. | ||
# max_results=1000 | ||
|
||
from google.cloud.storage import Client, transfer_manager | ||
|
||
storage_client = Client() | ||
bucket = storage_client.bucket(bucket_name) | ||
|
||
blob_names = [blob.name for blob in bucket.list_blobs(max_results=max_results)] | ||
|
||
results = transfer_manager.download_many_to_path( | ||
bucket, blob_names, destination_directory=destination_directory, max_workers=processes | ||
) | ||
|
||
for name, result in zip(blob_names, results): | ||
# The results list is either `None` or an exception for each blob in | ||
# the input list, in order. | ||
|
||
if isinstance(result, Exception): | ||
print("Failed to download {} due to exception: {}".format(name, result)) | ||
else: | ||
print("Downloaded {} to {}.".format(name, destination_directory + name)) | ||
# [END storage_transfer_manager_download_bucket] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters