-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
c24aa6e
commit 8efa87a
Showing
4 changed files
with
61 additions
and
21 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,31 @@ | ||
""" | ||
Azure Utils | ||
""" | ||
def az_file_url_to_query_components(url:str): | ||
""" | ||
Helper function to parse an Azure file URL into its components to then be used by `pandas`/`dask`/`fsspec` etc. | ||
|
||
>>> az_file_url_to_query_components("https://storageaccount.blob.core.windows.net/container/file_path.parquet") | ||
{'storage_account': 'storageaccount', 'container': 'container', 'file_path': 'file_path.parquet'} | ||
""" | ||
|
||
|
||
p = urlparse(url) | ||
assert not p.params, f"Invalid Params: {p.params}" | ||
assert not p.fragment, f"Invalid Fragment: {p.fragment}" | ||
assert not p.query, f"Invalid Params: {p.query}" | ||
|
||
netlocs = p.netloc.split('.') | ||
assert len(netlocs) == 5, f"Invalid netlocs: {p.netloc}: Not long enough" | ||
assert netlocs[2:] == ['core','windows','net'], f"Invalid netlocs: {p.netloc} should end in core.windows.net" | ||
assert netlocs[1] in ['blob','dfs'], f"Invalid netlocs: {p.netloc} should be one of blob/dfs" | ||
|
||
storage_account = netlocs[0] | ||
_, container, *paths = p.path.split('/') #path starts with a / so p.path.split('/')[0] == '' | ||
file_path = '/'.join(paths) | ||
|
||
return dict( | ||
storage_account = storage_account, | ||
container = container, | ||
file_path = file_path | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,18 +0,0 @@ | ||
import logging | ||
import time | ||
from functools import wraps | ||
|
||
|
||
def timed(func): | ||
"""This decorator prints the execution time for the decorated function.""" | ||
|
||
@wraps(func) | ||
def wrapper(*args, **kwargs): | ||
start = time.time() | ||
logging.info(f"Launching {func.__name__}") | ||
result = func(*args, **kwargs) | ||
end = time.time() | ||
logging.info(f"{func.__name__} ran in {round(end - start, 2)}s") | ||
return result | ||
|
||
return wrapper | ||