Skip to content

added progress bar to get_acoustic_data #157

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Aug 28, 2023
Merged

added progress bar to get_acoustic_data #157

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c298c0b
added progress bar to get_acoustic_data
John-Ragland Aug 23, 2023
74c1e98
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Aug 23, 2023
894e8a9
added a few more print outs
John-Ragland Aug 23, 2023
e568267
Merge branch 'status_bar' of https://github.com/Ocean-Data-Lab/ooipy …
John-Ragland Aug 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions ooipy/request/hydrophone_request.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import requests
from obspy import Stream, read
from obspy.core import UTCDateTime
from tqdm import tqdm

# Import all dependencies
from ooipy.hydrophone.basic import HydrophoneData
Expand Down Expand Up @@ -245,7 +246,9 @@ def get_acoustic_data(

# Code Below from Landung Setiawan
# removed max workers argument in following statement
st_list = __map_concurrency(__read_mseed, valid_data_url_list)
st_list = __map_concurrency(__read_mseed, valid_data_url_list, verbose=verbose)

# combine list of single traces into stream of straces
st_all = None
for st in st_list:
if st:
Expand All @@ -265,6 +268,8 @@ def get_acoustic_data(

# Merge all traces together
# Interpolation
if verbose:
print("Merging Data...")
if data_gap_mode == 0:
st_all.merge(fill_value="interpolate", method=1)
# Masked Array
Expand Down Expand Up @@ -503,7 +508,7 @@ def ooipy_read(
return hydrophone_data


def __map_concurrency(func, iterator, args=(), max_workers=-1):
def __map_concurrency(func, iterator, args=(), max_workers=-1, verbose=False):
# automatically set max_workers to 2x(available cores)
if max_workers == -1:
max_workers = 2 * mp.cpu_count()
Expand All @@ -512,7 +517,9 @@ def __map_concurrency(func, iterator, args=(), max_workers=-1):
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
# Start the load operations and mark each future with its URL
future_to_url = {executor.submit(func, i, *args): i for i in iterator}
for future in concurrent.futures.as_completed(future_to_url):
for future in tqdm(
concurrent.futures.as_completed(future_to_url), total=len(iterator), disable=not verbose
):
data = future.result()
results.append(data)
return results
Expand Down