Skip to content

Commit

Permalink
use imap unordered to increase speed (#34)
Browse files Browse the repository at this point in the history
This PR updates to pool.imap_unordered to prevent it from blocking, meaning increased speeds when batch querying.
  • Loading branch information
Theodlz committed Jan 22, 2024
1 parent 6943466 commit a861af8
Showing 1 changed file with 9 additions and 10 deletions.
19 changes: 9 additions & 10 deletions penquins/penquins.py
Original file line number Diff line number Diff line change
Expand Up @@ -607,16 +607,15 @@ def batch_query(self, queries: Sequence[Mapping], n_threads: int = 4):

with ThreadPool(processes=n_threads) as pool:
if self.v:
return list(
tqdm(
pool.imap(
self.single_query,
queries_name_tpl,
# chunksize=len(queries_name_tpl),
)
)
)
return list(pool.imap(self.single_query, queries_name_tpl))
with tqdm(total=len(queries_name_tpl)) as pbar:
results = []
for result in pool.imap_unordered(
self.single_query, queries_name_tpl
):
results.append(result)
pbar.update(1)
return results
return list(pool.imap_unordered(self.single_query, queries_name_tpl))

def single_query(self, query_tpl: Tuple[Mapping, str]):
"""Call Kowalski's /api/queries endpoint using multiple processes
Expand Down

0 comments on commit a861af8

Please sign in to comment.