From cc09908683ca9621712df7e954fa0657e9da31d6 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 4 Jun 2025 16:13:19 +0330 Subject: [PATCH] feat: added max 10 parallel processing! --- hivemind_etl/mediawiki/etl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hivemind_etl/mediawiki/etl.py b/hivemind_etl/mediawiki/etl.py index dadc1db..209cd62 100644 --- a/hivemind_etl/mediawiki/etl.py +++ b/hivemind_etl/mediawiki/etl.py @@ -106,7 +106,7 @@ def load(self, documents: list[Document]) -> None: batch_size = 1000 batches = [documents[i:i + batch_size] for i in range(0, len(documents), batch_size)] - with ThreadPoolExecutor() as executor: + with ThreadPoolExecutor(max_workers=10) as executor: # Submit all batch processing tasks future_to_batch = { executor.submit(ingestion_pipeline.run_pipeline, batch): i