Don't blindly create coroutines, but fire them off in batches

freqtrade/exchange/exchange.py

@@ -28,7 +28,7 @@
 from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES,
                                        EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, retrier,
                                        retrier_async)
-from freqtrade.misc import deep_merge_dicts, safe_value_fallback2
+from freqtrade.misc import chunks, deep_merge_dicts, safe_value_fallback2
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 
 
@@ -1237,19 +1237,22 @@ async def _async_get_historic_ohlcv(self, pair: str,
         input_coroutines = [self._async_get_candle_history(
             pair, timeframe, since) for since in
             range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)]
-
-        results = await asyncio.gather(*input_coroutines, return_exceptions=True)
-
         # Combine gathered results
+
         data: List = []
-        for res in results:
-            if isinstance(res, Exception):
-                logger.warning("Async code raised an exception: %s", res.__class__.__name__)
-                continue
-            # Deconstruct tuple if it's not an exception
-            p, _, new_data = res
-            if p == pair:
-                data.extend(new_data)
+        for input_coro in chunks(input_coroutines, 100):
+            logger.info(f"Starting with {len(input_coro)}")
+
+            results = await asyncio.gather(*input_coro, return_exceptions=True)
+            logger.info(f"Returned with {len(input_coro)}")
+            for res in results:
+                if isinstance(res, Exception):
+                    logger.warning("Async code raised an exception: %s", res.__class__.__name__)
+                    continue
+                # Deconstruct tuple if it's not an exception
+                p, _, new_data = res
+                if p == pair:
+                    data.extend(new_data)
         # Sort data again after extending the result - above calls return in "async order"
         data = sorted(data, key=lambda x: x[0])
         logger.info("Downloaded data for %s with length %s.", pair, len(data))