From 505e98c296c5b7e58cd240c500ac0779c0c6230f Mon Sep 17 00:00:00 2001 From: Giovanni Azua Garcia Date: Sun, 5 Oct 2025 12:51:15 +0200 Subject: [PATCH 1/2] Implementation for issue #1318 --- backtesting/backtesting.py | 412 ++++++++++++++++++++++--------------- 1 file changed, 243 insertions(+), 169 deletions(-) diff --git a/backtesting/backtesting.py b/backtesting/backtesting.py index 39fb80f1..39d417d1 100644 --- a/backtesting/backtesting.py +++ b/backtesting/backtesting.py @@ -16,6 +16,7 @@ from itertools import chain, product, repeat from math import copysign from numbers import Number +from difflib import get_close_matches from typing import Callable, List, Optional, Sequence, Tuple, Type, Union import numpy as np @@ -64,10 +65,12 @@ def __str__(self): def _check_params(self, params): for k, v in params.items(): if not hasattr(self, k): + suggestions = get_close_matches(k, [a for a in dir(self) if not a.startswith('_')], n=3) + hint = f" Did you mean: {', '.join(suggestions)}?" if suggestions else "" raise AttributeError( - f"Strategy '{self.__class__.__name__}' is missing parameter '{k}'." + f"Strategy '{self.__class__.__name__}' is missing parameter '{k}'. " "Strategy class should define parameters as class variables before they " - "can be optimized or run with.") + "can be optimized or run with." + hint) setattr(self, k, v) return params @@ -139,6 +142,8 @@ def _format_name(name: str) -> str: try: value = func(*args, **kwargs) + if isinstance(value, pd.Series): + value = value.to_numpy() except Exception as e: raise RuntimeError(f'Indicator "{name}" error. See traceback above.') from e @@ -158,6 +163,12 @@ def _format_name(name: str) -> str: f'Length of `name=` ({len(name)}) must agree with the number ' f'of arrays the indicator returns ({value.shape[0]}).') + # refine orientation heuristic: if last dim doesn't match data length but first does, transpose + if is_arraylike: + L = len(self._data.Close) + if value.shape[-1] != L and value.shape[0] == L: + value = value.T + if not is_arraylike or not 1 <= value.ndim <= 2 or value.shape[-1] != len(self._data.Close): raise ValueError( 'Indicators must return (optionally a tuple of) numpy.arrays of same ' @@ -337,7 +348,7 @@ def __getattr__(self, item): removed_attrs = ('entry', 'set_entry', 'is_long', 'is_short', 'sl', 'tp', 'set_sl', 'set_tp') if item in removed_attrs: - raise AttributeError(f'Strategy.orders.{"/.".join(removed_attrs)} were removed in' + raise AttributeError(f'Strategy.orders.{"/.".join(removed_attrs)} were removed in ' 'Backtesting 0.2.0. ' 'Use `Order` API instead. See docs.') raise AttributeError(f"'tuple' object has no attribute {item!r}") @@ -373,7 +384,8 @@ def pl(self) -> float: def pl_pct(self) -> float: """Profit (positive) or loss (negative) of the current position in percent.""" total_invested = sum(trade.entry_price * abs(trade.size) for trade in self.__broker.trades) - return (self.pl / total_invested) * 100 if total_invested else 0 + den = total_invested + return (self.pl / den) * 100 if den and den > 0 else 0.0 @property def is_long(self) -> bool: @@ -448,7 +460,7 @@ def __repr__(self): ('tp', self.__tp_price), ('contingent', self.is_contingent), ('tag', self.__tag), - ) if value is not None)) # noqa: E126 + ) if (value is not None and (not isinstance(value, bool) or value)))) # noqa: E126 def cancel(self): """Cancel the order.""" @@ -677,7 +689,10 @@ def pl(self): Commissions are reflected only after the Trade is closed. """ price = self.__exit_price or self.__broker.last_price - return (self.__size * (price - self.__entry_price)) - self._commissions + # optionally include entry commission while trade is open + open_comm = getattr(self, "_open_commission", 0) + include_open = (self.__exit_price is None and getattr(self.__broker, "_commission_when", "exit") == "both") + return (self.__size * (price - self.__entry_price)) - (self._commissions if self.__exit_price is not None else 0) - (open_comm if include_open else 0) @property def pl_pct(self): @@ -686,7 +701,10 @@ def pl_pct(self): gross_pl_pct = copysign(1, self.__size) * (price / self.__entry_price - 1) # Total commission across the entire trade size to individual units - commission_pct = self._commissions / (abs(self.__size) * self.__entry_price) + den = abs(self.__size) * self.__entry_price + commission_pct = (self._commissions / den) if den and den > 0 else 0.0 + if self.__exit_price is None and getattr(self.__broker, "_commission_when", "exit") == "both" and den and den > 0: + commission_pct += (getattr(self, "_open_commission", 0) / den) return gross_pl_pct - commission_pct @property @@ -768,6 +786,8 @@ def __init__(self, *, data, cash, spread, commission, margin, self._exclusive_orders = exclusive_orders self._equity = np.tile(np.nan, len(index)) + # commission timing: 'exit' (default) or 'both' (include entry commission in open PL) + self._commission_when = getattr(self, "_commission_when", "exit") self.orders: List[Order] = [] self.trades: List[Trade] = [] self.position = Position(self) @@ -797,6 +817,20 @@ def new_order(self, sl = sl and float(sl) tp = tp and float(tp) + # validate contingent SL/TP against entry price if modifying an existing trade + if trade: + ep = trade.entry_price + if trade.is_long: + if stop and stop >= ep: + raise ValueError('SL for long must be below entry price') + if limit and limit <= ep: + raise ValueError('TP for long must be above entry price') + else: + if stop and stop <= ep: + raise ValueError('SL for short must be above entry price') + if limit and limit >= ep: + raise ValueError('TP for short must be below entry price') + is_long = size > 0 assert size != 0, size adjusted_price = self._adjusted_price(size) @@ -866,179 +900,188 @@ def next(self): self._close_trade(trade, self._data.Close[-1], i) self._cash = 0 self._equity[i:] = 0 + self.orders.clear() raise _OutOfMoneyError def _process_orders(self): data = self._data - open, high, low = data.Open[-1], data.High[-1], data.Low[-1] - reprocess_orders = False + while True: + open, high, low = data.Open[-1], data.High[-1], data.Low[-1] + reprocess_orders = False - # Process orders - for order in list(self.orders): # type: Order - - # Related SL/TP order was already removed - if order not in self.orders: - continue - - # Check if stop condition was hit - stop_price = order.stop - if stop_price: - is_stop_hit = ((high >= stop_price) if order.is_long else (low <= stop_price)) - if not is_stop_hit: - continue + # Process orders + for order in list(self.orders): # type: Order - # > When the stop price is reached, a stop order becomes a market/limit order. - # https://www.sec.gov/fast-answers/answersstopordhtm.html - order._replace(stop_price=None) - - # Determine purchase price. - # Check if limit order can be filled. - if order.limit: - is_limit_hit = low <= order.limit if order.is_long else high >= order.limit - # When stop and limit are hit within the same bar, we pessimistically - # assume limit was hit before the stop (i.e. "before it counts") - is_limit_hit_before_stop = (is_limit_hit and - (order.limit <= (stop_price or -np.inf) - if order.is_long - else order.limit >= (stop_price or np.inf))) - if not is_limit_hit or is_limit_hit_before_stop: + # Related SL/TP order was already removed + if order not in self.orders: continue - # stop_price, if set, was hit within this bar - price = (min(stop_price or open, order.limit) - if order.is_long else - max(stop_price or open, order.limit)) - else: - # Market-if-touched / market order - # Contingent orders always on next open - prev_close = data.Close[-2] - price = prev_close if self._trade_on_close and not order.is_contingent else open + # Check if stop condition was hit + stop_price = order.stop if stop_price: - price = max(price, stop_price) if order.is_long else min(price, stop_price) - - # Determine entry/exit bar index - is_market_order = not order.limit and not stop_price - time_index = ( - (self._i - 1) - if is_market_order and self._trade_on_close and not order.is_contingent else - self._i) - - # If order is a SL/TP order, it should close an existing trade it was contingent upon - if order.parent_trade: - trade = order.parent_trade - _prev_size = trade.size - # If order.size is "greater" than trade.size, this order is a trade.close() - # order and part of the trade was already closed beforehand - size = copysign(min(abs(_prev_size), abs(order.size)), order.size) - # If this trade isn't already closed (e.g. on multiple `trade.close(.5)` calls) - if trade in self.trades: - self._reduce_trade(trade, price, size, time_index) - assert order.size != -_prev_size or trade not in self.trades - if price == stop_price: - # Set SL back on the order for stats._trades["SL"] - trade._sl_order._replace(stop_price=stop_price) - if order in (trade._sl_order, - trade._tp_order): - assert order.size == -trade.size - assert order not in self.orders # Removed when trade was closed - else: - # It's a trade.close() order, now done - assert abs(_prev_size) >= abs(size) >= 1 - self.orders.remove(order) - continue - - # Else this is a stand-alone trade - - # Adjust price to include commission (or bid-ask spread). - # In long positions, the adjusted price is a fraction higher, and vice versa. - adjusted_price = self._adjusted_price(order.size, price) - adjusted_price_plus_commission = \ - adjusted_price + self._commission(order.size, price) / abs(order.size) - - # If order size was specified proportionally, - # precompute true size in units, accounting for margin and spread/commissions - size = order.size - if -1 < size < 1: - size = copysign(int((self.margin_available * self._leverage * abs(size)) - // adjusted_price_plus_commission), size) - # Not enough cash/margin even for a single unit - if not size: - warnings.warn( - f'time={self._i}: Broker canceled the relative-sized ' - f'order due to insufficient margin.', category=UserWarning) - # XXX: The order is canceled by the broker? - self.orders.remove(order) - continue - assert size == round(size) - need_size = int(size) - - if not self._hedging: - # Fill position by FIFO closing/reducing existing opposite-facing trades. - # Existing trades are closed at unadjusted price, because the adjustment - # was already made when buying. - for trade in list(self.trades): - if trade.is_long == order.is_long: + is_stop_hit = ((high >= stop_price) if order.is_long else (low <= stop_price)) + if not is_stop_hit: continue - assert trade.size * order.size < 0 - # Order size greater than this opposite-directed existing trade, - # so it will be closed completely - if abs(need_size) >= abs(trade.size): - self._close_trade(trade, price, time_index) - need_size += trade.size - else: - # The existing trade is larger than the new order, - # so it will only be closed partially - self._reduce_trade(trade, price, need_size, time_index) - need_size = 0 + # > When the stop price is reached, a stop order becomes a market/limit order. + # https://www.sec.gov/fast-answers/answersstopordhtm.html + order._replace(stop_price=None) + + # Determine purchase price. + # Check if limit order can be filled. + if order.limit: + is_limit_hit = low <= order.limit if order.is_long else high >= order.limit + # When stop and limit are hit within the same bar, we pessimistically + # assume limit was hit before the stop (i.e. "before it counts") + is_limit_hit_before_stop = (is_limit_hit and + (order.limit <= (stop_price or -np.inf) + if order.is_long + else order.limit >= (stop_price or np.inf))) + if not is_limit_hit or is_limit_hit_before_stop: + continue - if not need_size: - break + # stop_price, if set, was hit within this bar + price = (min(stop_price or open, order.limit) + if order.is_long else + max(stop_price or open, order.limit)) + else: + # Market-if-touched / market order + # Contingent orders always on next open + prev_close = data.Close[-2] + price = prev_close if self._trade_on_close and not order.is_contingent else open + if stop_price: + price = max(price, stop_price) if order.is_long else min(price, stop_price) + + # Determine entry/exit bar index + is_market_order = not order.limit and not stop_price + time_index = ( + (self._i - 1) + if is_market_order and self._trade_on_close and not order.is_contingent else + self._i) + + # If order is a SL/TP order, it should close an existing trade it was contingent upon + if order.parent_trade: + trade = order.parent_trade + _prev_size = trade.size + # If order.size is "greater" than trade.size, this order is a trade.close() + # order and part of the trade was already closed beforehand + size = copysign(min(abs(_prev_size), abs(order.size)), order.size) + # If this trade isn't already closed (e.g. on multiple `trade.close(.5)` calls) + if trade in self.trades: + self._reduce_trade(trade, price, size, time_index) + assert order.size != -_prev_size or trade not in self.trades + if price == stop_price: + # Set SL back on the order for stats._trades["SL"] + trade._sl_order._replace(stop_price=stop_price) + if order in (trade._sl_order, + trade._tp_order): + assert order.size == -trade.size + assert order not in self.orders # Removed when trade was closed + else: + # It's a trade.close() order, now done + assert abs(_prev_size) >= abs(size) >= 1 + self.orders.remove(order) + continue - # If we don't have enough liquidity to cover for the order, the broker CANCELS it - if abs(need_size) * adjusted_price_plus_commission > \ - self.margin_available * self._leverage: - self.orders.remove(order) - continue - - # Open a new trade - if need_size: - self._open_trade(adjusted_price, - need_size, - order.sl, - order.tp, - time_index, - order.tag) - - # We need to reprocess the SL/TP orders newly added to the queue. - # This allows e.g. SL hitting in the same bar the order was open. - # See https://github.com/kernc/backtesting.py/issues/119 - if order.sl or order.tp: - if is_market_order: - reprocess_orders = True - # Order.stop and TP hit within the same bar, but SL wasn't. This case - # is not ambiguous, because stop and TP go in the same price direction. - elif stop_price and not order.limit and order.tp and ( - (order.is_long and order.tp <= high and (order.sl or -np.inf) < low) or - (order.is_short and order.tp >= low and (order.sl or np.inf) > high)): - reprocess_orders = True - elif (low <= (order.sl or -np.inf) <= high or - low <= (order.tp or -np.inf) <= high): + # Else this is a stand-alone trade + + # Adjust price to include commission (or bid-ask spread). + # In long positions, the adjusted price is a fraction higher, and vice versa. + adjusted_price = self._adjusted_price(order.size, price) + adjusted_price_plus_commission = \ + adjusted_price + self._commission(order.size, price) / abs(order.size) + + # If order size was specified proportionally, + # precompute true size in units, accounting for margin and spread/commissions + size = order.size + if -1 < size < 1: + size = copysign(int((self.margin_available * self._leverage * abs(size)) + // adjusted_price_plus_commission), size) + # Not enough cash/margin even for a single unit + if not size: warnings.warn( - f"({data.index[-1]}) A contingent SL/TP order would execute in the " - "same bar its parent stop/limit order was turned into a trade. " - "Since we can't assert the precise intra-candle " - "price movement, the affected SL/TP order will instead be executed on " - "the next (matching) price/bar, making the result (of this trade) " - "somewhat dubious. " - "See https://github.com/kernc/backtesting.py/issues/119", - UserWarning) + f'({data.index[self._i]}) broker canceled the relative-sized order ' + f'{order} due to insufficient margin ' + f'(equity={self.equity:.2f}, margin_available={self.margin_available:.2f}).', + category=UserWarning) + # XXX: The order is canceled by the broker? + self.orders.remove(order) + continue + assert size == round(size) + need_size = int(size) + + if not self._hedging: + # Fill position by FIFO closing/reducing existing opposite-facing trades. + # Existing trades are closed at unadjusted price, because the adjustment + # was already made when buying. + for trade in list(self.trades): + if trade.is_long == order.is_long: + continue + assert trade.size * order.size < 0 + + # Order size greater than this opposite-directed existing trade, + # so it will be closed completely + if abs(need_size) >= abs(trade.size): + self._close_trade(trade, price, time_index) + need_size += trade.size + else: + # The existing trade is larger than the new order, + # so it will only be closed partially + self._reduce_trade(trade, price, need_size, time_index) + need_size = 0 + + if not need_size: + break + + # If we don't have enough liquidity to cover for the order, the broker CANCELS it + if abs(need_size) * adjusted_price_plus_commission > \ + self.margin_available * self._leverage: + warnings.warn( + f'({data.index[self._i]}) broker canceled order {order} due to insufficient margin ' + f'(equity={self.equity:.2f}, margin_available={self.margin_available:.2f}).', + category=UserWarning) + self.orders.remove(order) + continue - # Order processed - self.orders.remove(order) + # Open a new trade + if need_size: + self._open_trade(adjusted_price, + need_size, + order.sl, + order.tp, + time_index, + order.tag) + + # We need to reprocess the SL/TP orders newly added to the queue. + # This allows e.g. SL hitting in the same bar the order was open. + # See https://github.com/kernc/backtesting.py/issues/119 + if order.sl or order.tp: + if is_market_order: + reprocess_orders = True + # Order.stop and TP hit within the same bar, but SL wasn't. This case + # is not ambiguous, because stop and TP go in the same price direction. + elif stop_price and not order.limit and order.tp and ( + (order.is_long and order.tp <= high and (order.sl or -np.inf) < low) or + (order.is_short and order.tp >= low and (order.sl or np.inf) > high)): + reprocess_orders = True + elif (low <= (order.sl or -np.inf) <= high or + low <= (order.tp or -np.inf) <= high): + warnings.warn( + f"({data.index[-1]}) A contingent SL/TP order would execute in the " + "same bar its parent stop/limit order was turned into a trade. " + "Since we can't assert the precise intra-candle " + "price movement, the affected SL/TP order will instead be executed on " + "the next (matching) price/bar, making the result (of this trade) " + "somewhat dubious. " + "See https://github.com/kernc/backtesting.py/issues/119", + UserWarning) + + # Order processed + self.orders.remove(order) - if reprocess_orders: - self._process_orders() + if not reprocess_orders: + break + # end while def _reduce_trade(self, trade: Trade, price: float, size: float, time_index: int): assert trade.size * size < 0 @@ -1085,7 +1128,9 @@ def _open_trade(self, price: float, size: int, trade = Trade(self, size, price, time_index, tag) self.trades.append(trade) # Apply broker commission at trade open - self._cash -= self._commission(size, price) + open_commission = self._commission(size, price) + trade._open_commission = open_commission + self._cash -= open_commission # Create SL/TP (bracket) orders. if tp: trade.tp = tp @@ -1151,7 +1196,7 @@ class Backtest: `margin` is the required margin (ratio) of a leveraged account. No difference is made between initial and maintenance margins. - To run the backtest using e.g. 50:1 leverge that your broker allows, + To run the backtest using e.g. 50:1 leverage that your broker allows, set margin to `0.02` (1 / leverage). If `trade_on_close` is `True`, market orders will be filled @@ -1189,6 +1234,8 @@ def __init__(self, hedging=False, exclusive_orders=False, finalize_trades=False, + risk_free_rate: float = 0.0, + commission_when: str = 'exit', ): if not (isinstance(strategy, type) and issubclass(strategy, Strategy)): raise TypeError('`strategy` must be a Strategy sub-type') @@ -1212,7 +1259,15 @@ def __init__(self, (data.index.is_numeric() and (data.index > pd.Timestamp('1975').timestamp()).mean() > .8)): try: - data.index = pd.to_datetime(data.index, infer_datetime_format=True) + # try seconds vs milliseconds heuristic first + idx = np.asarray(data.index) + med = np.median(idx.astype('float64')) + if med > 1e12: + data.index = pd.to_datetime(data.index, unit='ms') + elif med > 1e9: + data.index = pd.to_datetime(data.index, unit='s') + else: + data.index = pd.to_datetime(data.index, infer_datetime_format=True) except ValueError: pass @@ -1228,6 +1283,13 @@ def __init__(self, raise ValueError('Some OHLC values are missing (NaN). ' 'Please strip those lines with `df.dropna()` or ' 'fill them in with `df.interpolate()` or whatever.') + # Optional sanity warnings for inconsistent OHLC (non-breaking) + try: + if not ((data['High'] >= data[['Open', 'Close']].max(axis=1)) & + (data['Low'] <= data[['Open', 'Close']].min(axis=1))).all(): + warnings.warn('Some OHLC rows have inconsistent High/Low vs Open/Close.', stacklevel=2) + except Exception: + pass if np.any(data['Close'] > cash): warnings.warn('Some prices are larger than initial cash value. Note that fractional ' 'trading is not supported by this class. If you want to trade Bitcoin, ' @@ -1244,6 +1306,8 @@ def __init__(self, stacklevel=2) self._data: pd.DataFrame = data + self._risk_free_rate = float(risk_free_rate) + self._commission_when = commission_when if commission_when in ('exit', 'both') else 'exit' self._broker = partial( _Broker, cash=cash, spread=spread, commission=commission, margin=margin, trade_on_close=trade_on_close, hedging=hedging, @@ -1305,6 +1369,8 @@ def run(self, **kwargs) -> pd.Series: """ data = _Data(self._data.copy(deep=False)) broker: _Broker = self._broker(data=data) + # propagate commission PL behavior + broker._commission_when = self._commission_when strategy: Strategy = self._strategy(broker, data, kwargs) strategy.init() @@ -1357,12 +1423,20 @@ def run(self, **kwargs) -> pd.Series: # for future `indicator._opts['data'].index` calls to work data._set_length(len(self._data)) - equity = pd.Series(broker._equity).bfill().fillna(broker._cash).values + # backfill equity with NumPy to avoid pandas overhead + equity = broker._equity.copy() + if np.isnan(equity).any(): + idx = np.arange(len(equity)) + valid = ~np.isnan(equity) + if valid.any(): + last = np.maximum.accumulate(np.where(valid, idx, 0)) + equity = equity[last] + equity[np.isnan(equity)] = broker._cash self._results = compute_stats( trades=broker.closed_trades, equity=equity, ohlc_data=self._data, - risk_free_rate=0.0, + risk_free_rate=self._risk_free_rate, strategy_instance=strategy, ) From cc1ebc8953fbeeb1904345171e01fd804ce071c2 Mon Sep 17 00:00:00 2001 From: Giovanni Azua Garcia Date: Sun, 5 Oct 2025 13:11:18 +0200 Subject: [PATCH 2/2] Implementation for issue #1319 wip --- backtesting/backtesting.py | 399 ++++++++++++++++--------------------- 1 file changed, 168 insertions(+), 231 deletions(-) diff --git a/backtesting/backtesting.py b/backtesting/backtesting.py index 39d417d1..c1dc1c93 100644 --- a/backtesting/backtesting.py +++ b/backtesting/backtesting.py @@ -163,12 +163,6 @@ def _format_name(name: str) -> str: f'Length of `name=` ({len(name)}) must agree with the number ' f'of arrays the indicator returns ({value.shape[0]}).') - # refine orientation heuristic: if last dim doesn't match data length but first does, transpose - if is_arraylike: - L = len(self._data.Close) - if value.shape[-1] != L and value.shape[0] == L: - value = value.T - if not is_arraylike or not 1 <= value.ndim <= 2 or value.shape[-1] != len(self._data.Close): raise ValueError( 'Indicators must return (optionally a tuple of) numpy.arrays of same ' @@ -384,8 +378,7 @@ def pl(self) -> float: def pl_pct(self) -> float: """Profit (positive) or loss (negative) of the current position in percent.""" total_invested = sum(trade.entry_price * abs(trade.size) for trade in self.__broker.trades) - den = total_invested - return (self.pl / den) * 100 if den and den > 0 else 0.0 + return (self.pl / total_invested) * 100 if total_invested else 0 @property def is_long(self) -> bool: @@ -689,10 +682,7 @@ def pl(self): Commissions are reflected only after the Trade is closed. """ price = self.__exit_price or self.__broker.last_price - # optionally include entry commission while trade is open - open_comm = getattr(self, "_open_commission", 0) - include_open = (self.__exit_price is None and getattr(self.__broker, "_commission_when", "exit") == "both") - return (self.__size * (price - self.__entry_price)) - (self._commissions if self.__exit_price is not None else 0) - (open_comm if include_open else 0) + return (self.__size * (price - self.__entry_price)) - self._commissions @property def pl_pct(self): @@ -701,10 +691,7 @@ def pl_pct(self): gross_pl_pct = copysign(1, self.__size) * (price / self.__entry_price - 1) # Total commission across the entire trade size to individual units - den = abs(self.__size) * self.__entry_price - commission_pct = (self._commissions / den) if den and den > 0 else 0.0 - if self.__exit_price is None and getattr(self.__broker, "_commission_when", "exit") == "both" and den and den > 0: - commission_pct += (getattr(self, "_open_commission", 0) / den) + commission_pct = self._commissions / (abs(self.__size) * self.__entry_price) return gross_pl_pct - commission_pct @property @@ -786,8 +773,6 @@ def __init__(self, *, data, cash, spread, commission, margin, self._exclusive_orders = exclusive_orders self._equity = np.tile(np.nan, len(index)) - # commission timing: 'exit' (default) or 'both' (include entry commission in open PL) - self._commission_when = getattr(self, "_commission_when", "exit") self.orders: List[Order] = [] self.trades: List[Trade] = [] self.position = Position(self) @@ -817,20 +802,6 @@ def new_order(self, sl = sl and float(sl) tp = tp and float(tp) - # validate contingent SL/TP against entry price if modifying an existing trade - if trade: - ep = trade.entry_price - if trade.is_long: - if stop and stop >= ep: - raise ValueError('SL for long must be below entry price') - if limit and limit <= ep: - raise ValueError('TP for long must be above entry price') - else: - if stop and stop <= ep: - raise ValueError('SL for short must be above entry price') - if limit and limit >= ep: - raise ValueError('TP for short must be below entry price') - is_long = size > 0 assert size != 0, size adjusted_price = self._adjusted_price(size) @@ -900,188 +871,185 @@ def next(self): self._close_trade(trade, self._data.Close[-1], i) self._cash = 0 self._equity[i:] = 0 - self.orders.clear() raise _OutOfMoneyError def _process_orders(self): data = self._data - while True: - open, high, low = data.Open[-1], data.High[-1], data.Low[-1] - reprocess_orders = False - - # Process orders - for order in list(self.orders): # type: Order + open, high, low = data.Open[-1], data.High[-1], data.Low[-1] + reprocess_orders = False - # Related SL/TP order was already removed - if order not in self.orders: - continue + # Process orders + for order in list(self.orders): # type: Order - # Check if stop condition was hit - stop_price = order.stop - if stop_price: - is_stop_hit = ((high >= stop_price) if order.is_long else (low <= stop_price)) - if not is_stop_hit: - continue + # Related SL/TP order was already removed + if order not in self.orders: + continue - # > When the stop price is reached, a stop order becomes a market/limit order. - # https://www.sec.gov/fast-answers/answersstopordhtm.html - order._replace(stop_price=None) - - # Determine purchase price. - # Check if limit order can be filled. - if order.limit: - is_limit_hit = low <= order.limit if order.is_long else high >= order.limit - # When stop and limit are hit within the same bar, we pessimistically - # assume limit was hit before the stop (i.e. "before it counts") - is_limit_hit_before_stop = (is_limit_hit and - (order.limit <= (stop_price or -np.inf) - if order.is_long - else order.limit >= (stop_price or np.inf))) - if not is_limit_hit or is_limit_hit_before_stop: - continue + # Check if stop condition was hit + stop_price = order.stop + if stop_price: + is_stop_hit = ((high >= stop_price) if order.is_long else (low <= stop_price)) + if not is_stop_hit: + continue - # stop_price, if set, was hit within this bar - price = (min(stop_price or open, order.limit) - if order.is_long else - max(stop_price or open, order.limit)) - else: - # Market-if-touched / market order - # Contingent orders always on next open - prev_close = data.Close[-2] - price = prev_close if self._trade_on_close and not order.is_contingent else open - if stop_price: - price = max(price, stop_price) if order.is_long else min(price, stop_price) - - # Determine entry/exit bar index - is_market_order = not order.limit and not stop_price - time_index = ( - (self._i - 1) - if is_market_order and self._trade_on_close and not order.is_contingent else - self._i) - - # If order is a SL/TP order, it should close an existing trade it was contingent upon - if order.parent_trade: - trade = order.parent_trade - _prev_size = trade.size - # If order.size is "greater" than trade.size, this order is a trade.close() - # order and part of the trade was already closed beforehand - size = copysign(min(abs(_prev_size), abs(order.size)), order.size) - # If this trade isn't already closed (e.g. on multiple `trade.close(.5)` calls) - if trade in self.trades: - self._reduce_trade(trade, price, size, time_index) - assert order.size != -_prev_size or trade not in self.trades - if price == stop_price: - # Set SL back on the order for stats._trades["SL"] - trade._sl_order._replace(stop_price=stop_price) - if order in (trade._sl_order, - trade._tp_order): - assert order.size == -trade.size - assert order not in self.orders # Removed when trade was closed - else: - # It's a trade.close() order, now done - assert abs(_prev_size) >= abs(size) >= 1 - self.orders.remove(order) + # > When the stop price is reached, a stop order becomes a market/limit order. + # https://www.sec.gov/fast-answers/answersstopordhtm.html + order._replace(stop_price=None) + + # Determine purchase price. + # Check if limit order can be filled. + if order.limit: + is_limit_hit = low <= order.limit if order.is_long else high >= order.limit + # When stop and limit are hit within the same bar, we pessimistically + # assume limit was hit before the stop (i.e. "before it counts") + is_limit_hit_before_stop = (is_limit_hit and + (order.limit <= (stop_price or -np.inf) + if order.is_long + else order.limit >= (stop_price or np.inf))) + if not is_limit_hit or is_limit_hit_before_stop: continue - # Else this is a stand-alone trade - - # Adjust price to include commission (or bid-ask spread). - # In long positions, the adjusted price is a fraction higher, and vice versa. - adjusted_price = self._adjusted_price(order.size, price) - adjusted_price_plus_commission = \ - adjusted_price + self._commission(order.size, price) / abs(order.size) - - # If order size was specified proportionally, - # precompute true size in units, accounting for margin and spread/commissions - size = order.size - if -1 < size < 1: - size = copysign(int((self.margin_available * self._leverage * abs(size)) - // adjusted_price_plus_commission), size) - # Not enough cash/margin even for a single unit - if not size: - warnings.warn( - f'({data.index[self._i]}) broker canceled the relative-sized order ' - f'{order} due to insufficient margin ' - f'(equity={self.equity:.2f}, margin_available={self.margin_available:.2f}).', - category=UserWarning) - # XXX: The order is canceled by the broker? - self.orders.remove(order) - continue - assert size == round(size) - need_size = int(size) - - if not self._hedging: - # Fill position by FIFO closing/reducing existing opposite-facing trades. - # Existing trades are closed at unadjusted price, because the adjustment - # was already made when buying. - for trade in list(self.trades): - if trade.is_long == order.is_long: - continue - assert trade.size * order.size < 0 - - # Order size greater than this opposite-directed existing trade, - # so it will be closed completely - if abs(need_size) >= abs(trade.size): - self._close_trade(trade, price, time_index) - need_size += trade.size - else: - # The existing trade is larger than the new order, - # so it will only be closed partially - self._reduce_trade(trade, price, need_size, time_index) - need_size = 0 - - if not need_size: - break - - # If we don't have enough liquidity to cover for the order, the broker CANCELS it - if abs(need_size) * adjusted_price_plus_commission > \ - self.margin_available * self._leverage: + # stop_price, if set, was hit within this bar + price = (min(stop_price or open, order.limit) + if order.is_long else + max(stop_price or open, order.limit)) + else: + # Market-if-touched / market order + # Contingent orders always on next open + prev_close = data.Close[-2] + price = prev_close if self._trade_on_close and not order.is_contingent else open + if stop_price: + price = max(price, stop_price) if order.is_long else min(price, stop_price) + + # Determine entry/exit bar index + is_market_order = not order.limit and not stop_price + time_index = ( + (self._i - 1) + if is_market_order and self._trade_on_close and not order.is_contingent else + self._i) + + # If order is a SL/TP order, it should close an existing trade it was contingent upon + if order.parent_trade: + trade = order.parent_trade + _prev_size = trade.size + # If order.size is "greater" than trade.size, this order is a trade.close() + # order and part of the trade was already closed beforehand + size = copysign(min(abs(_prev_size), abs(order.size)), order.size) + # If this trade isn't already closed (e.g. on multiple `trade.close(.5)` calls) + if trade in self.trades: + self._reduce_trade(trade, price, size, time_index) + assert order.size != -_prev_size or trade not in self.trades + if price == stop_price: + # Set SL back on the order for stats._trades["SL"] + trade._sl_order._replace(stop_price=stop_price) + if order in (trade._sl_order, + trade._tp_order): + assert order.size == -trade.size + assert order not in self.orders # Removed when trade was closed + else: + # It's a trade.close() order, now done + assert abs(_prev_size) >= abs(size) >= 1 + self.orders.remove(order) + continue + + # Else this is a stand-alone trade + + # Adjust price to include commission (or bid-ask spread). + # In long positions, the adjusted price is a fraction higher, and vice versa. + adjusted_price = self._adjusted_price(order.size, price) + adjusted_price_plus_commission = \ + adjusted_price + self._commission(order.size, price) / abs(order.size) + + # If order size was specified proportionally, + # precompute true size in units, accounting for margin and spread/commissions + size = order.size + if -1 < size < 1: + size = copysign(int((self.margin_available * self._leverage * abs(size)) + // adjusted_price_plus_commission), size) + # Not enough cash/margin even for a single unit + if not size: warnings.warn( - f'({data.index[self._i]}) broker canceled order {order} due to insufficient margin ' + f'({data.index[self._i]}) broker canceled the relative-sized order ' + f'{order} due to insufficient margin ' f'(equity={self.equity:.2f}, margin_available={self.margin_available:.2f}).', category=UserWarning) + # XXX: The order is canceled by the broker? self.orders.remove(order) continue + assert size == round(size) + need_size = int(size) + + if not self._hedging: + # Fill position by FIFO closing/reducing existing opposite-facing trades. + # Existing trades are closed at unadjusted price, because the adjustment + # was already made when buying. + for trade in list(self.trades): + if trade.is_long == order.is_long: + continue + assert trade.size * order.size < 0 - # Open a new trade - if need_size: - self._open_trade(adjusted_price, - need_size, - order.sl, - order.tp, - time_index, - order.tag) - - # We need to reprocess the SL/TP orders newly added to the queue. - # This allows e.g. SL hitting in the same bar the order was open. - # See https://github.com/kernc/backtesting.py/issues/119 - if order.sl or order.tp: - if is_market_order: - reprocess_orders = True - # Order.stop and TP hit within the same bar, but SL wasn't. This case - # is not ambiguous, because stop and TP go in the same price direction. - elif stop_price and not order.limit and order.tp and ( - (order.is_long and order.tp <= high and (order.sl or -np.inf) < low) or - (order.is_short and order.tp >= low and (order.sl or np.inf) > high)): - reprocess_orders = True - elif (low <= (order.sl or -np.inf) <= high or - low <= (order.tp or -np.inf) <= high): - warnings.warn( - f"({data.index[-1]}) A contingent SL/TP order would execute in the " - "same bar its parent stop/limit order was turned into a trade. " - "Since we can't assert the precise intra-candle " - "price movement, the affected SL/TP order will instead be executed on " - "the next (matching) price/bar, making the result (of this trade) " - "somewhat dubious. " - "See https://github.com/kernc/backtesting.py/issues/119", - UserWarning) - - # Order processed + # Order size greater than this opposite-directed existing trade, + # so it will be closed completely + if abs(need_size) >= abs(trade.size): + self._close_trade(trade, price, time_index) + need_size += trade.size + else: + # The existing trade is larger than the new order, + # so it will only be closed partially + self._reduce_trade(trade, price, need_size, time_index) + need_size = 0 + + if not need_size: + break + + # If we don't have enough liquidity to cover for the order, the broker CANCELS it + if abs(need_size) * adjusted_price_plus_commission > \ + self.margin_available * self._leverage: + warnings.warn( + f'({data.index[self._i]}) broker canceled order {order} due to insufficient margin ' + f'(equity={self.equity:.2f}, margin_available={self.margin_available:.2f}).', + category=UserWarning) self.orders.remove(order) + continue + + # Open a new trade + if need_size: + self._open_trade(adjusted_price, + need_size, + order.sl, + order.tp, + time_index, + order.tag) + + # We need to reprocess the SL/TP orders newly added to the queue. + # This allows e.g. SL hitting in the same bar the order was open. + # See https://github.com/kernc/backtesting.py/issues/119 + if order.sl or order.tp: + if is_market_order: + reprocess_orders = True + # Order.stop and TP hit within the same bar, but SL wasn't. This case + # is not ambiguous, because stop and TP go in the same price direction. + elif stop_price and not order.limit and order.tp and ( + (order.is_long and order.tp <= high and (order.sl or -np.inf) < low) or + (order.is_short and order.tp >= low and (order.sl or np.inf) > high)): + reprocess_orders = True + elif (low <= (order.sl or -np.inf) <= high or + low <= (order.tp or -np.inf) <= high): + warnings.warn( + f"({data.index[-1]}) A contingent SL/TP order would execute in the " + "same bar its parent stop/limit order was turned into a trade. " + "Since we can't assert the precise intra-candle " + "price movement, the affected SL/TP order will instead be executed on " + "the next (matching) price/bar, making the result (of this trade) " + "somewhat dubious. " + "See https://github.com/kernc/backtesting.py/issues/119", + UserWarning) - if not reprocess_orders: - break - # end while + # Order processed + self.orders.remove(order) + + if reprocess_orders: + self._process_orders() def _reduce_trade(self, trade: Trade, price: float, size: float, time_index: int): assert trade.size * size < 0 @@ -1128,9 +1096,7 @@ def _open_trade(self, price: float, size: int, trade = Trade(self, size, price, time_index, tag) self.trades.append(trade) # Apply broker commission at trade open - open_commission = self._commission(size, price) - trade._open_commission = open_commission - self._cash -= open_commission + self._cash -= self._commission(size, price) # Create SL/TP (bracket) orders. if tp: trade.tp = tp @@ -1234,8 +1200,6 @@ def __init__(self, hedging=False, exclusive_orders=False, finalize_trades=False, - risk_free_rate: float = 0.0, - commission_when: str = 'exit', ): if not (isinstance(strategy, type) and issubclass(strategy, Strategy)): raise TypeError('`strategy` must be a Strategy sub-type') @@ -1259,15 +1223,7 @@ def __init__(self, (data.index.is_numeric() and (data.index > pd.Timestamp('1975').timestamp()).mean() > .8)): try: - # try seconds vs milliseconds heuristic first - idx = np.asarray(data.index) - med = np.median(idx.astype('float64')) - if med > 1e12: - data.index = pd.to_datetime(data.index, unit='ms') - elif med > 1e9: - data.index = pd.to_datetime(data.index, unit='s') - else: - data.index = pd.to_datetime(data.index, infer_datetime_format=True) + data.index = pd.to_datetime(data.index, infer_datetime_format=True) except ValueError: pass @@ -1283,13 +1239,6 @@ def __init__(self, raise ValueError('Some OHLC values are missing (NaN). ' 'Please strip those lines with `df.dropna()` or ' 'fill them in with `df.interpolate()` or whatever.') - # Optional sanity warnings for inconsistent OHLC (non-breaking) - try: - if not ((data['High'] >= data[['Open', 'Close']].max(axis=1)) & - (data['Low'] <= data[['Open', 'Close']].min(axis=1))).all(): - warnings.warn('Some OHLC rows have inconsistent High/Low vs Open/Close.', stacklevel=2) - except Exception: - pass if np.any(data['Close'] > cash): warnings.warn('Some prices are larger than initial cash value. Note that fractional ' 'trading is not supported by this class. If you want to trade Bitcoin, ' @@ -1306,8 +1255,6 @@ def __init__(self, stacklevel=2) self._data: pd.DataFrame = data - self._risk_free_rate = float(risk_free_rate) - self._commission_when = commission_when if commission_when in ('exit', 'both') else 'exit' self._broker = partial( _Broker, cash=cash, spread=spread, commission=commission, margin=margin, trade_on_close=trade_on_close, hedging=hedging, @@ -1369,8 +1316,6 @@ def run(self, **kwargs) -> pd.Series: """ data = _Data(self._data.copy(deep=False)) broker: _Broker = self._broker(data=data) - # propagate commission PL behavior - broker._commission_when = self._commission_when strategy: Strategy = self._strategy(broker, data, kwargs) strategy.init() @@ -1423,20 +1368,12 @@ def run(self, **kwargs) -> pd.Series: # for future `indicator._opts['data'].index` calls to work data._set_length(len(self._data)) - # backfill equity with NumPy to avoid pandas overhead - equity = broker._equity.copy() - if np.isnan(equity).any(): - idx = np.arange(len(equity)) - valid = ~np.isnan(equity) - if valid.any(): - last = np.maximum.accumulate(np.where(valid, idx, 0)) - equity = equity[last] - equity[np.isnan(equity)] = broker._cash + equity = pd.Series(broker._equity).bfill().fillna(broker._cash).values self._results = compute_stats( trades=broker.closed_trades, equity=equity, ohlc_data=self._data, - risk_free_rate=self._risk_free_rate, + risk_free_rate=0.0, strategy_instance=strategy, )