Merge pull request #153 from mhallsmoore/iqfeed_price_handler

Added a DTN IQFeed Price Handler. Also modified drawdown and drawdown…

examples/mac_backtest.py

@@ -76,5 +76,6 @@ def main(config, testing, tickers, filename):
     config = settings.from_file(config, testing)
     run(config, testing, tickers, filename)
 
+
 if __name__ == "__main__":
     main()

examples/mac_backtest_tearsheet.py

@@ -84,5 +84,6 @@ def main(config, testing, tickers, filename):
     config = settings.from_file(config, testing)
     run(config, testing, tickers, filename)
 
+
 if __name__ == "__main__":
     main()

qstrader/price_handler/iq_feed_intraday_csv_bar.py

@@ -0,0 +1,148 @@
+import os
+
+import pandas as pd
+
+from ..price_parser import PriceParser
+from .base import AbstractBarPriceHandler
+from ..event import BarEvent
+
+
+class IQFeedIntradayCsvBarPriceHandler(AbstractBarPriceHandler):
+    """
+    IQFeedIntradayCsvBarPriceHandler is designed to read
+    intraday bar CSV files downloaded from DTN IQFeed, consisting
+    of Open-Low-High-Close-Volume-OpenInterest (OHLCVI) data
+    for each requested financial instrument and stream those to
+    the provided events queue as BarEvents.
+    """
+    def __init__(
+        self, csv_dir, events_queue,
+        init_tickers=None,
+        start_date=None, end_date=None
+    ):
+        """
+        Takes the CSV directory, the events queue and a possible
+        list of initial ticker symbols then creates an (optional)
+        list of ticker subscriptions and associated prices.
+        """
+        self.csv_dir = csv_dir
+        self.events_queue = events_queue
+        self.continue_backtest = True
+        self.tickers = {}
+        self.tickers_data = {}
+        if init_tickers is not None:
+            for ticker in init_tickers:
+                self.subscribe_ticker(ticker)
+        self.start_date = start_date
+        self.end_date = end_date
+        self.bar_stream = self._merge_sort_ticker_data()
+
+    def _open_ticker_price_csv(self, ticker):
+        """
+        Opens the CSV files containing the equities ticks from
+        the specified CSV data directory, converting them into
+        them into a pandas DataFrame, stored in a dictionary.
+        """
+        ticker_path = os.path.join(self.csv_dir, "%s.csv" % ticker)
+
+        self.tickers_data[ticker] = pd.read_csv(
+            ticker_path,
+            names=[
+                "Date", "Open", "Low", "High",
+                "Close", "Volume", "OpenInterest"
+            ],
+            index_col="Date", parse_dates=True
+        )
+        self.tickers_data[ticker]["Ticker"] = ticker
+
+    def _merge_sort_ticker_data(self):
+        """
+        Concatenates all of the separate equities DataFrames
+        into a single DataFrame that is time ordered, allowing tick
+        data events to be added to the queue in a chronological fashion.
+
+        Note that this is an idealised situation, utilised solely for
+        backtesting. In live trading ticks may arrive "out of order".
+        """
+        df = pd.concat(self.tickers_data.values()).sort_index()
+        start = None
+        end = None
+        if self.start_date is not None:
+            start = df.index.searchsorted(self.start_date)
+        if self.end_date is not None:
+            end = df.index.searchsorted(self.end_date)
+        # Determine how to slice
+        if start is None and end is None:
+            return df.iterrows()
+        elif start is not None and end is None:
+            return df.ix[start:].iterrows()
+        elif start is None and end is not None:
+            return df.ix[:end].iterrows()
+        else:
+            return df.ix[start:end].iterrows()
+
+    def subscribe_ticker(self, ticker):
+        """
+        Subscribes the price handler to a new ticker symbol.
+        """
+        if ticker not in self.tickers:
+            try:
+                self._open_ticker_price_csv(ticker)
+                dft = self.tickers_data[ticker]
+                row0 = dft.iloc[0]
+
+                close = PriceParser.parse(row0["Close"])
+
+                ticker_prices = {
+                    "close": close,
+                    "adj_close": close,
+                    "timestamp": dft.index[0]
+                }
+                self.tickers[ticker] = ticker_prices
+            except OSError:
+                print(
+                    "Could not subscribe ticker %s "
+                    "as no data CSV found for pricing." % ticker
+                )
+        else:
+            print(
+                "Could not subscribe ticker %s "
+                "as is already subscribed." % ticker
+            )
+
+    def _create_event(self, index, period, ticker, row):
+        """
+        Obtain all elements of the bar from a row of dataframe
+        and return a BarEvent
+        """
+        open_price = PriceParser.parse(row["Open"])
+        low_price = PriceParser.parse(row["Low"])
+        high_price = PriceParser.parse(row["High"])
+        close_price = PriceParser.parse(row["Close"])
+        adj_close_price = PriceParser.parse(row["Close"])
+        volume = int(row["Volume"])
+        bev = BarEvent(
+            ticker, index, period, open_price,
+            high_price, low_price, close_price,
+            volume, adj_close_price
+        )
+        return bev
+
+    def stream_next(self):
+        """
+        Place the next BarEvent onto the event queue.
+        """
+        try:
+            index, row = next(self.bar_stream)
+        except StopIteration:
+            self.continue_backtest = False
+            return
+        # Obtain all elements of the bar from the dataframe
+        ticker = row["Ticker"]
+        period = 60  # Seconds in a minute
+        # Create the tick event for the queue
+        bev = self._create_event(index, period, ticker, row)
+        # Store event
+        self._store_event(bev)
+        # Send event to queue
+        self.events_queue.put(bev)

qstrader/scripts/generate_simulated_prices.py

@@ -99,5 +99,6 @@ def run(outdir, ticker, init_price, seed, s0, spread, mu_dt, sigma_dt, year, mon
 def main(outdir, ticker, init_price, seed, s0, spread, mu_dt, sigma_dt, year, month, days, config=None):
     return run(outdir, ticker, init_price, seed, s0, spread, mu_dt, sigma_dt, year, month, days, config=config)
 
+
 if __name__ == "__main__":
     main()

qstrader/statistics/performance.py

@@ -1,3 +1,5 @@
+from itertools import groupby
+
 import numpy as np
 import pandas as pd
 from scipy.stats import linregress
@@ -76,27 +78,30 @@ def create_drawdowns(returns):
     Returns:
     drawdown, drawdown_max, duration
     """
-
     # Calculate the cumulative returns curve
     # and set up the High Water Mark
-    hwm = [0]
-
-    # Create the drawdown and duration series
     idx = returns.index
-    drawdown = pd.Series(index=idx)
-    duration = pd.Series(index=idx)
+    hwm = np.zeros(len(idx))
 
-    # Loop over the index range
+    # Create the high water mark
     for t in range(1, len(idx)):
-        hwm.append(max(hwm[t - 1], returns.ix[t]))
-        drawdown.ix[t] = (hwm[t] - returns.ix[t]) / hwm[t]
-        duration.ix[t] = (0 if drawdown.ix[t] == 0 else duration.ix[t - 1] + 1)
+        hwm[t] = max(hwm[t - 1], returns.ix[t])
 
-    return drawdown, drawdown.max(), duration.max()
+    # Calculate the drawdown and duration statistics
+    perf = pd.DataFrame(index=idx)
+    perf["Drawdown"] = (hwm - returns) / hwm
+    perf["Drawdown"].ix[0] = 0.0
+    perf["DurationCheck"] = np.where(perf["Drawdown"] == 0, 0, 1)
+    duration = max(
+        sum(1 for i in g if i == 1)
+        for k, g in groupby(perf["DurationCheck"])
+    )
+    return perf["Drawdown"], np.max(perf["Drawdown"]), duration
 
 
 def rsquared(x, y):
-    """ Return R^2 where x and y are array-like."""
-
+    """
+    Return R^2 where x and y are array-like.
+    """
     slope, intercept, r_value, p_value, std_err = linregress(x, y)
     return r_value**2

qstrader/statistics/tearsheet.py

@@ -19,7 +19,10 @@
 class TearsheetStatistics(AbstractStatistics):
     """
     """
-    def __init__(self, config, portfolio_handler, title=None, benchmark=None):
+    def __init__(
+        self, config, portfolio_handler,
+        title=None, benchmark=None, periods=252
+    ):
         """
         Takes in a portfolio handler.
         """
@@ -28,6 +31,7 @@ def __init__(self, config, portfolio_handler, title=None, benchmark=None):
         self.price_handler = portfolio_handler.price_handler
         self.title = '\n'.join(title)
         self.benchmark = benchmark
+        self.periods = periods
         self.equity = {}
         self.equity_benchmark = {}
         self.log_scale = False
@@ -64,7 +68,9 @@ def get_results(self):
         statistics = {}
 
         # Equity statistics
-        statistics["sharpe"] = perf.create_sharpe_ratio(returns_s)
+        statistics["sharpe"] = perf.create_sharpe_ratio(
+            returns_s, self.periods
+        )
         statistics["drawdowns"] = dd_s
         # TODO: need to have max_drawdown so it can be printed at end of test
         statistics["max_drawdown"] = max_dd
@@ -274,9 +280,9 @@ def format_perc(x, pos):
         ax.yaxis.set_major_formatter(FuncFormatter(y_axis_formatter))
 
         tot_ret = cum_returns[-1] - 1.0
-        cagr = perf.create_cagr(cum_returns)
-        sharpe = perf.create_sharpe_ratio(returns)
-        sortino = perf.create_sortino_ratio(returns)
+        cagr = perf.create_cagr(cum_returns, self.periods)
+        sharpe = perf.create_sharpe_ratio(returns, self.periods)
+        sortino = perf.create_sortino_ratio(returns, self.periods)
         rsq = perf.rsquared(range(cum_returns.shape[0]), cum_returns)
         dd, dd_max, dd_dur = perf.create_drawdowns(cum_returns)
         trd_yr = positions.shape[0] / ((returns.index[-1] - returns.index[0]).days / 365.0)

tests/test_priceparser.py

@@ -58,5 +58,6 @@ def test_unparsed_display(self):
         displayed = PriceParser.display(self.float)
         self.assertEqual(displayed, 10.12)
 
+
 if __name__ == "__main__":
     unittest.main()

tests/test_statistics.py

@@ -161,5 +161,6 @@ def test_calculating_statistics(self):
         self.assertEqual(results["max_drawdown_pct"], 0.1908)
         self.assertAlmostEqual(float(results["sharpe"]), 1.7575)
 
+
 if __name__ == "__main__":
     unittest.main()