Merge pull request #117 from AsyncAlgoTrading/coinbase

add initial periodic support in backtest
AsyncAlgoTrading · Nov 2, 2020 · 9377c38 · 9377c38
2 parents ad80e91 + 01812ae
commit 9377c38
Show file tree

Hide file tree

Showing 7 changed files with 168 additions and 70 deletions.
diff --git a/Makefile b/Makefile
@@ -76,6 +76,7 @@ testjs:  ## Make js tests
 
 testruns:  ## Run a few examples as a live end-to-end test
 	$(PYTHON) -m aat.strategy.sample.readonly
+	$(PYTHON) -m aat.strategy.sample.readonly_periodic
 
 lint: lintpy lintjs lintcpp  ## run all linters
 

diff --git a/aat/engine/dispatch/manager.py b/aat/engine/dispatch/manager.py
@@ -4,6 +4,7 @@
 from typing import List, TYPE_CHECKING
 
 from .order_entry import StrategyManagerOrderEntryMixin
+from .periodic import PeriodicManagerMixin
 from .portfolio import StrategyManagerPortfolioMixin
 from .risk import StrategyManagerRiskMixin
 from .utils import StrategyManagerUtilsMixin
@@ -16,7 +17,7 @@
     from aat.engine import TradingEngine
 
 
-class StrategyManager(StrategyManagerOrderEntryMixin, StrategyManagerRiskMixin, StrategyManagerPortfolioMixin, StrategyManagerUtilsMixin, EventHandler):
+class StrategyManager(StrategyManagerOrderEntryMixin, StrategyManagerRiskMixin, StrategyManagerPortfolioMixin, StrategyManagerUtilsMixin, PeriodicManagerMixin, EventHandler):
     def __init__(self,
                  trading_engine: 'TradingEngine',
                  trading_type: TradingType,

diff --git a/aat/engine/dispatch/periodic.py b/aat/engine/dispatch/periodic.py
@@ -1,4 +1,4 @@
-from typing import Callable, Awaitable
+from typing import Callable, Awaitable, List
 from temporalcache.utils import should_expire  # type: ignore
 
 
@@ -16,7 +16,33 @@ def __init__(self, loop, last_ts, function, second, minute, hour):
     def stop(self) -> None:
         self._continue = False
 
+    def expires(self, timestamp):
+        return should_expire(self._last, timestamp, self._second, self._minute, self._hour)
+
     async def execute(self, timestamp):
-        if should_expire(self._last, timestamp, self._second, self._minute, self._hour):
+        if self.expires(timestamp):
             await self._function()
             self._last = timestamp
+
+
+class PeriodicManagerMixin(object):
+    _periodics: List[Periodic] = []
+
+    def periodics(self):
+        return self._periodics
+
+    def periodicIntervals(self) -> int:
+        '''return the interval required for periodics, to optimize call times
+        1 - secondly
+        60 - minutely
+        3600 - hourly
+        '''
+        ret = 3600
+        for p in self._periodics:
+            if p._second == '*':
+                # if any secondly, return 0 right away
+                return 1
+            elif p._minute == '*':
+                # if any require minutely, drop to 1
+                ret = 60
+        return ret
diff --git a/aat/engine/engine.py b/aat/engine/engine.py
@@ -6,7 +6,7 @@
 from aiostream.stream import merge  # type: ignore
 from collections import deque
 from concurrent.futures import ThreadPoolExecutor
-from datetime import datetime
+from datetime import datetime, timedelta
 from traitlets.config.application import Application  # type: ignore
 from traitlets import validate, TraitError, Unicode, Bool, List, Instance  # type: ignore
 from tornado.web import StaticFileHandler, RedirectHandler, Application as TornadoApplication
@@ -236,6 +236,24 @@ async def run(self):
         async with merge(*(exch.tick() for exch in self.exchanges + [self] if inspect.isasyncgenfunction(exch.tick))).stream() as stream:
             # stream through all events
             async for event in stream:
+                # TODO move out of critical path
+                if self._offline():
+                    # inject periodics
+                    # TODO optimize
+                    # Manager should keep track of the intervals for its periodics,
+                    # then we don't need to go through seconds (which is what the
+                    # live engine's `tick` method does below). Instead we can just
+                    # calculate exactly the intervals
+                    if self._latest != datetime.fromtimestamp(0) and hasattr(event, 'target') and hasattr(event.target, 'timestamp'):
+                        # TODO in progress optimization
+                        intervals = self.manager.periodicIntervals()
+
+                        # not the first tick
+                        for _ in range(int((event.target.timestamp - self._latest).total_seconds() / intervals)):
+                            self._latest = self._latest + timedelta(seconds=1 * intervals)
+                            if any(p.expires(self._latest) for p in self.manager.periodics()):
+                                await asyncio.gather(*(asyncio.create_task(p.execute(self._latest)) for p in self.manager.periodics() if p.expires(self._latest)))
+
                 # tick exchange event to handlers
                 await self.processEvent(event)
 
@@ -262,7 +280,7 @@ async def run(self):
                     await self.processEvent(event, strat)
 
                 # process any periodics
-                await asyncio.gather(*(asyncio.create_task(p.execute(self._latest)) for p in self.manager._periodics))
+                await asyncio.gather(*(asyncio.create_task(p.execute(self._latest)) for p in self.manager.periodics() if p.expires(self._latest)))
 
         # Before engine shutdown, send an exit event
         await self.processEvent(Event(type=EventType.EXIT, target=None))
@@ -304,8 +322,8 @@ async def tick(self):
         '''helper method to ensure periodic methods execute periodically in absence
         of market data'''
 
-        # TODO periodic strategies in backtest/simulation
         if self._offline():
+            # periodics injected manually, see main event loop above
             return
 
         while True:
@@ -326,5 +344,6 @@ def start(self):
             self.event_loop.run_until_complete(self.run())
         except KeyboardInterrupt:
             pass
+
         # send exit event to all callbacks
         asyncio.ensure_future(self.processEvent(Event(type=EventType.EXIT, target=None)))
diff --git a/aat/exchange/generic/csv.py b/aat/exchange/generic/csv.py
@@ -1,5 +1,6 @@
 import csv
 from collections import deque
+from datetime import datetime
 from typing import List
 from aat.config import EventType, InstrumentType, Side, TradingType
 from aat.core import ExchangeType, Event, Instrument, Trade, Order
@@ -37,6 +38,12 @@ async def connect(self):
                 )
                 )
                 order.filled = float(row['volume'])
+                if 'time' in row:
+                    order.timestamp = datetime.fromtimestamp(float(row['time']))
+                elif 'date' in row:
+                    order.timestamp = datetime.fromisoformat(row['date'])
+                elif 'datetime' in row:
+                    order.timestamp = datetime.fromisoformat(row['datetime'])
 
                 self._data.append(Trade(volume=float(row['volume']),
                                         price=float(row['close']),

diff --git a/aat/strategy/sample/data/aapl.csv b/aat/strategy/sample/data/aapl.csv
@@ -1,65 +1,65 @@
 date,symbol,open,close,high,low,volume
-6/11/20,AAPL-EQUITY,87.33,83.98,87.77,83.87,201662452
-6/12/20,AAPL-EQUITY,86.18,84.7,86.95,83.56,200146052
-6/15/20,AAPL-EQUITY,83.31,85.75,86.42,83.15,138808920
-6/16/20,AAPL-EQUITY,87.87,88.02,88.3,86.18,165428728
-6/17/20,AAPL-EQUITY,88.79,87.9,88.85,87.77,114406504
-6/18/20,AAPL-EQUITY,87.85,87.93,88.36,87.31,96820384
-6/19/20,AAPL-EQUITY,88.66,87.43,89.14,86.29,264475808
-6/22/20,AAPL-EQUITY,87.84,89.72,89.87,87.79,135445264
-6/23/20,AAPL-EQUITY,91,91.63,93.1,90.57,212155476
-6/24/20,AAPL-EQUITY,91.25,90.02,92.2,89.63,192623396
-6/25/20,AAPL-EQUITY,90.18,91.21,91.25,89.39,137522512
-6/26/20,AAPL-EQUITY,91.1,88.41,91.33,88.26,205256844
-6/29/20,AAPL-EQUITY,88.31,90.45,90.54,87.82,130646076
-6/30/20,AAPL-EQUITY,90.02,91.2,91.5,90,140223284
-7/1/20,AAPL-EQUITY,91.28,91.03,91.84,90.98,110737236
-7/2/20,AAPL-EQUITY,91.96,91.03,92.62,90.91,114041468
-7/6/20,AAPL-EQUITY,92.5,93.46,93.95,92.47,118655652
-7/7/20,AAPL-EQUITY,93.85,93.17,94.66,93.06,112424456
-7/8/20,AAPL-EQUITY,94.18,95.34,95.38,94.09,117091880
-7/9/20,AAPL-EQUITY,96.26,95.68,96.32,94.67,125642608
-7/10/20,AAPL-EQUITY,95.34,95.92,95.98,94.71,90257320
-7/13/20,AAPL-EQUITY,97.27,95.48,99.96,95.26,191649140
-7/14/20,AAPL-EQUITY,94.84,97.06,97.26,93.88,170989364
-7/15/20,AAPL-EQUITY,98.99,97.73,99.25,96.49,153197932
-7/16/20,AAPL-EQUITY,96.56,96.52,97.41,95.91,110577672
-7/17/20,AAPL-EQUITY,96.99,96.33,97.15,95.84,92186900
-7/20/20,AAPL-EQUITY,96.42,98.36,98.5,96.06,90317908
-7/21/20,AAPL-EQUITY,99.17,97,99.25,96.74,103645836
-7/22/20,AAPL-EQUITY,96.69,97.27,97.98,96.6,89001652
-7/23/20,AAPL-EQUITY,97,92.85,97.08,92.01,197004432
-7/24/20,AAPL-EQUITY,90.99,92.62,92.97,89.15,185438864
-7/27/20,AAPL-EQUITY,93.71,94.81,94.91,93.48,121214192
-7/28/20,AAPL-EQUITY,94.37,93.25,94.55,93.25,103625500
-7/29/20,AAPL-EQUITY,93.75,95.04,95.23,93.71,90329256
-7/30/20,AAPL-EQUITY,94.19,96.19,96.3,93.77,158130020
-7/31/20,AAPL-EQUITY,102.88,106.26,106.42,100.83,374295468
-8/3/20,AAPL-EQUITY,108.2,108.94,111.64,107.89,308151388
-8/4/20,AAPL-EQUITY,109.13,109.67,110.79,108.39,172792368
-8/5/20,AAPL-EQUITY,109.38,110.06,110.39,108.9,121991952
-8/6/20,AAPL-EQUITY,110.41,113.9,114.41,109.8,202428900
-8/7/20,AAPL-EQUITY,113.21,111.11,113.68,110.29,198045612
-8/10/20,AAPL-EQUITY,112.6,112.73,113.78,110,212403424
-8/11/20,AAPL-EQUITY,111.97,109.38,112.48,109.11,187902376
-8/12/20,AAPL-EQUITY,110.5,113.01,113.28,110.3,165944820
-8/13/20,AAPL-EQUITY,114.43,115.01,116.04,113.93,210082064
-8/14/20,AAPL-EQUITY,114.83,114.91,115,113.05,165565208
-8/17/20,AAPL-EQUITY,116.06,114.61,116.09,113.96,119561444
-8/18/20,AAPL-EQUITY,114.35,115.56,116,114.01,105633540
-8/19/20,AAPL-EQUITY,115.98,115.71,117.16,115.61,145538008
-8/20/20,AAPL-EQUITY,115.75,118.28,118.39,115.73,126907188
-8/21/20,AAPL-EQUITY,119.26,124.37,124.87,119.25,338054640
-8/24/20,AAPL-EQUITY,128.7,125.86,128.79,123.94,345937768
-8/25/20,AAPL-EQUITY,124.7,124.83,125.18,123.05,211495788
-8/26/20,AAPL-EQUITY,126.18,126.52,126.99,125.08,163022268
-8/27/20,AAPL-EQUITY,127.14,125.01,127.49,123.83,155552384
-8/28/20,AAPL-EQUITY,126.01,124.81,126.44,124.58,187629916
-8/31/20,AAPL-EQUITY,127.58,129.04,131,126,225702688
-9/1/20,AAPL-EQUITY,132.76,134.18,134.8,130.53,152470142
-9/2/20,AAPL-EQUITY,137.59,131.4,137.98,127,200118991
-9/3/20,AAPL-EQUITY,126.91,120.88,128.84,120.5,257599640
-9/4/20,AAPL-EQUITY,120.07,120.96,123.7,110.89,332607163
-9/8/20,AAPL-EQUITY,113.95,112.82,118.99,112.68,231366563
-9/9/20,AAPL-EQUITY,117.26,117.32,119.14,115.26,176940455
-9/10/20,AAPL-EQUITY,120.36,113.49,120.5,112.5,182274391
+2020-06-11T16:00:00,AAPL-EQUITY,87.33,83.98,87.77,83.87,201662452
+2020-06-12T16:00:00,AAPL-EQUITY,86.18,84.7,86.95,83.56,200146052
+2020-06-15T16:00:00,AAPL-EQUITY,83.31,85.75,86.42,83.15,138808920
+2020-06-16T16:00:00,AAPL-EQUITY,87.87,88.02,88.3,86.18,165428728
+2020-06-17T16:00:00,AAPL-EQUITY,88.79,87.9,88.85,87.77,114406504
+2020-06-18T16:00:00,AAPL-EQUITY,87.85,87.93,88.36,87.31,96820384
+2020-06-19T16:00:00,AAPL-EQUITY,88.66,87.43,89.14,86.29,264475808
+2020-06-22T16:00:00,AAPL-EQUITY,87.84,89.72,89.87,87.79,135445264
+2020-06-23T16:00:00,AAPL-EQUITY,91,91.63,93.1,90.57,212155476
+2020-06-24T16:00:00,AAPL-EQUITY,91.25,90.02,92.2,89.63,192623396
+2020-06-25T16:00:00,AAPL-EQUITY,90.18,91.21,91.25,89.39,137522512
+2020-06-26T16:00:00,AAPL-EQUITY,91.1,88.41,91.33,88.26,205256844
+2020-06-29T16:00:00,AAPL-EQUITY,88.31,90.45,90.54,87.82,130646076
+2020-06-30T16:00:00,AAPL-EQUITY,90.02,91.2,91.5,90,140223284
+2020-07-01T16:00:00,AAPL-EQUITY,91.28,91.03,91.84,90.98,110737236
+2020-07-02T16:00:00,AAPL-EQUITY,91.96,91.03,92.62,90.91,114041468
+2020-07-06T16:00:00,AAPL-EQUITY,92.5,93.46,93.95,92.47,118655652
+2020-07-07T16:00:00,AAPL-EQUITY,93.85,93.17,94.66,93.06,112424456
+2020-07-08T16:00:00,AAPL-EQUITY,94.18,95.34,95.38,94.09,117091880
+2020-07-09T16:00:00,AAPL-EQUITY,96.26,95.68,96.32,94.67,125642608
+2020-07-10T16:00:00,AAPL-EQUITY,95.34,95.92,95.98,94.71,90257320
+2020-07-13T16:00:00,AAPL-EQUITY,97.27,95.48,99.96,95.26,191649140
+2020-07-14T16:00:00,AAPL-EQUITY,94.84,97.06,97.26,93.88,170989364
+2020-07-15T16:00:00,AAPL-EQUITY,98.99,97.73,99.25,96.49,153197932
+2020-07-16T16:00:00,AAPL-EQUITY,96.56,96.52,97.41,95.91,110577672
+2020-07-17T16:00:00,AAPL-EQUITY,96.99,96.33,97.15,95.84,92186900
+2020-07-20T16:00:00,AAPL-EQUITY,96.42,98.36,98.5,96.06,90317908
+2020-07-21T16:00:00,AAPL-EQUITY,99.17,97,99.25,96.74,103645836
+2020-07-22T16:00:00,AAPL-EQUITY,96.69,97.27,97.98,96.6,89001652
+2020-07-23T16:00:00,AAPL-EQUITY,97,92.85,97.08,92.01,197004432
+2020-07-24T16:00:00,AAPL-EQUITY,90.99,92.62,92.97,89.15,185438864
+2020-07-27T16:00:00,AAPL-EQUITY,93.71,94.81,94.91,93.48,121214192
+2020-07-28T16:00:00,AAPL-EQUITY,94.37,93.25,94.55,93.25,103625500
+2020-07-29T16:00:00,AAPL-EQUITY,93.75,95.04,95.23,93.71,90329256
+2020-07-30T16:00:00,AAPL-EQUITY,94.19,96.19,96.3,93.77,158130020
+2020-07-31T16:00:00,AAPL-EQUITY,102.88,106.26,106.42,100.83,374295468
+2020-08-03T16:00:00,AAPL-EQUITY,108.2,108.94,111.64,107.89,308151388
+2020-08-04T16:00:00,AAPL-EQUITY,109.13,109.67,110.79,108.39,172792368
+2020-08-05T16:00:00,AAPL-EQUITY,109.38,110.06,110.39,108.9,121991952
+2020-08-06T16:00:00,AAPL-EQUITY,110.41,113.9,114.41,109.8,202428900
+2020-08-07T16:00:00,AAPL-EQUITY,113.21,111.11,113.68,110.29,198045612
+2020-08-10T16:00:00,AAPL-EQUITY,112.6,112.73,113.78,110,212403424
+2020-08-11T16:00:00,AAPL-EQUITY,111.97,109.38,112.48,109.11,187902376
+2020-08-12T16:00:00,AAPL-EQUITY,110.5,113.01,113.28,110.3,165944820
+2020-08-13T16:00:00,AAPL-EQUITY,114.43,115.01,116.04,113.93,210082064
+2020-08-14T16:00:00,AAPL-EQUITY,114.83,114.91,115,113.05,165565208
+2020-08-17T16:00:00,AAPL-EQUITY,116.06,114.61,116.09,113.96,119561444
+2020-08-18T16:00:00,AAPL-EQUITY,114.35,115.56,116,114.01,105633540
+2020-08-19T16:00:00,AAPL-EQUITY,115.98,115.71,117.16,115.61,145538008
+2020-08-20T16:00:00,AAPL-EQUITY,115.75,118.28,118.39,115.73,126907188
+2020-08-21T16:00:00,AAPL-EQUITY,119.26,124.37,124.87,119.25,338054640
+2020-08-24T16:00:00,AAPL-EQUITY,128.7,125.86,128.79,123.94,345937768
+2020-08-25T16:00:00,AAPL-EQUITY,124.7,124.83,125.18,123.05,211495788
+2020-08-26T16:00:00,AAPL-EQUITY,126.18,126.52,126.99,125.08,163022268
+2020-08-27T16:00:00,AAPL-EQUITY,127.14,125.01,127.49,123.83,155552384
+2020-08-28T16:00:00,AAPL-EQUITY,126.01,124.81,126.44,124.58,187629916
+2020-08-31T16:00:00,AAPL-EQUITY,127.58,129.04,131,126,225702688
+2020-09-01T16:00:00,AAPL-EQUITY,132.76,134.18,134.8,130.53,152470142
+2020-09-02T16:00:00,AAPL-EQUITY,137.59,131.4,137.98,127,200118991
+2020-09-03T16:00:00,AAPL-EQUITY,126.91,120.88,128.84,120.5,257599640
+2020-09-04T16:00:00,AAPL-EQUITY,120.07,120.96,123.7,110.89,332607163
+2020-09-08T16:00:00,AAPL-EQUITY,113.95,112.82,118.99,112.68,231366563
+2020-09-09T16:00:00,AAPL-EQUITY,117.26,117.32,119.14,115.26,176940455
+2020-09-10T16:00:00,AAPL-EQUITY,120.36,113.49,120.5,112.5,182274391
diff --git a/aat/strategy/sample/readonly_periodic.py b/aat/strategy/sample/readonly_periodic.py
@@ -0,0 +1,44 @@
+import os
+import os.path
+from aat import Strategy, Event
+from pprint import pprint
+
+
+class ReadOnlyStrategy(Strategy):
+    def __init__(self, *args, **kwargs) -> None:
+        super(ReadOnlyStrategy, self).__init__(*args, **kwargs)
+        self.count = 0
+
+    async def onStart(self, event: Event) -> None:
+        pprint(self.instruments())
+        pprint(self.positions())
+
+        for i in self.instruments():
+            await self.subscribe(i)
+        self.periodic(self.onPeriodic, second=0, minute=0, hour=1)
+
+    async def onTrade(self, event: Event) -> None:
+        pprint(event)
+
+    async def onOrder(self, event):
+        pprint(event)
+
+    async def onExit(self, event: Event) -> None:
+        print('Finishing...')
+
+    async def onPeriodic(self):
+        print('here: {}'.format(self.count))
+        self.count += 1
+
+
+if __name__ == "__main__":
+    from aat import TradingEngine, parseConfig
+    cfg = parseConfig(['--trading_type', 'backtest',
+                       '--load_accounts',
+                       '--exchanges', 'aat.exchange.generic:CSV,{}'.format(os.path.join(os.path.dirname(__file__), 'data', 'aapl.csv')),
+                       '--strategies', 'aat.strategy.sample.readonly_periodic:ReadOnlyStrategy'
+                       ])
+    print(cfg)
+    t = TradingEngine(**cfg)
+    t.start()
+    assert t.strategies[0].count == 92