[Framework] Added historical trade feature

ctc_executioner/action.py

@@ -174,7 +174,8 @@ def run(self, orderbook):
         """
         matchEngine = self.getMatchEngine(orderbook)
         counterTrades, qtyRemain, index = matchEngine.matchOrder(self.getOrder(), self.getRuntime())
-        self.setTrades(self.getTrades() + counterTrades)
+        self.setTrades(self.getTrades() + counterTrades) # appends trades!
+        #self.setTrades(counterTrades) # only current trades!
         self.setOrderbookIndex(index=index)
         self.setOrderbookState(orderbook.getState(index))
         return self, counterTrades

ctc_executioner/action_space.py

@@ -145,7 +145,7 @@ def update(self, t, i, force_execution=False):
         a = self.ai.chooseAction(aiState)
         # print('Random action: ' + str(level) + ' for state: ' + str(aiState))
         action = self.createAction(level=a, state=aiState, force_execution=force_execution)
-        action.run(self.orderbook)
+        action, counterTrades = action.run(self.orderbook)
         i_next = self.determineNextInventory(action)
         t_next = self.determineNextTime(t)
         reward = action.getReward()
@@ -226,9 +226,9 @@ def backtest(self, q=None, episodes=10, average=False, fixed_a=None):
                     else:
                         try:
                             a_next = self.ai.getQAction(state_next, 0)
-                            print("t: " + str(t_next))
-                            print("i: " + str(i_next))
-                            print("Action: " + str(a_next))
+                            # print("t: " + str(t_next))
+                            # print("i: " + str(i_next))
+                            # print("Action: " + str(a_next))
                             # print("Q action for next state " + str(state_next) + ": " + str(a_next))
                         except:
                             # State might not be in Q-Table yet, more training requried.

ctc_executioner/action_space_env.py

@@ -63,7 +63,7 @@ def updateAction(self, action, level, state, orderbookIndex=None, force_executio
             action.setOrderbookState(orderbookState)
             action.setOrderbookIndex(orderbookIndex)
 
-        if runtime <= 0.0 or level is None:
+        if runtime <= 0 or level is None:
             price = None
             ot = OrderType.MARKET
         else:

ctc_executioner/action_state.py

@@ -1,4 +1,5 @@
 import numpy as np
+from ctc_executioner.feature_type import FeatureType
 
 class ActionState(object):
 
@@ -25,14 +26,24 @@ def __repr__(self):
         return self.__str__()
 
     def toArray(self):
-        # arr = [np.array([self.getT()]), np.array([self.getI()])]
-        # for k, v in self.getMarket().items():
-        #     arr.append(v)
-        # return np.array([arr])
-        features = self.market['bidask']
-        return features.reshape((1, features.shape[0], features.shape[1], features.shape[2])) # required for custom DQN
-        return features # (2*lookback, levels, count(features))
-
+        if FeatureType.ORDERS.value in self.market:
+            # arr = [np.array([self.getT()]), np.array([self.getI()])]
+            # for k, v in self.getMarket().items():
+            #     arr.append(v)
+            # return np.array([arr])
+            features = self.market[FeatureType.ORDERS.value]
+            arr = np.zeros(shape=(1,features.shape[1],2), dtype=float)
+            arr[0,0] = np.array([self.t, self.i])
+            features = np.vstack((arr, features))
+            #return features.reshape((1, features.shape[0], features.shape[1], features.shape[2])) # required for custom DQN
+            return features.reshape((features.shape[0], features.shape[1], features.shape[2])) # required for baseline DQN
+            #return features # (2*lookback, levels, count(features))
+        elif FeatureType.TRADES.value in self.market:
+            features = self.market[FeatureType.TRADES.value]
+            features = np.vstack((np.array([self.t, self.i, 0]), features))
+            return features
+        else:
+            Exception("Feature not known to ActionState.")
 
     def getT(self):
         return self.t

ctc_executioner/agent_utils/action_reward_log.py

@@ -0,0 +1,36 @@
+import numpy as np
+import matplotlib.pyplot as plt
+from rl.callbacks import Callback
+
+class ActionRewardLog(Callback):
+    def __init__(self, file_name_prefix, nb_episodes=10000, avgwindow=20):
+        self.rewards = np.zeros(nb_episodes) - 1000.0
+        self.X = np.arange(1, nb_episodes+1)
+        self.avgrewards = np.zeros(nb_episodes) - 1000.0
+        self.avgwindow = avgwindow
+        self.rewardbuf = []
+        self.episode = 0
+        self.nb_episodes = nb_episodes
+        self.file_actions = file_name_prefix + "_actions.py"
+        self.file_rewards = file_name_prefix + "_rewards.py"
+        self.file_rewards_mean = file_name_prefix + "_rewards_mean.py"
+
+    def on_episode_end(self, episode, logs):
+        if self.episode >= self.nb_episodes:
+            return
+        rw = logs['episode_reward']
+        actions = logs['episode_actions']
+        self.rewardbuf.append(rw)
+        if len(self.rewardbuf) > self.avgwindow:
+            del self.rewardbuf[0]
+        self.rewards[self.episode] = rw
+        rw_avg = np.mean(self.rewardbuf)
+        self.avgrewards[self.episode] = rw_avg
+        self.episode += 1
+        self.write(self.file_actions, actions)
+        self.write(self.file_rewards, rw)
+        self.write(self.file_rewards_mean, rw_avg)
+
+    def write(self, file, value):
+        f = open(file, "a")
+        f.write(str(value) + ',\n')

ctc_executioner/agent_utils/live_plot_callback.py

@@ -11,6 +11,7 @@ def __init__(self, nb_episodes=4000, avgwindow=20):
         self.rewardbuf = []
         self.episode = 0
         self.nb_episodes = nb_episodes
+        self.filename = "liveplot"
         plt.ion()
         self.fig = plt.figure()
         self.grphinst = plt.plot(self.X, self.rewards, color='b')[0]
@@ -36,8 +37,10 @@ def on_episode_end(self, episode, logs):
         self.avgrewards[self.episode] = np.mean(self.rewardbuf)
         self.plot()
         self.episode += 1
+
     def plot(self):
         self.grphinst.set_ydata(self.rewards)
         self.grphavg.set_ydata(self.avgrewards)
         plt.draw()
+        #if self.episode == 0:
         plt.pause(0.01)

ctc_executioner/agent_utils/ui.py

@@ -4,9 +4,12 @@
 class UI:
 
     @staticmethod
-    def animate(f, interval=5000, axis=[0, 100, -50, 50], frames=None):
-        fig = plt.figure()
+    def animate(f, interval=5000, axis=[0, 100, -50, 50], frames=None, title=""):
+        fig = plt.figure()#(figsize=(24, 18))
         ax1 = fig.add_subplot(1, 1, 1)
+        ax1.tick_params(axis='both', which='major', labelsize=25)
+
+        ax1.tick_params(axis='both', which='minor', labelsize=25)
         ax1.axis(axis)
         ax1.autoscale(True)
         xs = []
@@ -21,6 +24,8 @@ def do_animate(i, f, ax1, xs, ys):
             ys.append(y)
             ax1.clear()
             ax1.plot(xs, ys)
+            ax1.grid(linestyle='-', linewidth=2)
+            ax1.legend([title], prop={'size': 30})
 
         ani = animation.FuncAnimation(
             fig,

ctc_executioner/feature_type.py

@@ -0,0 +1,5 @@
+from enum import Enum
+
+class FeatureType(Enum):
+    ORDERS = 'bidask'
+    TRADES = 'trades'

ctc_executioner/match_engine.py

@@ -12,13 +12,15 @@ def __init__(self, orderbook, index=0, maxRuntime=100):
         self.orderbook = orderbook
         self.index = index
         self.maxRuntime = maxRuntime
-        self.matched = set()
+        self.matches = set()
+        self.recordMatches = False
 
     def _removePosition(self, side, price, qty):
-        self.matched.add((side, price, qty))
+        if self.recordMatches == True:
+            self.matches.add((side, price, qty))
 
     def _isRemoved(self, side, price, qty):
-        return (side, price, qty) in self.matched
+        return (side, price, qty) in self.matches
 
     def setIndex(self, index):
         self.index = index
@@ -65,13 +67,13 @@ def isMatchingPosition(p):
             if not partialTrades and qty >= order.getCty():
                 logging.debug("Full execution: " + str(qty) + " pcs available")
                 t = Trade(orderSide=order.getSide(), orderType=OrderType.LIMIT, cty=remaining, price=price, timestamp=orderbookState.getTimestamp())
-                self._removePosition(side=order.getSide(), price=price, qty=qty)
+                #self._removePosition(side=order.getSide(), price=price, qty=qty)
                 return [t]
             else:
                 logging.debug("Partial execution: " + str(qty) + " pcs available")
                 t = Trade(orderSide=order.getSide(), orderType=OrderType.LIMIT, cty=min(qty, remaining), price=price, timestamp=orderbookState.getTimestamp())
                 partialTrades.append(t)
-                self._removePosition(side=order.getSide(), price=price, qty=qty)
+                #self._removePosition(side=order.getSide(), price=price, qty=qty)
                 sidePosition = sidePosition + 1
                 remaining = remaining - qty
 

ctc_executioner/orderbook.py

@@ -1,12 +1,14 @@
 from dateutil import parser
 from ctc_executioner.order_side import OrderSide
+#from order_side import OrderSide
 import numpy as np
 import random
 from sklearn.preprocessing import MinMaxScaler
 from collections import OrderedDict
 import pandas as pd
 from diskcache import Cache
 from datetime import datetime
+import time
 
 class OrderbookEntry(object):
 
@@ -96,6 +98,9 @@ def getSellers(self):
     def getTimestamp(self):
         return self.timestamp
 
+    def getUnixTimestamp(self):
+        return time.mktime(self.getTimestamp().timetuple())
+
     def getBidAskMid(self):
         firstBuy = self.getBuyers()[0]
         firstSell = self.getSellers()[0]
@@ -143,11 +148,13 @@ def getPriceAtLevel(self, side, level):
         # priceEstimated = priceEnd + missingLevels * derivative_price
         # return priceEstimated
 
+
 class Orderbook(object):
 
     def __init__(self, extraFeatures=False):
         self.cache = Cache('/tmp/ctc-executioner')
         self.dictBook = None
+        self.trades = {}
         self.states = []
         self.extraFeatures = extraFeatures
         self.tmp = {}
@@ -472,12 +479,15 @@ def loadFromDict(self, d):
         from datetime import datetime
 
         # skip states until at least 1 bid and 1 ask is available
+        skip = 0
         while True:
             head_key = next(iter(d))
             head = d[head_key]
             if len(head["bids"]) > 0 and len(head["asks"]) > 0:
                 break
             del d[head_key]
+            skip = skip + 1
+        print("Skipped dict entries: " + str(skip))
 
         for ts in iter(d.keys()):
             state = d[ts]
@@ -498,6 +508,28 @@ def loadFromDict(self, d):
     def loadFromEventsFrame(self, events_pd):
         self.dictBook = Orderbook.generateDictFromEvents(events_pd)
         self.loadFromDict(self.dictBook)
+        self.trades = Orderbook.generateTradesFromEvents(events_pd)
+
+    @staticmethod
+    def generateTradesFromEvents(events_pd):
+        """ Generates dictionary based on historical trades.
+
+        dict :: {timestamp: trade}
+        state :: {'price': float, 'size': float, side: OrderSide}
+        """
+        import copy
+        trades = {}
+        for e in events_pd.itertuples():
+            if e.is_trade:
+                if e.is_bid:
+                    #side = OrderSide.BUY
+                    side = 0
+                else:
+                    #side = OrderSide.SELL
+                    side = 1
+                trades[e.ts] = {'price': e.price, 'size': e.size, 'side': side}
+        return trades
+
 
     def loadFromEvents(self, file, cols = ["ts", "seq", "size", "price", "is_bid", "is_trade", "ttype"], clean=50):
         print('Attempt to load from cache.')
@@ -506,6 +538,7 @@ def loadFromEvents(self, file, cols = ["ts", "seq", "size", "price", "is_bid", "
             print('Order book in cache. Load...')
             self.states = o.states
             self.dictBook = o.dictBook
+            self.trades = o.trades
         else:
             print('Order book not in cache. Read from file...')
             import pandas as pd
@@ -521,6 +554,9 @@ def loadFromEvents(self, file, cols = ["ts", "seq", "size", "price", "is_bid", "
 
     def plot(self, show_bidask=False, max_level=-1, show=True):
         import matplotlib.pyplot as plt
+        plt.figure(figsize=(24, 18))
+        plt.tick_params(axis='both', which='major', labelsize=25)
+        plt.tick_params(axis='both', which='minor', labelsize=25)
         price = [x.getBidAskMid() for x in self.getStates()]
         times = [x.getTimestamp() for x in self.getStates()]
         plt.plot(times, price)
@@ -633,13 +669,37 @@ def getBidAskFeatures(self, state_index, lookback, qty=None, price=True, size=Tr
                 features = np.vstack((features, features_next))
             i = i + 1
         return features
-#
 
-#o = Orderbook()
-#o.loadFromEvents('ob-1-small.tsv')
+    def get_hist_trades(self, ts, lookback=20):
+        ts -= 3600
+        acc_trades = {}
+        for ts_tmp, trade_tmp in sorted(list(self.trades.items()), reverse=True):
+            if ts_tmp <= ts:
+                acc_trades[ts_tmp] = trade_tmp
+                if len(acc_trades) == lookback:
+                    break
+        return acc_trades
+
+    def getHistTradesFeature(self, ts, lookback=20, normalize=True, norm_price=None, norm_size=None):
+        trades = self.get_hist_trades(ts, lookback=lookback)
+        trades = list(map(lambda v: [v['price'], v['size'], v['side']], trades.values()))
+        arr = np.array(trades)
+        if normalize:
+            arr = np.column_stack((arr[:,0]/norm_price, arr[:,1]/norm_size, arr[:,2]))
+        return arr
+
+
+# o = Orderbook()
+# o.loadFromEvents('data/events/ob-1-small.tsv')
+# ts = o.getStates()[100].getUnixTimestamp()
+# print(ts)
+# o.getHistTradesFeature(ts, normalize=True, norm_price=2.0, norm_size=2.0)
+#trades = o.getHistTradesFeature(ts)
+# print(trades[0:1])
+
+
 #o.generateDict()
 #print(o.dictBook[list(o.dictBook.keys())[0]])
-
 #o.plot()
 
 #o = Orderbook()

ctc_executioner/qlearn.py

@@ -4,25 +4,26 @@
 class QLearn:
     """Qlearner."""
 
-    def __init__(self, actions, epsilon=0.1, alpha=0.2, gamma=0.9):
+    def __init__(self, actions, epsilon=0.1, alpha=0.1, gamma=0.1, exploration_decay=1.000001):
         """Initialize Q-table and assign parameters."""
         self.q = {}
         self.epsilon = epsilon
         self.alpha = alpha
         self.gamma = gamma
+        self.exploration_decay = exploration_decay
         self.actions = actions
 
     def getQ(self, state, action, default=0.0):
         """Q-value lookup for state and action,  or else returns default."""
         return self.q.get((state, action), default)
 
-    def getQAction(self, state, default=None):
+    def getQAction(self, state, default=0.0):
         """Best action based on Q-Table for given state."""
         values = []
         for x in list(reversed(self.actions)):
-            q_value = self.q.get((state, x), None)
-            if q_value is not None:
-                values.append(q_value)
+            q_value = self.q.get((state, x), 0.0)
+            #if q_value is not 0.0:
+            values.append(q_value)
             # else:
             #    raise Exception("Q-Table does not contain: " + str((state, x)))
 
@@ -34,8 +35,8 @@ def getQAction(self, state, default=None):
         return a
 
     def learnQ(self, state, action, reward, value):
-        oldv = self.q.get((state, action), None)
-        if oldv is None:
+        oldv = self.q.get((state, action), 0.0)
+        if oldv is 0.0:
             self.q[(state, action)] = reward
         else:
             self.q[(state, action)] = oldv + self.alpha * (value - oldv)
@@ -46,7 +47,9 @@ def learn(self, state1, action1, reward, state2):
 
     def chooseAction(self, state, return_q=False):
         """Chooses most rewarding action."""
-        if random.random() < self.epsilon:
+        self.epsilon = self.exploration_decay * self.epsilon
+
+        if random.random() > self.epsilon:
             action = random.choice(self.actions)
         else:
             q = [self.getQ(state, a) for a in self.actions]