Merge pull request #372 from liampauling/release/1.16.2

Minor selection_exposure optimisations by caching properties

HISTORY.rst

@@ -3,6 +3,16 @@
 Release History
 ---------------
 
+1.16.2 (2021-02-05)
++++++++++++++++++++
+
+**Improvements**
+
+- Blotter strategy orders added for faster lookup
+- Strategy name hash cached
+- Minor selection_exposure optimisations
+- Simulated optimisations
+
 1.16.1 (2021-01-28)
 +++++++++++++++++++
 

flumine/__version__.py

@@ -1,6 +1,6 @@
 __title__ = "flumine"
 __description__ = "Betfair trading framework"
 __url__ = "https://github.com/liampauling/flumine"
-__version__ = "1.16.1"
+__version__ = "1.16.2"
 __author__ = "Liam Pauling"
 __license__ = "MIT"

flumine/backtest/simulated.py

@@ -35,12 +35,12 @@ def __init__(self, order):
 
     def __call__(self, market_book: MarketBook, runner_analytics) -> None:
         # simulates order matching
-        runner = self._get_runner(market_book)
         if (
             self._bsp_reconciled is False
             and market_book.bsp_reconciled
             and self.take_sp
         ):
+            runner = self._get_runner(market_book)
             self._process_sp(market_book.publish_time_epoch, runner)
 
         elif (

flumine/markets/blotter.py

@@ -1,5 +1,6 @@
 import logging
 from typing import Iterable
+from collections import defaultdict
 
 from ..order.ordertype import OrderTypes
 from ..utils import (
@@ -18,10 +19,27 @@
 
 
 class Blotter:
+
+    """
+    Simple and fast class to hold all orders for
+    a particular market.
+
+    `customer_order_ref` used as the key and various
+    caches available for faster access.
+
+        blotter["abc"] = <Order>  # set
+        "abc" in blotter  # contains
+        orders = [o for o in blotter]  # iter
+        order = blotter["abc"]  # get
+    """
+
     def __init__(self, market_id: str):
         self.market_id = market_id
         self._orders = {}  # {Order.id: Order}
         self._live_orders = []  # cached list of live orders
+        self._strategy_orders = defaultdict(
+            list
+        )  # cache list per strategy (faster lookup)
         # pending orders, list of (<Order>, {..})
         self.pending_place = []
         self.pending_cancel = []
@@ -30,7 +48,7 @@ def __init__(self, market_id: str):
 
     def strategy_orders(self, strategy) -> list:
         """Returns all orders related to a strategy."""
-        return [order for order in self if order.trade.strategy == strategy]
+        return self._strategy_orders[strategy]
 
     def process_orders(self, client, bet_delay: int = 0) -> list:
         packages = []
@@ -128,22 +146,24 @@ def selection_exposure(self, strategy, lookup: tuple) -> float:
         ub, ul = [], []  # unmatched bets, (price, size)
         moc_win_liability = 0.0
         moc_lose_liability = 0.0
-        for order in self:
-            if order.trade.strategy == strategy and order.lookup == lookup:
+        for order in self.strategy_orders(strategy):
+            if order.lookup == lookup:
                 if order.status == OrderStatus.VIOLATION:
                     continue
 
                 if order.order_type.ORDER_TYPE == OrderTypes.LIMIT:
-                    if order.size_matched:
+                    _size_matched = order.size_matched  # cache
+                    if _size_matched:
                         if order.side == "BACK":
-                            mb.append((order.average_price_matched, order.size_matched))
+                            mb.append((order.average_price_matched, _size_matched))
                         else:
-                            ml.append((order.average_price_matched, order.size_matched))
-                    if order.order_type.price and order.size_remaining:
+                            ml.append((order.average_price_matched, _size_matched))
+                    _size_remaining = order.size_remaining  # cache
+                    if order.order_type.price and _size_remaining:
                         if order.side == "BACK":
-                            ub.append((order.order_type.price, order.size_remaining))
+                            ub.append((order.order_type.price, _size_remaining))
                         else:
-                            ul.append((order.order_type.price, order.size_remaining))
+                            ul.append((order.order_type.price, _size_remaining))
                 elif order.order_type.ORDER_TYPE in (
                     OrderTypes.LIMIT_ON_CLOSE,
                     OrderTypes.MARKET_ON_CLOSE,
@@ -182,6 +202,7 @@ def has_order(self, customer_order_ref: str) -> bool:
     def __setitem__(self, customer_order_ref: str, order) -> None:
         self._orders[customer_order_ref] = order
         self._live_orders.append(order)
+        self._strategy_orders[order.trade.strategy].append(order)
 
     def __getitem__(self, customer_order_ref: str):
         return self._orders[customer_order_ref]

flumine/strategy/strategy.py

@@ -75,6 +75,8 @@ def __init__(
         self.streams = []  # list of streams strategy is subscribed
         self.historic_stream_ids = []
         self.log_validation_failures = log_validation_failures
+        # cache
+        self.name_hash = create_cheap_hash(self.name, STRATEGY_NAME_HASH_LENGTH)
 
     def check_market(self, market: Market, market_book: MarketBook) -> bool:
         if market_book.streaming_unique_id not in self.stream_ids:
@@ -259,10 +261,6 @@ def info(self) -> dict:
     def name(self) -> str:
         return self._name or self.__class__.__name__
 
-    @property
-    def name_hash(self) -> str:
-        return create_cheap_hash(self.name, STRATEGY_NAME_HASH_LENGTH)
-
     def __str__(self):
         return "{0}".format(self.name)
 

tests/test_blotter.py

@@ -22,7 +22,7 @@ def test_init(self):
     def test_strategy_orders(self):
         mock_order = mock.Mock()
         mock_order.trade.strategy = 69
-        self.blotter._orders = {"12345": mock_order}
+        self.blotter["12345"] = mock_order
         self.assertEqual(self.blotter.strategy_orders(12), [])
         self.assertEqual(self.blotter.strategy_orders(69), [mock_order])
 
@@ -150,7 +150,7 @@ def test_selection_exposure(self):
             size_remaining=0.0,
             order_type=LimitOrder(price=5.6, size=2.0),
         )
-        self.blotter._orders = {"12345": mock_order}
+        self.blotter["12345"] = mock_order
         self.assertEqual(
             self.blotter.selection_exposure(mock_strategy, mock_order.lookup),
             2.0,
@@ -168,7 +168,7 @@ def test_selection_exposure_raises_value_error(self):
             size_remaining=0.0,
             order_type=mock.Mock(ORDER_TYPE="INVALID"),
         )
-        self.blotter._orders = {"12345": mock_order}
+        self.blotter["12345"] = mock_order
 
         with self.assertRaises(ValueError) as e:
             self.blotter.selection_exposure(mock_strategy, mock_order.lookup)
@@ -203,7 +203,8 @@ def test_selection_exposure_with_price_none(self):
             size_remaining=2.0,
             order_type=LimitOrder(price=None, size=2.0),
         )
-        self.blotter._orders = {"12345": mock_order1, "23456": mock_order2}
+        self.blotter["12345"] = mock_order1
+        self.blotter["23456"] = mock_order2
         self.assertEqual(
             self.blotter.selection_exposure(mock_strategy, lookup),
             2.0,
@@ -221,7 +222,7 @@ def test_selection_exposure_no_match(self):
             size_remaining=0.0,
             order_type=LimitOrder(price=5.6, size=2.0),
         )
-        self.blotter._orders = {"12345": mock_order}
+        self.blotter["12345"] = mock_order
         self.assertEqual(
             self.blotter.selection_exposure(mock_strategy, mock_order.lookup),
             0.0,
@@ -239,7 +240,7 @@ def test_selection_exposure_from_unmatched_back(self):
             size_remaining=2.0,
             order_type=LimitOrder(price=6, size=4.0),
         )
-        self.blotter._orders = {"12345": mock_order}
+        self.blotter["12345"] = mock_order
         # On the win side, we have 2.0 * (5.6-1.0) = 9.2
         # On the lose side, we have -2.0-2.0=-4.0
         self.assertEqual(
@@ -259,7 +260,7 @@ def test_selection_exposure_from_unmatched_lay(self):
             size_remaining=2.0,
             order_type=LimitOrder(price=6, size=4.0),
         )
-        self.blotter._orders = {"12345": mock_order}
+        self.blotter["12345"] = mock_order
         # On the win side, we have -2.0 * (5.6-1.0) -2.0 * (6.0-1.0) = -19.2
         # On the lose side, we have 2.0 from size_matched
         self.assertEqual(
@@ -276,7 +277,7 @@ def test_selection_exposure_from_market_on_close_back(self):
             side="BACK",
             order_type=MarketOnCloseOrder(liability=10.0),
         )
-        self.blotter._orders = {"12345": mock_order}
+        self.blotter["12345"] = mock_order
         self.assertEqual(
             self.blotter.selection_exposure(mock_strategy, mock_order.lookup),
             10.0,
@@ -291,7 +292,7 @@ def test_selection_exposure_from_market_on_close_lay(self):
             side="LAY",
             order_type=MarketOnCloseOrder(liability=10.0),
         )
-        self.blotter._orders = {"12345": mock_order}
+        self.blotter["12345"] = mock_order
         self.assertEqual(
             self.blotter.selection_exposure(mock_strategy, mock_order.lookup),
             10.0,
@@ -306,7 +307,7 @@ def test_selection_exposure_from_limit_on_close_lay(self):
             side="LAY",
             order_type=LimitOnCloseOrder(price=1.01, liability=10.0),
         )
-        self.blotter._orders = {"12345": mock_order}
+        self.blotter["12345"] = mock_order
         self.assertEqual(
             self.blotter.selection_exposure(mock_strategy, mock_order.lookup),
             10.0,
@@ -322,7 +323,7 @@ def test_selection_exposure_voided(self):
             order_type=LimitOrder(price=5, size=10.0),
             status=OrderStatus.VIOLATION,
         )
-        self.blotter._orders = {"12345": mock_order}
+        self.blotter["12345"] = mock_order
         self.assertEqual(
             self.blotter.selection_exposure(mock_strategy, mock_order.lookup),
             0,
@@ -338,9 +339,13 @@ def test__contains(self):
         self.assertNotIn("321", self.blotter)
 
     def test__setitem(self):
-        self.blotter["123"] = "test"
-        self.assertEqual(self.blotter._orders, {"123": "test"})
-        self.assertEqual(self.blotter._live_orders, ["test"])
+        mock_order = mock.Mock()
+        self.blotter["123"] = mock_order
+        self.assertEqual(self.blotter._orders, {"123": mock_order})
+        self.assertEqual(self.blotter._live_orders, [mock_order])
+        self.assertEqual(
+            self.blotter._strategy_orders, {mock_order.trade.strategy: [mock_order]}
+        )
 
     def test__getitem(self):
         self.blotter._orders = {"12345": "test", "54321": "test2"}

tests/test_strategy.py

@@ -70,6 +70,7 @@ def test_init(self):
         self.assertEqual(self.strategy.max_live_trade_count, 4)
         self.assertEqual(self.strategy.streams, [])
         self.assertEqual(self.strategy.historic_stream_ids, [])
+        self.assertEqual(self.strategy.name_hash, "a94a8fe5ccb19")
 
     def test_check_market_no_subscribed(self):
         mock_market = mock.Mock()
@@ -261,8 +262,5 @@ def test_info(self):
     def test_name(self):
         self.assertEqual(self.strategy.name, "test")
 
-    def test_name_hash(self):
-        self.assertEqual(self.strategy.name_hash, "a94a8fe5ccb19")
-
     def test_str(self):
         self.assertEqual(str(self.strategy), "test")

tests/test_tradingcontrols.py

@@ -377,7 +377,8 @@ def test_validate_limit_with_multiple_strategies_fails(self, mock_on_error):
         order2.average_price_matched = 0.0
         order2.size_matched = 0
 
-        self.market.blotter._orders = {"order1": order1, "order2": order2}
+        self.market.blotter["order1"] = order1
+        self.market.blotter["order2"] = order2
 
         order_package = mock.Mock()
         order_package.package_type = OrderPackageType.PLACE