Merge pull request #369 from liampauling/release/1.15.4

history and version update

HISTORY.rst

@@ -3,6 +3,21 @@
 Release History
 ---------------
 
+1.15.4 (2021-01-18)
++++++++++++++++++++
+
+**Improvements**
+
+- Restrict catalogue requests to market version update
+
+**Bug Fixes**
+
+- #192 correctly lapse limit orders
+
+**Libraries**
+
+- betfairlightweight upgraded to 2.11.2
+
 1.15.3 (2021-01-11)
 +++++++++++++++++++
 

flumine/__version__.py

@@ -1,6 +1,6 @@
 __title__ = "flumine"
 __description__ = "Betfair trading framework"
 __url__ = "https://github.com/liampauling/flumine"
-__version__ = "1.15.3"
+__version__ = "1.15.4"
 __author__ = "Liam Pauling"
 __license__ = "MIT"

flumine/backtest/simulated.py

@@ -29,12 +29,12 @@ def __init__(self, order):
         self.size_cancelled = 0.0
         self.size_lapsed = 0.0
         self.size_voided = 0.0
+        self.market_version = None  # version at place so we can lapse if needed
         self._piq = 0.0
         self._bsp_reconciled = False
 
-    def __call__(self, market_book: MarketBook, runner_analytics):
+    def __call__(self, market_book: MarketBook, runner_analytics) -> None:
         # simulates order matching
-        # todo handle limit lapsing
         runner = self._get_runner(market_book)
         if (
             self._bsp_reconciled is False
@@ -46,6 +46,13 @@ def __call__(self, market_book: MarketBook, runner_analytics):
         elif (
             self.order.order_type.ORDER_TYPE == OrderTypes.LIMIT and self.size_remaining
         ):
+            if market_book.version != self.market_version:
+                self.market_version = market_book.version  # update for next time
+                if market_book.status == "SUSPENDED":  # Material change
+                    if self.order.order_type.persistence_type == "LAPSE":
+                        self.size_lapsed += self.size_remaining
+                        return
+
             # todo estimated piq cancellations
             self._process_traded(
                 market_book.publish_time_epoch, runner_analytics.traded
@@ -56,7 +63,9 @@ def place(
     ) -> SimulatedPlaceResponse:
         # simulates placeOrder request->matching->response
         # todo instruction/fillkill/timeInForce etc
-        # todo check marketVersion
+        # todo check marketVersion or reject entire package?
+
+        self.market_version = market_book.version
         if self.order.order_type.ORDER_TYPE == OrderTypes.LIMIT:
             runner = self._get_runner(market_book)
 

flumine/baseflumine.py

@@ -228,6 +228,7 @@ def _process_market_catalogues(self, event: events.MarketCatalogueEvent) -> None
                     )
                 else:
                     market.market_catalogue = market_catalogue
+                market.update_market_catalogue = False
 
     def _process_current_orders(self, event: events.CurrentOrdersEvent) -> None:
         process_current_orders(self.markets, self.strategies, event)  # update state

flumine/markets/market.py

@@ -24,10 +24,13 @@ def __init__(
         self.date_time_closed = None
         self.market_book = market_book
         self.market_catalogue = market_catalogue
+        self.update_market_catalogue = True
         self.context = {"simulated": {}}  # data store (raceCard / scores etc)
         self.blotter = Blotter(market_id)
 
     def __call__(self, market_book: MarketBook):
+        if market_book.version != self.market_book.version:
+            self.update_market_catalogue = True
         self.market_book = market_book
 
     def open_market(self) -> None:

flumine/markets/middleware.py

@@ -99,10 +99,6 @@ def _process_runner_removal(
                         extra=order.info,
                     )
                 else:
-                    if order.status == OrderStatus.EXECUTABLE:
-                        # todo cancel if not PERSIST
-                        # todo does a market version bump occur if withdrawal is below the limit?
-                        pass
                     if (
                         removal_adjustment_factor
                         and removal_adjustment_factor >= WIN_MINIMUM_ADJUSTMENT_FACTOR

flumine/worker.py

@@ -116,8 +116,12 @@ def keep_alive(context: dict, flumine) -> None:
 
 def poll_market_catalogue(context: dict, flumine) -> None:
     client = flumine.client
-    live_markets = list(flumine.markets.markets.keys())
-    for market_ids in chunks(live_markets, 25):
+    markets = [
+        m.market_id
+        for m in list(flumine.markets.markets.values())
+        if m.update_market_catalogue
+    ]
+    for market_ids in chunks(markets, 25):
         try:
             market_catalogues = client.betting_client.betting.list_market_catalogue(
                 filter=filters.market_filter(market_ids=market_ids),

requirements.txt

@@ -1,4 +1,4 @@
-betfairlightweight==2.11.1
+betfairlightweight==2.11.2
 tenacity==5.0.3
 python-json-logger==2.0.1
 requests

tests/test_baseflumine.py

@@ -208,7 +208,7 @@ def test__process_raw_data_no_id(self, mock__add_market):
 
     @mock.patch("flumine.baseflumine.events")
     @mock.patch("flumine.baseflumine.BaseFlumine.log_control")
-    def test__process_market_catalogue(self, mock_log_control, mock_events):
+    def test__process_market_catalogues(self, mock_log_control, mock_events):
         mock_market = mock.Mock()
         mock_market.market_catalogue = None
         mock_markets = mock.Mock()
@@ -221,6 +221,7 @@ def test__process_market_catalogue(self, mock_log_control, mock_events):
         self.base_flumine._process_market_catalogues(mock_event)
         self.assertEqual(mock_market.market_catalogue, mock_market_catalogue)
         mock_log_control.assert_called_with(mock_events.MarketEvent(mock_market))
+        self.assertFalse(mock_market.update_market_catalogue)
 
     def test__process_current_orders(self):
         mock_event = mock.Mock()

tests/test_markets.py

@@ -112,12 +112,14 @@ def test_init(self):
         self.assertIsNone(self.market.date_time_closed)
         self.assertEqual(self.market.market_book, self.mock_market_book)
         self.assertEqual(self.market.market_catalogue, self.mock_market_catalogue)
+        self.assertTrue(self.market.update_market_catalogue)
         self.assertEqual(self.market.context, {"simulated": {}})
 
     def test_call(self):
         mock_market_book = mock.Mock()
         self.market(mock_market_book)
         self.assertEqual(self.market.market_book, mock_market_book)
+        self.assertTrue(self.market.update_market_catalogue)
 
     def test_open_market(self):
         self.market.open_market()

tests/test_simulated.py

@@ -28,13 +28,32 @@ def test_init(self):
         self.assertEqual(self.simulated.size_lapsed, 0)
         self.assertEqual(self.simulated.size_voided, 0)
         self.assertEqual(self.simulated._piq, 0)
+        self.assertIsNone(self.simulated.market_version)
         self.assertFalse(self.simulated._bsp_reconciled)
 
     @mock.patch("flumine.backtest.simulated.Simulated._get_runner")
     @mock.patch("flumine.backtest.simulated.Simulated.take_sp", return_value=True)
     @mock.patch("flumine.backtest.simulated.Simulated._process_traded")
     @mock.patch("flumine.backtest.simulated.Simulated._process_sp")
-    def test_call(self, mock__process_sp, mock__process_traded, _, mock__get_runner):
+    def test_call_process_traded(
+        self, mock__process_sp, mock__process_traded, _, mock__get_runner
+    ):
+        mock_market_book = mock.Mock()
+        mock_market_book.bsp_reconciled = False
+        mock_runner_analytics = mock.Mock()
+        self.simulated(mock_market_book, mock_runner_analytics)
+        mock__process_sp.assert_not_called()
+        mock__process_traded.assert_called_with(
+            mock_market_book.publish_time_epoch, mock_runner_analytics.traded
+        )
+
+    @mock.patch("flumine.backtest.simulated.Simulated._get_runner")
+    @mock.patch("flumine.backtest.simulated.Simulated.take_sp", return_value=True)
+    @mock.patch("flumine.backtest.simulated.Simulated._process_traded")
+    @mock.patch("flumine.backtest.simulated.Simulated._process_sp")
+    def test_call_process_sp(
+        self, mock__process_sp, mock__process_traded, _, mock__get_runner
+    ):
         mock_market_book = mock.Mock()
         mock_market_book.bsp_reconciled = True
         mock_runner_analytics = mock.Mock()
@@ -44,6 +63,25 @@ def test_call(self, mock__process_sp, mock__process_traded, _, mock__get_runner)
         )
         mock__process_traded.assert_not_called()
 
+    @mock.patch("flumine.backtest.simulated.Simulated._get_runner")
+    @mock.patch("flumine.backtest.simulated.Simulated.take_sp", return_value=True)
+    @mock.patch("flumine.backtest.simulated.Simulated._process_traded")
+    @mock.patch("flumine.backtest.simulated.Simulated._process_sp")
+    def test_call_market_version(
+        self, mock__process_sp, mock__process_traded, _, mock__get_runner
+    ):
+        self.simulated.market_version = 123
+        self.simulated.order.order_type.persistence_type = "LAPSE"
+        mock_market_book = mock.Mock(
+            bsp_reconciled=False, version=124, status="SUSPENDED"
+        )
+        mock_runner_analytics = mock.Mock()
+        self.simulated(mock_market_book, mock_runner_analytics)
+        self.assertEqual(self.simulated.size_lapsed, 2.0)
+        self.assertEqual(self.simulated.size_remaining, 0.0)
+        mock__process_sp.assert_not_called()
+        mock__process_traded.assert_not_called()
+
     @mock.patch("flumine.backtest.simulated.Simulated._get_runner")
     @mock.patch("flumine.backtest.simulated.Simulated.take_sp", return_value=True)
     @mock.patch("flumine.backtest.simulated.Simulated._process_traded")
@@ -89,6 +127,7 @@ def test_place_limit_back(self, mock__get_runner):
         mock_runner.ex.available_to_lay = [{"price": 13, "size": 120}]
         mock__get_runner.return_value = mock_runner
         resp = self.simulated.place(mock_client, mock_market_book, {}, 1)
+        self.assertEqual(self.simulated.market_version, mock_market_book.version)
         self.assertEqual(resp.average_price_matched, 12)
         self.assertEqual(resp.size_matched, 2)
         self.assertEqual(

tests/test_worker.py

@@ -91,11 +91,16 @@ def test_keep_alive_ka_error(self):
     def test_poll_market_catalogue(self, mock_events):
         mock_context = mock.Mock()
         mock_flumine = mock.Mock()
-        mock_flumine.markets.markets = {"1.234": None, "5.678": None}
+        mock_market_one = mock.Mock(market_id="1.234", update_market_catalogue=True)
+        mock_market_two = mock.Mock(market_id="5.678", update_market_catalogue=False)
+        mock_flumine.markets.markets = {
+            "1.234": mock_market_one,
+            "5.678": mock_market_two,
+        }
 
         worker.poll_market_catalogue(mock_context, mock_flumine)
         mock_flumine.client.betting_client.betting.list_market_catalogue.assert_called_with(
-            filter={"marketIds": list(mock_flumine.markets.markets.keys())},
+            filter={"marketIds": ["1.234"]},
             market_projection=[
                 "COMPETITION",
                 "EVENT",