Merge pull request #354 from liampauling/release/2.10.0

orjson upgrade

.github/workflows/test.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [3.6, 3.7, 3.8, 3.9]
 
     steps:
     - uses: actions/checkout@v2

HISTORY.rst

@@ -3,6 +3,18 @@
 Release History
 ---------------
 
+2.10.0 (2020-11-02)
++++++++++++++++++++
+
+**Improvements**
+
+- #352 exchange stream API release (10/11/20)
+- Add py3.9 actions test
+
+**Dependencies**
+
+- orjson upgraded to 3.4.3
+
 2.9.2 (2020-10-26)
 +++++++++++++++++++
 

betfairlightweight/__version__.py

@@ -1,6 +1,6 @@
 __title__ = "betfairlightweight"
 __description__ = "Lightweight python wrapper for Betfair API-NG"
 __url__ = "https://github.com/liampauling/betfair"
-__version__ = "2.9.2"
+__version__ = "2.10.0"
 __author__ = "Liam Pauling"
 __license__ = "MIT"

betfairlightweight/streaming/cache.py

@@ -418,7 +418,7 @@ def __init__(self, **kwargs):
         self.market_id = kwargs.get("id")
         self.closed = kwargs.get("closed")
         self.streaming_update = None
-        self.runners = []
+        self.runners = {}  # (selectionId, handicap):
 
     def update_cache(self, order_book: dict, publish_time: int) -> None:
         self._datetime_updated = self.strip_datetime(publish_time)
@@ -430,16 +430,18 @@ def update_cache(self, order_book: dict, publish_time: int) -> None:
         for order_changes in order_book.get("orc", []):
             selection_id = order_changes["id"]
             handicap = order_changes.get("hc", 0)
-            runner = self.runner_dict.get((selection_id, handicap))
-            if runner:
+            full_image = order_changes.get("fullImage")
+            _lookup = (selection_id, handicap)
+            runner = self.runners.get(_lookup)
+            if full_image or runner is None:
+                self.runners[_lookup] = OrderBookRunner(**order_changes)
+            else:
                 if "ml" in order_changes:
                     runner.matched_lays.update(order_changes["ml"])
                 if "mb" in order_changes:
                     runner.matched_backs.update(order_changes["mb"])
                 if "uo" in order_changes:
                     runner.update_unmatched(order_changes["uo"])
-            else:
-                self.runners.append(OrderBookRunner(**order_changes))
 
     def create_resource(
         self, unique_id: int, lightweight: bool, snap: bool = False
@@ -459,15 +461,10 @@ def create_resource(
                 **data
             )
 
-    @property
-    def runner_dict(self) -> dict:
-        return {
-            (runner.selection_id, runner.handicap): runner for runner in self.runners
-        }
-
     @property
     def serialise(self) -> dict:
+        runners = list(self.runners.values())  # runner may be added
         orders = []
-        for runner in self.runners:
+        for runner in runners:
             orders.extend(runner.serialise_orders(self.market_id))
         return {"currentOrders": orders, "moreAvailable": False}

betfairlightweight/streaming/stream.py

@@ -149,10 +149,11 @@ def _process(self, data: list, publish_time: int) -> bool:
         output_market_book, img = [], False
         for market_book in data:
             market_id = market_book["id"]
+            full_image = market_book.get("img", False)
             market_book_cache = self._caches.get(market_id)
 
             if (
-                market_book.get("img") or market_book_cache is None
+                full_image or market_book_cache is None
             ):  # historic data does not contain img
                 img = True
                 if "marketDefinition" not in market_book:
@@ -190,9 +191,10 @@ def _process(self, data: list, publish_time: int) -> bool:
         output_order_book, img = [], False
         for order_book in data:
             market_id = order_book["id"]
+            full_image = order_book.get("fullImage", False)
             order_book_cache = self._caches.get(market_id)
 
-            if order_book_cache is None:
+            if full_image or order_book_cache is None:
                 img = True
                 order_book_cache = OrderBookCache(
                     publish_time=publish_time, **order_book

requirements-speed.txt

@@ -1,2 +1,2 @@
 ciso8601==2.1.3
-orjson==3.4.1
+orjson==3.4.3

tests/resources/streaming_ocm_FULL_IMAGE.json

@@ -1 +1 @@
-{"op":"ocm","id":3,"clk":"AMYkAJokAJAnAIk2ANYk","pt":1566480793554,"oc":[{"id":"1.161613698","orc":[{"fullImage":true,"id":7017905,"hc":8.5,"uo":[{"id":"175706685825","p":2,"s":2,"side":"B","status":"EC","pt":"L","ot":"L","pd":1566480793000,"md":1566480793000,"avp":2,"sm":2,"sr":0,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_LGA","rfo":"","rfs":""}],"mb":[[2,2]]},{"fullImage":true,"id":7017905,"hc":7.5,"uo":[{"id":"175706685826","p":2,"s":2,"side":"B","status":"EC","pt":"L","ot":"L","pd":1566480793000,"md":1566480793000,"avp":2,"sm":2,"sr":0,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_LGA","rfo":"","rfs":""}],"mb":[[2,2]]},{"fullImage":true,"id":7017905,"uo":[{"id":"175706685827","p":2,"s":2,"side":"B","status":"EC","pt":"L","ot":"L","pd":1566480793000,"md":1566480793000,"avp":2,"sm":2,"sr":0,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_LGA","rfo":"","rfs":""}],"mb":[[2,2]]},{"fullImage":true,"id":7017905,"uo":[{"id":"175706685828","p":2,"s":2,"side":"B","status":"EC","pt":"L","ot":"L","pd":1566480793000,"md":1566480793000,"avp":2,"sm":2,"sr":0,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_LGA","rfo":"","rfs":""}],"mb":[[2,2]]},{"fullImage":true,"id":1017905,"uo":[{"id":"175706685828","p":2,"s":2,"side":"B","status":"EC","pt":"L","ot":"L","pd":1566480793000,"md":1566480793000,"avp":2,"sm":2,"sr":0,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_LGA","rfo":"","rfs":""}],"mb":[[2,2]]}]}]}
+{"op":"ocm","id":3,"clk":"AMYkAJokAJAnAIk2ANYk","pt":1566480793554,"oc":[{"id":"1.161613698","orc":[{"fullImage":true,"id":7017905,"hc":8.5,"uo":[{"id":"175706685825","p":2,"s":2,"side":"B","status":"EC","pt":"L","ot":"L","pd":1566480793000,"md":1566480793000,"avp":2,"sm":2,"sr":0,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_LGA","rfo":"","rfs":""}],"mb":[[2,2]]},{"fullImage":true,"id":7017905,"hc":7.5,"uo":[{"id":"175706685826","p":2,"s":2,"side":"B","status":"EC","pt":"L","ot":"L","pd":1566480793000,"md":1566480793000,"avp":2,"sm":2,"sr":0,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_LGA","rfo":"","rfs":""}],"mb":[[2,2]]},{"fullImage":true,"id":7017905,"hc":6.5,"uo":[{"id":"175706685827","p":2,"s":2,"side":"B","status":"EC","pt":"L","ot":"L","pd":1566480793000,"md":1566480793000,"avp":2,"sm":2,"sr":0,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_LGA","rfo":"","rfs":""}],"mb":[[2,2]]},{"fullImage":true,"id":7017905,"hc":5.5,"uo":[{"id":"175706685828","p":2,"s":2,"side":"B","status":"EC","pt":"L","ot":"L","pd":1566480793000,"md":1566480793000,"avp":2,"sm":2,"sr":0,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_LGA","rfo":"","rfs":""}],"mb":[[2,2]]},{"fullImage":true,"id":1017905,"uo":[{"id":"175706685828","p":2,"s":2,"side":"B","status":"EC","pt":"L","ot":"L","pd":1566480793000,"md":1566480793000,"avp":2,"sm":2,"sr":0,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_LGA","rfo":"","rfs":""}],"mb":[[2,2]]}]}]}

tests/resources/streaming_ocm_NEW_FULL_IMAGE.json

@@ -0,0 +1 @@
+{"op":"ocm","id":3,"clk":"AMYkAJokAJAnAIk2ANYk","pt":1566480793554,"oc":[{"id":"1.161613698","fullImage": true,"orc":[{"fullImage":true,"id":7017905,"hc":8.5,"uo":[{"id":"175706685825","p":2,"s":2,"side":"B","status":"EC","pt":"L","ot":"L","pd":1566480793000,"md":1566480793000,"avp":2,"sm":2,"sr":0,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_LGA","rfo":"","rfs":""}],"mb":[[2,2]]},{"fullImage":true,"id":7017905,"hc":7.5,"uo":[{"id":"175706685826","p":2,"s":2,"side":"B","status":"EC","pt":"L","ot":"L","pd":1566480793000,"md":1566480793000,"avp":2,"sm":2,"sr":0,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_LGA","rfo":"","rfs":""}],"mb":[[2,2]]},{"fullImage":true,"id":7017905,"uo":[{"id":"175706685827","p":2,"s":2,"side":"B","status":"EC","pt":"L","ot":"L","pd":1566480793000,"md":1566480793000,"avp":2,"sm":2,"sr":0,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_LGA","rfo":"","rfs":""}],"mb":[[2,2]]},{"fullImage":true,"id":7017905,"uo":[{"id":"175706685828","p":2,"s":2,"side":"B","status":"EC","pt":"L","ot":"L","pd":1566480793000,"md":1566480793000,"avp":2,"sm":2,"sr":0,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_LGA","rfo":"","rfs":""}],"mb":[[2,2]]},{"fullImage":true,"id":1017905,"uo":[{"id":"175706685828","p":2,"s":2,"side":"B","status":"EC","pt":"L","ot":"L","pd":1566480793000,"md":1566480793000,"avp":2,"sm":2,"sr":0,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_LGA","rfo":"","rfs":""}],"mb":[[2,2]]}]}]}

tests/unit/test_cache.py

@@ -332,9 +332,10 @@ def setUp(self):
         self.runner.handicap = 0
         self.runner.serialise_orders = mock.Mock(return_value=[])
         self.runner.unmatched_orders = [1]
-        self.order_book_cache.runners = [self.runner]
+        self.order_book_cache.runners = {(10895629, 0): self.runner}
 
     def test_full_image(self):
+        self.order_book_cache.runners = {}
         mock_response = create_mock_json(
             "tests/resources/streaming_ocm_FULL_IMAGE.json"
         )
@@ -343,12 +344,8 @@ def test_full_image(self):
             self.assertEqual(self.order_book_cache.streaming_update, order_book)
 
         self.assertEqual(len(self.order_book_cache.runners), 5)
-        self.assertEqual(len(self.order_book_cache.runner_dict), 5)
-        for k, v in self.order_book_cache.runner_dict.items():
-            if k == (7017905, 0):
-                self.assertEqual(len(v.unmatched_orders), 2)
-            else:
-                self.assertEqual(len(v.unmatched_orders), 1)
+        for k, v in self.order_book_cache.runners.items():
+            self.assertEqual(len(v.unmatched_orders), 1)
 
     def test_update_cache(self):
         mock_response = create_mock_json("tests/resources/streaming_ocm_UPDATE.json")
@@ -365,7 +362,7 @@ def test_update_cache(self):
 
     @mock.patch("betfairlightweight.streaming.cache.OrderBookRunner")
     def test_update_cache_new(self, mock_order_book_runner):
-        self.runner.selection_id = 108956
+        self.order_book_cache.runners = {(108956, 0): self.runner}
         mock_response = create_mock_json("tests/resources/streaming_ocm_UPDATE.json")
         for order_book in mock_response.json().get("oc"):
             self.order_book_cache.update_cache(order_book, 1234)
@@ -400,21 +397,17 @@ def test_create_resource(self, mock_current_orders, mock_serialise):
         current_orders = self.order_book_cache.create_resource(123, False)
         assert current_orders == mock_current_orders()
 
-    def test_runner_dict(self):
-        class Runner:
-            def __init__(self, selection_id, name, handicap):
-                self.selection_id = selection_id
-                self.name = name
-                self.handicap = handicap
-
-        (a, b) = (Runner(123, "a", 0), Runner(456, "b", 1))
-        self.order_book_cache.runners = [a, b]
-        assert self.order_book_cache.runner_dict == {(123, 0): a, (456, 1): b}
-
     def test_serialise(self):
+        mock_runner_one = mock.Mock()
+        mock_runner_one.serialise_orders.return_value = [1]
+        mock_runner_two = mock.Mock()
+        mock_runner_two.serialise_orders.return_value = [2, 3]
+        self.order_book_cache.runners = {
+            (123, 0): mock_runner_one,
+            (123, 1): mock_runner_two,
+        }
         serialised = self.order_book_cache.serialise
-
-        assert serialised == {"currentOrders": [], "moreAvailable": False}
+        assert serialised == {"currentOrders": [1, 2, 3], "moreAvailable": False}
 
 
 class TestOrderBookRunner(unittest.TestCase):

tests/unit/test_stream.py

@@ -285,6 +285,18 @@ def test_process(self, mock_on_process, mock_cache):
         self.assertEqual(len(self.stream), len(data))
         self.assertFalse(self.stream._process(data, 123))
 
+    @mock.patch("betfairlightweight.streaming.stream.OrderBookCache")
+    @mock.patch("betfairlightweight.streaming.stream.OrderStream.on_process")
+    def test_process_new_image(self, mock_on_process, mock_cache):
+        self.stream._caches = {"1.161613698": mock.Mock()}
+        sub_image = create_mock_json(
+            "tests/resources/streaming_ocm_NEW_FULL_IMAGE.json"
+        )
+        data = sub_image.json()["oc"]
+        self.assertTrue(self.stream._process(data, 123))
+        self.assertEqual(len(self.stream), len(data))
+        self.assertTrue(self.stream._process(data, 123))
+
     def test_str(self):
         assert str(self.stream) == "OrderStream"