Merge 223d1ac into b8de319

betcode-org · Nov 28, 2020 · 455084b · 455084b
2 parents b8de319 + 223d1ac
commit 455084b
Show file tree

Hide file tree

Showing 14 changed files with 113 additions and 65 deletions.
diff --git a/HISTORY.rst b/HISTORY.rst
@@ -3,6 +3,18 @@
 Release History
 ---------------
 
+2.10.2 (2020-11-28)
++++++++++++++++++++
+
+**Improvements**
+
+- #359 Exchange Stream API Release - Tuesday 8th December – New field - cancelledDate
+- Historical gen updated to only yield on data (reduces function calls in flumine)
+
+**Dependencies**
+
+- orjson upgraded to 3.4.4
+
 2.10.1 (2020-11-24)
 +++++++++++++++++++
 

diff --git a/betfairlightweight/__version__.py b/betfairlightweight/__version__.py
@@ -1,6 +1,6 @@
 __title__ = "betfairlightweight"
 __description__ = "Lightweight python wrapper for Betfair API-NG"
 __url__ = "https://github.com/liampauling/betfair"
-__version__ = "2.10.1"
+__version__ = "2.10.2"
 __author__ = "Liam Pauling"
 __license__ = "MIT"
diff --git a/betfairlightweight/compat.py b/betfairlightweight/compat.py
@@ -1,6 +1,9 @@
 from typing import Optional
 from datetime import datetime
 
+
+BETFAIR_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
+
 basestring = (str, bytes)
 numeric_types = (int, float)
 integer_types = (int,)
@@ -27,6 +30,6 @@ def parse_datetime(datetime_string: str) -> Optional[datetime]:
 
     def parse_datetime(datetime_string: str) -> Optional[datetime]:
         try:
-            return datetime.strptime(datetime_string, "%Y-%m-%dT%H:%M:%S.%fZ")
+            return datetime.strptime(datetime_string, BETFAIR_DATE_FORMAT)
         except ValueError:
             return
diff --git a/betfairlightweight/resources/bettingresources.py b/betfairlightweight/resources/bettingresources.py
@@ -648,6 +648,9 @@ def __init__(
         customerStrategyRef: str = None,
         customerOrderRef: str = None,
         regulatorAuthCode: str = None,
+        lapsedDate: str = None,
+        lapseStatusReasonCode: str = None,
+        cancelledDate: str = None,
     ):
         self.bet_id = betId
         self.average_price_matched = averagePriceMatched
@@ -671,6 +674,9 @@ def __init__(
         self.customer_strategy_ref = customerStrategyRef
         self.customer_order_ref = customerOrderRef
         self.price_size = PriceSize(**priceSize)
+        self.lapsed_date = BaseResource.strip_datetime(lapsedDate)
+        self.lapse_status_reason_code = lapseStatusReasonCode
+        self.cancelled_date = BaseResource.strip_datetime(cancelledDate)
 
 
 class CurrentOrders(BaseResource):

diff --git a/betfairlightweight/streaming/betfairstream.py b/betfairlightweight/streaming/betfairstream.py
@@ -339,7 +339,9 @@ def _read_loop(self) -> dict:
                 if not self._running:
                     break
                 else:
-                    yield self.listener.snap()
+                    data = self.listener.snap()
+                    if data:  # can return empty list
+                        yield data
             else:
                 # if f has finished, also stop the stream
                 self.stop()
diff --git a/betfairlightweight/streaming/cache.py b/betfairlightweight/streaming/cache.py
@@ -9,6 +9,7 @@
     StreamingStatus,
 )
 from ..exceptions import CacheError
+from ..utils import create_date_string
 
 
 class Available:
@@ -315,6 +316,7 @@ def __init__(
         rac: str = None,
         rc: str = None,
         lsrc: str = None,
+        cd: int = None,
         **kwargs
     ):
         self.bet_id = id
@@ -326,9 +328,9 @@ def __init__(
         self.persistence_type = pt
         self.order_type = ot
         self.placed_date = BaseResource.strip_datetime(pd)
-        self.placed_date_string = self.create_placed_date_string()
+        self._placed_date_string = create_date_string(self.placed_date)
         self.matched_date = BaseResource.strip_datetime(md)
-        self.matched_date_string = self.create_matched_date_string()
+        self._matched_date_string = create_date_string(self.matched_date)
         self.average_price_matched = avp
         self.size_matched = sm
         self.size_remaining = sr
@@ -340,15 +342,10 @@ def __init__(
         self.reference_order = rfo
         self.reference_strategy = rfs
         self.lapsed_date = BaseResource.strip_datetime(ld)
-        self.lapse_status_reason_code = lsrc  # todo add to output?
-
-    def create_placed_date_string(self) -> str:
-        if self.placed_date:
-            return self.placed_date.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
-
-    def create_matched_date_string(self) -> str:
-        if self.matched_date:
-            return self.matched_date.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+        self._lapsed_date_string = create_date_string(self.lapsed_date)
+        self.lapse_status_reason_code = lsrc
+        self.cancelled_date = BaseResource.strip_datetime(cd)
+        self._cancelled_date_string = create_date_string(self.cancelled_date)
 
     def serialise(self, market_id: str, selection_id: int, handicap: int) -> dict:
         return {
@@ -357,13 +354,14 @@ def serialise(self, market_id: str, selection_id: int, handicap: int) -> dict:
             "bspLiability": self.bsp_liability,
             "handicap": handicap,
             "marketId": market_id,
-            "matchedDate": self.matched_date_string,
+            "matchedDate": self._matched_date_string,
             "orderType": StreamingOrderType[self.order_type].value,
             "persistenceType": StreamingPersistenceType[self.persistence_type].value
             if self.persistence_type
             else None,
-            "placedDate": self.placed_date_string,
+            "placedDate": self._placed_date_string,
             "priceSize": {"price": self.price, "size": self.size},
+            "regulatorAuthCode": self.regulator_auth_code,
             "regulatorCode": self.regulator_code,
             "selectionId": selection_id,
             "side": StreamingSide[self.side].value,
@@ -375,6 +373,9 @@ def serialise(self, market_id: str, selection_id: int, handicap: int) -> dict:
             "status": StreamingStatus[self.status].value,
             "customerStrategyRef": self.reference_strategy,
             "customerOrderRef": self.reference_order,
+            "lapsedDate": self._lapsed_date_string,
+            "lapseStatusReasonCode": self.lapse_status_reason_code,
+            "cancelledDate": self._cancelled_date_string,
         }
 
 

diff --git a/betfairlightweight/utils.py b/betfairlightweight/utils.py
@@ -1,5 +1,8 @@
 import requests
+import datetime
+from typing import Optional
 
+from .compat import BETFAIR_DATE_FORMAT
 from .exceptions import StatusCodeError
 from .__version__ import __title__, __version__
 
@@ -62,3 +65,12 @@ def to_camel_case(snake_str: str) -> str:
 
 def default_user_agent():
     return "{0}/{1}".format(__title__, __version__)
+
+
+def create_date_string(date: datetime.datetime) -> Optional[str]:
+    """
+    Convert datetime to betfair
+    date string.
+    """
+    if date:
+        return date.strftime(BETFAIR_DATE_FORMAT)
diff --git a/requirements-speed.txt b/requirements-speed.txt
@@ -1,2 +1,2 @@
 ciso8601==2.1.3
-orjson==3.4.3
+orjson==3.4.4
diff --git a/tests/resources/streaming_ocm_FULL_IMAGE.json b/tests/resources/streaming_ocm_FULL_IMAGE.json
@@ -1 +1 @@
-{"op":"ocm","id":3,"clk":"AMYkAJokAJAnAIk2ANYk","pt":1566480793554,"oc":[{"id":"1.161613698","orc":[{"fullImage":true,"id":7017905,"hc":8.5,"uo":[{"id":"175706685825","p":2,"s":2,"side":"B","status":"EC","pt":"L","ot":"L","pd":1566480793000,"md":1566480793000,"avp":2,"sm":2,"sr":0,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_LGA","rfo":"","rfs":""}],"mb":[[2,2]]},{"fullImage":true,"id":7017905,"hc":7.5,"uo":[{"id":"175706685826","p":2,"s":2,"side":"B","status":"EC","pt":"L","ot":"L","pd":1566480793000,"md":1566480793000,"avp":2,"sm":2,"sr":0,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_LGA","rfo":"","rfs":""}],"mb":[[2,2]]},{"fullImage":true,"id":7017905,"hc":6.5,"uo":[{"id":"175706685827","p":2,"s":2,"side":"B","status":"EC","pt":"L","ot":"L","pd":1566480793000,"md":1566480793000,"avp":2,"sm":2,"sr":0,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_LGA","rfo":"","rfs":""}],"mb":[[2,2]]},{"fullImage":true,"id":7017905,"hc":5.5,"uo":[{"id":"175706685828","p":2,"s":2,"side":"B","status":"EC","pt":"L","ot":"L","pd":1566480793000,"md":1566480793000,"avp":2,"sm":2,"sr":0,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_LGA","rfo":"","rfs":""}],"mb":[[2,2]]},{"fullImage":true,"id":1017905,"uo":[{"id":"175706685828","p":2,"s":2,"side":"B","status":"EC","pt":"L","ot":"L","pd":1566480793000,"md":1566480793000,"avp":2,"sm":2,"sr":0,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_LGA","rfo":"","rfs":""}],"mb":[[2,2]]}]}]}
+{"op":"ocm","id":3,"clk":"AMYkAJokAJAnAIk2ANYk","pt":1566480793554,"oc":[{"id":"1.161613698","orc":[{"fullImage":true,"id":7017905,"hc":8.5,"uo":[{"id":"175706685825","p":2,"s":2,"side":"B","status":"EC","pt":"L","ot":"L","pd":1566480793000,"md":1566480793000,"cd":1478546670000,"avp":2,"sm":2,"sr":0,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_LGA","rfo":"","rfs":""}],"mb":[[2,2]]},{"fullImage":true,"id":7017905,"hc":7.5,"uo":[{"id":"175706685826","p":2,"s":2,"side":"B","status":"EC","pt":"L","ot":"L","pd":1566480793000,"md":1566480793000,"avp":2,"sm":2,"sr":0,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_LGA","rfo":"","rfs":""}],"mb":[[2,2]]},{"fullImage":true,"id":7017905,"hc":6.5,"uo":[{"id":"175706685827","p":2,"s":2,"side":"B","status":"EC","pt":"L","ot":"L","pd":1566480793000,"md":1566480793000,"avp":2,"sm":2,"sr":0,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_LGA","rfo":"","rfs":""}],"mb":[[2,2]]},{"fullImage":true,"id":7017905,"hc":5.5,"uo":[{"id":"175706685828","p":2,"s":2,"side":"B","status":"EC","pt":"L","ot":"L","pd":1566480793000,"md":1566480793000,"avp":2,"sm":2,"sr":0,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_LGA","rfo":"","rfs":""}],"mb":[[2,2]]},{"fullImage":true,"id":1017905,"uo":[{"id":"175706685828","p":2,"s":2,"side":"B","status":"EC","pt":"L","ot":"L","pd":1566480793000,"md":1566480793000,"avp":2,"sm":2,"sr":0,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_LGA","rfo":"","rfs":""}],"mb":[[2,2]]}]}]}
diff --git a/tests/resources/streaming_ocm_UPDATE.json b/tests/resources/streaming_ocm_UPDATE.json
@@ -1 +1 @@
-{"op":"ocm","id":12345,"clk":"AOkVAIMZALsQAKsQAO8M","pt":1478546671115,"oc":[{"id":"1.128126331","orc":[{"id":10895629,"uo":[{"id":"78996704480","p":1.02,"s":2,"side":"L","status":"E","pt":"L","ot":"L","pd":1478546670000,"sm":0,"sr":2,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_GGC","rfo":"","rfs":""}]}]}]}
+{"op":"ocm","id":12345,"clk":"AOkVAIMZALsQAKsQAO8M","pt":1478546671115,"oc":[{"id":"1.128126331","orc":[{"id":10895629,"uo":[{"id":"78996704480","p":1.02,"s":2,"side":"L","status":"E","pt":"L","ot":"L","pd":1478546670000,"cd":1478546670001,"sm":0,"sr":2,"sl":0,"sc":0,"sv":0,"rac":"","rc":"REG_GGC","rfo":"","rfs":""}]}]}]}
diff --git a/tests/unit/test_bettingresources.py b/tests/unit/test_bettingresources.py
@@ -8,6 +8,7 @@
     MarketOnCloseOrder,
     PriceSize,
 )
+from betfairlightweight.compat import BETFAIR_DATE_FORMAT
 from tests.unit.tools import create_mock_json
 
 
@@ -56,10 +57,10 @@ def test_time_range_result(self):
             assert resource.elapsed_time == self.ELAPSED_TIME
             assert resource.market_count == time_range["marketCount"]
             assert resource.time_range._from == datetime.datetime.strptime(
-                time_range["timeRange"]["from"], "%Y-%m-%dT%H:%M:%S.%fZ"
+                time_range["timeRange"]["from"], BETFAIR_DATE_FORMAT
             )
             assert resource.time_range.to == datetime.datetime.strptime(
-                time_range["timeRange"]["to"], "%Y-%m-%dT%H:%M:%S.%fZ"
+                time_range["timeRange"]["to"], BETFAIR_DATE_FORMAT
             )
 
     def test_event_result(self):
@@ -75,7 +76,7 @@ def test_event_result(self):
             assert resource.market_count == event_result["marketCount"]
             assert resource.event.id == event_result["event"]["id"]
             assert resource.event.open_date == datetime.datetime.strptime(
-                event_result["event"]["openDate"], "%Y-%m-%dT%H:%M:%S.%fZ"
+                event_result["event"]["openDate"], BETFAIR_DATE_FORMAT
             )
             assert resource.event.time_zone == event_result["event"]["timezone"]
             assert resource.event.country_code == event_result["event"]["countryCode"]
@@ -135,15 +136,15 @@ def test_market_catalogue(self):
             assert resource.market_name == market_catalogue["marketName"]
             assert resource.total_matched == market_catalogue["totalMatched"]
             assert resource.market_start_time == datetime.datetime.strptime(
-                market_catalogue["marketStartTime"], "%Y-%m-%dT%H:%M:%S.%fZ"
+                market_catalogue["marketStartTime"], BETFAIR_DATE_FORMAT
             )
 
             assert resource.competition.id == market_catalogue["competition"]["id"]
             assert resource.competition.name == market_catalogue["competition"]["name"]
 
             assert resource.event.id == market_catalogue["event"]["id"]
             assert resource.event.open_date == datetime.datetime.strptime(
-                market_catalogue["event"]["openDate"], "%Y-%m-%dT%H:%M:%S.%fZ"
+                market_catalogue["event"]["openDate"], BETFAIR_DATE_FORMAT
             )
             assert resource.event.time_zone == market_catalogue["event"]["timezone"]
             assert (
@@ -172,7 +173,7 @@ def test_market_catalogue(self):
                 == market_catalogue["description"]["marketBaseRate"]
             )
             assert resource.description.market_time == datetime.datetime.strptime(
-                market_catalogue["description"]["marketTime"], "%Y-%m-%dT%H:%M:%S.%fZ"
+                market_catalogue["description"]["marketTime"], BETFAIR_DATE_FORMAT
             )
             assert (
                 resource.description.market_type
@@ -194,7 +195,7 @@ def test_market_catalogue(self):
                 == market_catalogue["description"]["rulesHasDate"]
             )
             assert resource.description.suspend_time == datetime.datetime.strptime(
-                market_catalogue["description"]["suspendTime"], "%Y-%m-%dT%H:%M:%S.%fZ"
+                market_catalogue["description"]["suspendTime"], BETFAIR_DATE_FORMAT
             )
             assert (
                 resource.description.turn_in_play_enabled
@@ -263,7 +264,7 @@ def test_market_book(self):
             assert resource.inplay == market_book["inplay"]
             assert resource.is_market_data_delayed == market_book["isMarketDataDelayed"]
             assert resource.last_match_time == datetime.datetime.strptime(
-                market_book["lastMatchTime"], "%Y-%m-%dT%H:%M:%S.%fZ"
+                market_book["lastMatchTime"], BETFAIR_DATE_FORMAT
             )
             assert (
                 resource.number_of_active_runners
@@ -306,7 +307,7 @@ def test_market_book(self):
 
                 if runner.get("removalDate"):
                     assert resource_runner.removal_date == datetime.datetime.strptime(
-                        runner["removalDate"], "%Y-%m-%dT%H:%M:%S.%fZ"
+                        runner["removalDate"], BETFAIR_DATE_FORMAT
                     )
                 # else:
                 #     assert resource_runner.sp.near_price == runner['sp']['nearPrice']
@@ -430,7 +431,7 @@ def test_place_orders(self):
             assert resource.place_instruction_reports[
                 0
             ].placed_date == datetime.datetime.strptime(
-                order["placedDate"], "%Y-%m-%dT%H:%M:%S.%fZ"
+                order["placedDate"], BETFAIR_DATE_FORMAT
             )
             assert resource.place_instruction_reports[0].error_code == order.get(
                 "errorCode"
@@ -492,7 +493,7 @@ def test_cancel_orders(self):
             assert resource.cancel_instruction_reports[
                 0
             ].cancelled_date == datetime.datetime.strptime(
-                order["cancelledDate"], "%Y-%m-%dT%H:%M:%S.%fZ"
+                order["cancelledDate"], BETFAIR_DATE_FORMAT
             )
 
             assert (

diff --git a/tests/unit/test_cache.py b/tests/unit/test_cache.py
@@ -517,6 +517,7 @@ def setUp(self):
             "lsrc": 17,
             "error": "test",
             "md": 4,
+            "cd": 18,
         }
         self.unmatched_order = UnmatchedOrder(**order)
 
@@ -538,40 +539,36 @@ def test_init(self):
         assert self.unmatched_order.reference_strategy == 15
         assert self.unmatched_order.lapsed_date == BaseResource.strip_datetime(16)
         assert self.unmatched_order.lapse_status_reason_code == 17
-
-    def test_placed_date_string(self):
-        now = BaseResource.strip_datetime(8)
-        assert self.unmatched_order.placed_date_string == now.strftime(
-            "%Y-%m-%dT%H:%M:%S.%fZ"
-        )
-
-    def test_matched_date_string(self):
-        now = BaseResource.strip_datetime(4)
-        assert self.unmatched_order.matched_date_string == now.strftime(
-            "%Y-%m-%dT%H:%M:%S.%fZ"
-        )
+        assert self.unmatched_order.cancelled_date == BaseResource.strip_datetime(18)
 
     def test_serialise(self):
-        assert self.unmatched_order.serialise("1.23", 12345, 0.0) == {
-            "sizeLapsed": 11,
-            "persistenceType": "LAPSE",
-            "sizeRemaining": 10,
-            "placedDate": "1970-01-01T00:00:00.008000Z",
-            "sizeVoided": 13,
-            "sizeCancelled": 12,
-            "betId": 1,
-            "customerOrderRef": 14,
-            "orderType": "LIMIT",
-            "marketId": "1.23",
-            "side": "LAY",
-            "selectionId": 12345,
-            "bspLiability": None,
-            "sizeMatched": 9,
-            "handicap": 0.0,
-            "averagePriceMatched": 0.0,
-            "status": "EXECUTABLE",
-            "customerStrategyRef": 15,
-            "regulatorCode": None,
-            "priceSize": {"price": 2, "size": 3},
-            "matchedDate": "1970-01-01T00:00:00.004000Z",
-        }
+        self.assertEqual(
+            self.unmatched_order.serialise("1.23", 12345, 0.0),
+            {
+                "sizeLapsed": 11,
+                "persistenceType": "LAPSE",
+                "sizeRemaining": 10,
+                "placedDate": "1970-01-01T00:00:00.008000Z",
+                "sizeVoided": 13,
+                "sizeCancelled": 12,
+                "betId": 1,
+                "customerOrderRef": 14,
+                "orderType": "LIMIT",
+                "marketId": "1.23",
+                "side": "LAY",
+                "selectionId": 12345,
+                "bspLiability": None,
+                "sizeMatched": 9,
+                "handicap": 0.0,
+                "averagePriceMatched": 0.0,
+                "status": "EXECUTABLE",
+                "customerStrategyRef": 15,
+                "regulatorCode": None,
+                "regulatorAuthCode": None,
+                "priceSize": {"price": 2, "size": 3},
+                "matchedDate": "1970-01-01T00:00:00.004000Z",
+                "lapsedDate": "1970-01-01T00:00:00.016000Z",
+                "lapseStatusReasonCode": 17,
+                "cancelledDate": "1970-01-01T00:00:00.018000Z",
+            },
+        )
diff --git a/tests/unit/test_compat.py b/tests/unit/test_compat.py
@@ -5,6 +5,12 @@
 
 
 class CompatTest(unittest.TestCase):
+    def test_defaults(self):
+        self.assertEqual(compat.BETFAIR_DATE_FORMAT, "%Y-%m-%dT%H:%M:%S.%fZ")
+        self.assertEqual(compat.basestring, (str, bytes))
+        self.assertEqual(compat.numeric_types, (int, float))
+        self.assertEqual(compat.integer_types, (int,))
+
     def test_json_loads(self):
         self.assertEqual(
             compat.json.loads('{"lastTradedPrice": 1000}'), {"lastTradedPrice": 1000}

diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py
@@ -1,4 +1,5 @@
 import unittest
+import datetime
 from unittest import mock
 
 from betfairlightweight import utils
@@ -31,3 +32,10 @@ def test_convert_to_camel_case(self):
 
     def test_default_user_agent(self):
         assert utils.default_user_agent()
+
+    def test_create_date_string(self):
+        self.assertIsNone(utils.create_date_string(None))
+        self.assertEqual(
+            utils.create_date_string(datetime.datetime(2020, 11, 27)),
+            "2020-11-27T00:00:00.000000Z",
+        )