Merge pull request #6 from vfrazao-ns1/develop/0.0.5/Allow-TOPS-1.5

Develop/0.0.5/allow tops 1.5
lvfrazao · Apr 29, 2019 · 6058305 · 6058305
2 parents 9c6b986 + a24e707
commit 6058305
Show file tree

Hide file tree

Showing 5 changed files with 95 additions and 72 deletions.
diff --git a/IEXTools/IEXparser.py b/IEXTools/IEXparser.py
@@ -53,17 +53,24 @@ class Parser(object):
     """
 
     def __init__(
-        self, file_path: str, tops: bool = True, deep: bool = False
+        self, file_path: str, tops: bool = True, deep: bool = False, tops_version: float = 1.6
     ) -> None:
         self.file_path = file_path
         self.tops = tops
         self.deep = deep
         self.file = self._load(file_path)
         # IEX TP Header Structure
+        # Many of these byte strings are hardcoded and may cause compatibility
+        # issues with future or previous versions of TOPS, DEEP, or the EIX
+        # Transport Protocol
         self.version = b"\x01"
         self.reserved = b"\x00"
         if tops and not deep:
-            self.protocol_id = b"\x03\x80"
+            protcol_ids = {
+                1.5: b"\x02\x80",
+                1.6: b"\x03\x80",
+            }
+            self.protocol_id = protcol_ids[tops_version]
         elif deep:
             self.protocol_id = b"\x04\x80"
             raise NotImplementedError("Parsing of DEEP files not implemented")
@@ -94,7 +101,7 @@ def __init__(
             messages.QuoteUpdate: b"\x51",
         }
 
-        self.decoder = messages.MessageDecoder()
+        self.decoder = messages.MessageDecoder(version=tops_version)
 
     def __repr__(self) -> str:
         return f'Parser("{self.file_path}", tops={self.tops}, deep={self.deep})'

diff --git a/IEXTools/messages.py b/IEXTools/messages.py
@@ -36,7 +36,7 @@
 
 
 class MessageDecoder(object):
-    def __init__(self) -> None:
+    def __init__(self, version: float = 1.6) -> None:
         """
         Some notes on data types used in decoding IEX messages:
         B: unsigned byte
@@ -45,70 +45,91 @@ def __init__(self) -> None:
         s: string (size denoted by preceding number)
         q: signed long long (8 bytes)
         """
-        self.message_types: Dict[bytes, Dict[str, Union[str, AllMessages]]] = {
-            b"\x53": {
-                "str": "System Event Message",
-                "cls": SystemEvent,
-                "fmt": "<Bq",
+        self.message_types: Dict[
+            float, Dict[bytes, Dict[str, Union[str, AllMessages]]]
+        ] = {
+            1.6: {
+                b"\x53": {
+                    "str": "System Event Message",
+                    "cls": SystemEvent,
+                    "fmt": "<Bq",
+                },
+                b"\x44": {
+                    "str": "Security Directory Message",
+                    "cls": SecurityDirective,
+                    "fmt": "<Bq8sLqB",
+                },
+                b"\x48": {
+                    "str": "Trading Status Message",
+                    "cls": TradingStatus,
+                    "fmt": "<1sq8s4s",
+                },
+                b"\x4f": {
+                    "str": "Operational Halt Status Message",
+                    "cls": OperationalHalt,
+                    "fmt": "<1sq8s",
+                },
+                b"\x50": {
+                    "str": "Short Sale Price Test Status Message",
+                    "cls": ShortSalePriceSale,
+                    "fmt": "<Bq8s1s",
+                },
+                b"\x51": {
+                    "str": "Quote Update Message",
+                    "cls": QuoteUpdate,
+                    "fmt": "<Bq8sLqqL",
+                },
+                b"\x54": {
+                    "str": "Trade Report Message",
+                    "cls": TradeReport,
+                    "fmt": "<Bq8sLqq",
+                },
+                b"\x58": {
+                    "str": "Official Price Message",
+                    "cls": OfficialPrice,
+                    "fmt": "<1sq8sq",
+                },
+                b"\x42": {
+                    "str": "Trade Break Message",
+                    "cls": TradeBreak,
+                    "fmt": "<1sq8sqq",
+                },
+                b"\x41": {
+                    "str": "Auction Information Message",
+                    "cls": AuctionInformation,
+                    "fmt": "<1sq8sLqqL1sBLqqqq",
+                },
             },
-            b"\x44": {
-                "str": "Security Directory Message",
-                "cls": SecurityDirective,
-                "fmt": "<Bq8sLqB",
-            },
-            b"\x48": {
-                "str": "Trading Status Message",
-                "cls": TradingStatus,
-                "fmt": "<1sq8s4s",
-            },
-            b"\x4f": {
-                "str": "Operational Halt Status Message",
-                "cls": OperationalHalt,
-                "fmt": "<1sq8s",
-            },
-            b"\x50": {
-                "str": "Short Sale Price Test Status Message",
-                "cls": ShortSalePriceSale,
-                "fmt": "<Bq8s1s",
-            },
-            b"\x51": {
-                "str": "Quote Update Message",
-                "cls": QuoteUpdate,
-                "fmt": "<Bq8sLqqL",
-            },
-            b"\x54": {
-                "str": "Trade Report Message",
-                "cls": TradeReport,
-                "fmt": "<Bq8sLqq",
-            },
-            b"\x58": {
-                "str": "Official Price Message",
-                "cls": OfficialPrice,
-                "fmt": "<1sq8sq",
-            },
-            b"\x42": {
-                "str": "Trade Break Message",
-                "cls": TradeBreak,
-                "fmt": "<1sq8sqq",
-            },
-            b"\x41": {
-                "str": "Auction Information Message",
-                "cls": AuctionInformation,
-                "fmt": "<1sq8sLqqL1sBLqqqq",
+            1.5: {
+                b"\x51": {
+                    "str": "Quote Update Message",
+                    "cls": QuoteUpdate,
+                    "fmt": "<Bq8sLqqL",
+                },
+                b"\x54": {
+                    "str": "Trade Report Message",
+                    "cls": TradeReport,
+                    "fmt": "<Bq8sLqqxxxx",
+                },
+                b"\x42": {
+                    "str": "Trade Break Message",
+                    "cls": TradeBreak,
+                    "fmt": "<1sq8sqqxxxx",
+                },
             },
         }
         self.DECODE_FMT: Dict[int, str] = {
-            msg[0]: self.message_types[msg]["fmt"] for msg in self.message_types
+            msg[0]: self.message_types[version][msg]["fmt"] for msg in self.message_types[version]
         }
         self.MSG_CLS: Dict[int, Type[AllMessages]] = {
-            msg[0]: self.message_types[msg]["cls"] for msg in self.message_types
+            msg[0]: self.message_types[version][msg]["cls"] for msg in self.message_types[version]
         }
 
     def decode_message(self, msg_type: int, binary_msg: bytes) -> AllMessages:
         try:
             fmt = self.DECODE_FMT[msg_type]
         except KeyError as e:
-            raise ProtocolException(f'Unknown message type: {e.args}')
+            raise ProtocolException(f"Unknown message type: {e.args}")
         decoded_msg = struct.unpack(fmt, binary_msg)
         msg = self.MSG_CLS[msg_type](*decoded_msg)
         return msg
@@ -134,11 +155,7 @@ def __post_init__(self):
         for attrib in str_fields:
             if hasattr(self, attrib):
                 if isinstance(getattr(self, attrib), bytes):
-                    setattr(
-                        self,
-                        attrib,
-                        getattr(self, attrib).decode("utf-8").strip(),
-                    )
+                    setattr(self, attrib, getattr(self, attrib).decode("utf-8").strip())
 
         int_prices = [p for p in self.__slots__ if "price" in p and "int" in p]
 
@@ -212,13 +229,7 @@ class TradingStatus(Message):
         o MCB2: Market-Wide Circuit Breaker Level 2 Breached
     """
 
-    __slots__ = (
-        "status",
-        "timestamp",
-        "symbol",
-        "reason",
-        "trading_status_message",
-    )
+    __slots__ = ("status", "timestamp", "symbol", "reason", "trading_status_message")
     status: str  # 1 byte
     timestamp: int  # 8 bytes, nanosecond epoch time
     symbol: str  # 8 bytes

diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # IEXTools
 
-v 0.0.4
+v 0.0.5
 
 This package provides tools for working with data provided by IEX's REST API and tools to decode and use IEX's binary market data (dubbed "HIST"). For more information on the type of data offered by IEX please visit their website: <https://iextrading.com/developer/docs> and <https://iextrading.com/trading/market-data/>
 
@@ -14,7 +14,7 @@ This package is under active development and may be subject to regular breaking
 
 ## Executive Summary
 
-The Investors Exchange (IEX) was founded in 2012 by Brad Katsuyama to combat the effect that high frequency trading was having on other stock exchanges. The story of IEX was made famous by John Lewis in his book, _Fast Boys_.
+The Investors Exchange (IEX) was founded in 2012 by Brad Katsuyama to combat the effect that high frequency trading was having on other stock exchanges. The story of IEX was made famous by Michael Lewis in his book, _Fast Boys_.
 
 This package aims to provide a variety of tools for working with stock data provided by IEX such as:
 
@@ -155,7 +155,7 @@ Usage:
 
 Purpose: Parse the binary PCAP / HIST files offered by IEX.
 
-To create a Parser object simply supply the file path as an argument.
+To create a Parser object simply supply the file path as an argument. Please note that if using a version 1.5 TOPS file then the `tops_version` parameter must be set to `1.5` on instantiation.
 
 ```Python
 >>> from IEXTools import Parser, messages
@@ -275,6 +275,11 @@ By not specifying the `allowed` argument the parser returns 1,000,000 parsed mes
 
 - Added documentation (via Sphinx)
 
+### 0.0.5
+
+- Added support for version 1.5 TOPS files
+- Updated urllib3 version in requirements.txt due to vulnerability
+
 ### Future Focus
 
 - Additional testing

diff --git a/requirements.txt b/requirements.txt
diff --git a/setup.py b/setup.py
@@ -15,7 +15,7 @@
 EMAIL = ''
 AUTHOR = 'Victor Frazao'
 REQUIRES_PYTHON = '>=3.7.0'
-VERSION = '0.0.4'
+VERSION = '0.0.5'
 
 # What packages are required for this module to be executed?
 REQUIRED = ['requests']