From 08bd8b54c2a20ba00fa42c70de42b266d783e0b0 Mon Sep 17 00:00:00 2001
From: Eric Buehl <715650+ericbuehl@users.noreply.github.com>
Date: Tue, 4 Jun 2019 11:23:44 -0700
Subject: [PATCH] initial support for python3 (#133)

---
 conversion/common.py |  2 +-
 requirements.txt     |  1 +
 se/data.py           |  4 +--
 se/datadevices.py    |  6 ++--
 se/env.py            |  2 +-
 se/files.py          |  7 +++--
 se/logutils.py       |  2 +-
 se/msg.py            | 74 ++++++++++++++++++++++----------------------
 semonitor.py         | 30 ++++++++++--------
 9 files changed, 68 insertions(+), 60 deletions(-)

diff --git a/conversion/common.py b/conversion/common.py
index 3227c2c..dde9b56 100644
--- a/conversion/common.py
+++ b/conversion/common.py
@@ -20,7 +20,7 @@ def nice(k):
         return str(k).replace("\x00", '').replace(" ", "_").replace(
             "devices", "")
 
-    for k, v in mydict.iteritems():
+    for k, v in mydict.items():
         if "Date" in v.keys():
             yield nice(k), v
         else:
diff --git a/requirements.txt b/requirements.txt
index 496c02b..12893e9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
+future==0.17.1
 netifaces==0.10.6
 pycrypto==2.6.1
 pyserial==3.4
diff --git a/se/data.py b/se/data.py
index 97e93f0..9a94469 100644
--- a/se/data.py
+++ b/se/data.py
@@ -251,7 +251,7 @@ def writeData(msgDict, outFile):
         msg = json.dumps(msgDict, sort_keys=True)
         logger.message("<--", outSeq, msg, outFile.name)
         logger.data(msg)
-        outFile.write(msg + "\n")
+        outFile.write(msg.encode('latin-1') + b"\n")
         outFile.flush()
 
 # remove the extra bit that is sometimes set in a device ID and upcase the letters
@@ -277,5 +277,5 @@ def formatDateTime(timeStamp):
 # formatted print of device data
 def logDevice(devType, seType, seId, devLen, devData):
     logger.data("%s %s type: %04x len: %04x", devType, seId, seType, devLen)
-    for k,v in devData.iteritems():
+    for k,v in devData.items():
         logger.data("    %s : %s", k, v)
diff --git a/se/datadevices.py b/se/datadevices.py
index 4782d4d..ffffec5 100644
--- a/se/datadevices.py
+++ b/se/datadevices.py
@@ -151,10 +151,10 @@ def hexData(data):
         lineSize = 16
 
         def hexLine(data):
-            return ' '.join(x.encode('hex') for x in data)
+            return ' '.join("{:02x}".format(x) for x in data)
 
         hexLines = []
-        if data != "":
+        if data:
             printPtr = 0
             while len(data) - printPtr >= lineSize:
                 hexLines.append(hexLine(data[printPtr:printPtr + lineSize]))
@@ -770,7 +770,7 @@ def merge_update(dict1, dict2):
     :param dict2: A new subsidiary nested dictionary, to be merged into dict1.
     :return: None, but the master dictionary is updated with **only** the new parts of dict2 merged in.
     """
-    for k2, v2 in dict2.iteritems():
+    for k2, v2 in dict2.items():
         if k2 not in dict1.keys():
             dict1.update({k2: v2})
         elif isinstance(v2, dict):
diff --git a/se/env.py b/se/env.py
index 08ee7d7..6bfa9fc 100644
--- a/se/env.py
+++ b/se/env.py
@@ -63,7 +63,7 @@ def validated_ports(ports_str):
     parser.add_argument("-c", dest="commands", type=validated_commands, default=[], help="send the specified command functions")
     parser.add_argument("-d", dest="logfile", default="stderr", help="where to write log messages.  either a file name or one of ['stderr', 'syslog']")
     parser.add_argument("-f", dest="follow", action="store_true", default=False, help="wait for appended data as the input file grows (as in tail -f)")
-    parser.add_argument("-k", dest="keyfile", type=argparse.FileType('r'), help="file containing a hex encoded encryption key")
+    parser.add_argument("-k", dest="keyfile", type=argparse.FileType('rb'), help="file containing a hex encoded encryption key")
     parser.add_argument("-m", dest="master", action="store_true", default=False, help="function as a RS485 master")
     parser.add_argument("-n", dest="interface", type=netifaces.ifaddresses, help="run DHCP and DNS network services on the specified interface")
     parser.add_argument("-o", dest="outfile", default="stdout", help="write performance data to the specified file in JSON format (default: stdout)")
diff --git a/se/files.py b/se/files.py
index ee06bea..643f01d 100644
--- a/se/files.py
+++ b/se/files.py
@@ -54,7 +54,10 @@ def openSerial(inFileName, baudRate):
 
 def openInFile(inFileName):
     if inFileName == "stdin":
-        return sys.stdin
+        if sys.version_info >= (3,0):
+            return sys.stdin.buffer
+        else:
+            return sys.stdin
     else:
         # Explicitly specify mode rb to keep windows happy!
         return open(inFileName, 'rb')
@@ -67,7 +70,7 @@ def closeData(dataFile, networkDevice):
     dataFile.close()
 
 # open in output file if it is specified
-def openOutFile(fileName, writeMode="w"):
+def openOutFile(fileName, writeMode="wb"):
     if fileName:
         return open(fileName, writeMode)
 
diff --git a/se/logutils.py b/se/logutils.py
index 9139e1e..c3413b2 100644
--- a/se/logutils.py
+++ b/se/logutils.py
@@ -35,7 +35,7 @@ def _message_log(self, direction, seq, msg, endPoint=""):
 def format_data(data):
     line_width = 16
     for i in range(0, len(data), line_width):
-        yield "data:       " + ' '.join(x.encode('hex') for x in data[i:i+line_width])
+        yield "data:       " + ' '.join("{:02x}".format(x) for x in data[i:i+line_width])
 
 logging.Logger.data = _data_log
 logging.Logger.raw = _raw_log
diff --git a/se/msg.py b/se/msg.py
index 4aa44ef..1768128 100644
--- a/se/msg.py
+++ b/se/msg.py
@@ -7,6 +7,7 @@
 import datetime
 import se.logutils
 import binascii
+from builtins import bytes
 from Crypto.Cipher import AES
 from Crypto.Random import random
 
@@ -30,32 +31,32 @@ def __init__(self, key, msg0503):
         curtime = datetime.datetime.now()
         mystrtime = curtime.strftime("%Y-%m-%d %H:%M:%S")
         # Create a key by encrypting the data from Solar Edge with our key)
-        enkey1 = map(ord, AES.new(key).encrypt(msg0503[:16]))
+        enkey1 = bytes(AES.new(key).encrypt(msg0503[:16]))
         # Store the 0503 message in a hex string
         hex_msg0503 = binascii.hexlify(msg0503)
         # Format the line in the last0503.msg file
         # Format is: String Timestamp (for us humans),Epoch in seconds (for easy math),hex encoded previous message
-        outstr = mystrtime + "," + str(int(time.time())) + "," + hex_msg0503
+        outstr = mystrtime + "," + str(int(time.time())) + "," + str(hex_msg0503)
         # Write the outstr to the last0503.msg file, clobbering the previous (hence 'w' write mode)
         ko = open(LAST0503FILE, "w")
         ko.write(outstr)
         ko.close
         # self.cipher is an AES object
-        self.cipher = AES.new("".join(
-            map(chr, (enkey1[i] ^ ord(msg0503[i + 16]) for i in range(16)))))
+        self.cipher = AES.new(bytes(list((enkey1[i] ^ msg0503[i + 16] for i in range(16)))))
         self.encrypt_seq = random.randint(0, 0xffff)
 
     def crypt(self, msg003d):
         """
-        msg003d: the contents of the 003d message to crypt, as list(int).
+        msg003d: the contents of the 003d message to crypt, as bytes.
 
-        Modifies the list in-place and returns it.
+        Returns the new list
         """
-        rand1 = msg003d[:16]
+        msg003d = list(msg003d)
+        rand1 = list(msg003d[:16])
         pos = 16
         while pos < len(msg003d):
             if not pos % 16:
-                rand = map(ord, self.cipher.encrypt("".join(map(chr, rand1))))
+                rand = bytes(self.cipher.encrypt(bytes(rand1)))
                 for posc in range(15, -1, -1):
                     rand1[posc] = (rand1[posc] + 1) & 0xff
                     if rand1[posc]:
@@ -68,13 +69,12 @@ def decrypt(self, msg003d):
         """
         msg003d: the contents of the 003d message to decrypt, as string.
 
-        Returns a tuple(int(sequenceNumber), str(data)).
+        Returns a tuple(int(sequenceNumber), bytes(data)).
         """
-        msg003d = self.crypt(map(ord, msg003d))
+        msg003d = self.crypt(msg003d)
         for i in range(len(msg003d) - 22):
             msg003d[i + 22] ^= msg003d[18 + (i & 3)]
-        return (msg003d[16] + (msg003d[17] << 8),
-                "".join(map(chr, msg003d[22:])))
+        return (msg003d[16] + (msg003d[17] << 8), bytes(msg003d[22:]))
 
     def encrypt(self, msg):
         """
@@ -86,17 +86,17 @@ def encrypt(self, msg):
         rand1 = [random.randint(0, 255) for x in range(16)]
         rand2 = [random.randint(0, 255) for x in range(4)]
         seqnr = [self.encrypt_seq & 0xff, self.encrypt_seq >> 8 & 0xff]
-        msg003d = rand1 + seqnr + rand2 + map(ord, msg)
+        msg003d = rand1 + seqnr + rand2 + msg
         for i in range(len(msg)):
             msg003d[i + 22] ^= msg003d[18 + (i & 3)]
-        return "".join(map(chr, self.crypt(msg003d)))
+        return self.crypt(msg003d)
 
 # cryptography object variable and global indicator if the load of last0503.msg was attempted
 cipher = None
 bcipher = False
 
 # message constants
-magic = "\x12\x34\x56\x79"
+magic = b"\x12\x34\x56\x79"
 magicLen = len(magic)
 msgHdrLen = 16
 checksumLen = 2
@@ -109,13 +109,13 @@ def encrypt(self, msg):
 def readMsg(inFile, recFile, mode):
     global dataInSeq
     dataInSeq += 1
-    msg = ""
+    msg = b""
     eof = False
     if not (mode.passiveMode or (mode.serialType == 4)):
         # active mode that is not rs485
         # read the magic number and header
         msg = readBytes(inFile, recFile, magicLen + msgHdrLen, mode)
-        if msg == "":   # end of file
+        if not msg:   # end of file
             return (msg, True)
         (dataLen, dataLenInv, msgSeq, fromAddr, toAddr, function) = \
             struct.unpack("<HHHLLH", msg[magicLen:])
@@ -127,7 +127,7 @@ def readMsg(inFile, recFile, mode):
         # read 1 byte at a time until the next magic number
         while msg[-magicLen:] != magic:
             nextByte = readBytes(inFile, recFile, 1, mode)
-            if nextByte == "":  # end of file
+            if not nextByte:  # end of file
                 eof = True
                 msg += magic  # append a magic number to end the loop
             else:
@@ -140,11 +140,11 @@ def readMsg(inFile, recFile, mode):
 # return the specified number of bytes
 def readBytes(inFile, recFile, length, mode):
     try:
-        inBuf = inFile.read(length)
-        if inBuf == "":  # end of file
+        inBuf = bytes(inFile.read(length))
+        if not inBuf:  # end of file
             if mode.following:
                 # wait for more data
-                while inBuf == "":
+                while not inBuf:
                     time.sleep(sleepInterval)
                     inBuf = inFile.read(length)
         recordMsg(inBuf, recFile)
@@ -152,7 +152,7 @@ def readBytes(inFile, recFile, length, mode):
     # treat exceptions as end of file
     except Exception as ex:
         logger.info("Exception while reading data: "+str(ex))
-        return ""
+        return b""
 
 # A function to attempt to load the the rotating key from the last0503.msg file
 def loadRotKey(keyStr):
@@ -167,10 +167,10 @@ def loadRotKey(keyStr):
         logger.data("Could not open last0503.msg file, not loading")
 
     # If the try block doing the file operation was successful, mydata will now not be blank, so we try to operate on it. 
-    if mydata != "":
+    if mydata:
         # These are the variables where we keep the last load time and the last key data
         lastsave = 0
-        lastkey = ""
+        lastkey = b""
         # We attempt to load the file and split the contents. If this try fails, we assume the file to have been corrupted (manual edit etc) and we give up, not loading the key
         try:
             lastsave = int(mydata.split(",")[1])
@@ -178,7 +178,7 @@ def loadRotKey(keyStr):
         except:
             logger.data("last0503.msg not in proper format - not loading")
         # If we have both a last load time and data in our variables, we assume the try block succeeded and we do some more validation
-        if lastsave != 0 and lastkey != "":
+        if lastsave != 0 and lastkey:
             # First we check the time, if the last load time is more than 24 hours ago, we give up. No point in trying decryption with an old key
             curtime = int(time.time())
             if curtime - lastsave <= 86400: #86400 seconds is 24 hours 60 secs * 60 min * 24 hours
@@ -187,7 +187,7 @@ def loadRotKey(keyStr):
                     logger.data("Attempting to load data from last0503.msg")
                     # Our last try if this fails, we log an unknown error
                     try:
-                        cipher = SECrypto(keyStr.decode("hex"), binascii.unhexlify(lastkey))
+                        cipher = SECrypto(binascii.unhexlify(keyStr), binascii.unhexlify(lastkey))
                         logger.data("Rotated key from last0503.msg loaded successfully!")
                     except:
                         logger.data("Unknown error in loading rotating key. Not using")
@@ -213,15 +213,15 @@ def parseMsg(msg, keyStr=""):
         bcipher = loadRotKey(keyStr)
     if len(msg) < msgHdrLen + checksumLen:  # throw out messages that are too short
         logger.data("Threw out a message that was too short")
-        return (0, 0, 0, 0, "")
+        return (0, 0, 0, 0, b"")
     else:
         (msgSeq, fromAddr, toAddr, function, data) = validateMsg(msg)
         # encryption key
         if function == 0x0503:
             if keyStr:
                 logger.data("Creating cipher object with key", keyStr)
-                cipher = SECrypto(keyStr.decode("hex"), data)
-            return (msgSeq, fromAddr, toAddr, function, "")
+                cipher = SECrypto(binascii.unhexlify(keyStr), data)
+            return (msgSeq, fromAddr, toAddr, function, b"")
         # encrypted message
         elif function == 0x003d:
             if cipher:
@@ -231,12 +231,12 @@ def parseMsg(msg, keyStr=""):
                 if dataMsg[0:4] != magic:
                     logger.data("Invalid decryption key - Clearing Cipher")
                     cipher = None
-                    return (0, 0, 0, 0, "")
+                    return (0, 0, 0, 0, b"")
                 else:
                     (msgSeq, fromAddr, toAddr, function, data) = validateMsg(dataMsg[4:])
             else:  # don't have a key yet
                 logger.data("Decryption key not yet available")
-                return (0, 0, 0, 0, "")
+                return (0, 0, 0, 0, b"")
         return (msgSeq, fromAddr, toAddr, function, data)
 
 # parse the header and validate the message
@@ -246,7 +246,7 @@ def validateMsg(msg):
         logger.error("Message too short")
         for l in se.logutils.format_data(msg):
             logger.data(l)
-        return (0, 0, 0, 0, "")
+        return (0, 0, 0, 0, b"")
     # parse the message header
     (dataLen, dataLenInv, msgSeq, fromAddr, toAddr, function) = struct.unpack("<HHHLLH", msg[0:msgHdrLen])
     logMsgHdr(dataLen, dataLenInv, msgSeq, fromAddr, toAddr, function)
@@ -255,13 +255,13 @@ def validateMsg(msg):
         logger.error("Data length is too big for the message")
         for l in se.logutils.format_data(msg):
             logger.data(l)
-        return (0, 0, 0, 0, "")
+        return (0, 0, 0, 0, b"")
     # data length must match inverse length
     if dataLen != ~dataLenInv & 0xffff:
         logger.error("Data length doesn't match inverse length")
         for l in se.logutils.format_data(msg):
             logger.data(l)
-        return (0, 0, 0, 0, "")
+        return (0, 0, 0, 0, b"")
     data = msg[msgHdrLen:msgHdrLen + dataLen]
     # discard extra bytes after the message
     extraLen = len(msg) - (msgHdrLen + dataLen + checksumLen)
@@ -277,11 +277,11 @@ def validateMsg(msg):
         logger.error("Checksum error. Expected 0x%04x, got 0x%04x" % (checksum, calcsum))
         for l in se.logutils.format_data(msg):
             logger.data(l)
-        return (0, 0, 0, 0, "")
+        return (0, 0, 0, 0, b"")
     return (msgSeq, fromAddr, toAddr, function, data)
 
 # format a message
-def formatMsg(msgSeq, fromAddr, toAddr, function, data="", encrypt=True):
+def formatMsg(msgSeq, fromAddr, toAddr, function, data=b"", encrypt=True):
     checksum = calcCrc(struct.pack(">HLLH", msgSeq, fromAddr, toAddr, function) + data)
     msg = struct.pack("<HHHLLH", len(data), ~len(data) & 0xffff, msgSeq,
                       fromAddr, toAddr, function) + data + struct.pack("<H", checksum)
@@ -350,7 +350,7 @@ def recordMsg(msg, recFile):
 def calcCrc(data):
     crc = 0x5a5a  # initial value
     for d in data:
-        crc = crcTable[(crc ^ ord(d)) & 0xff] ^ (crc >> 8)
+        crc = crcTable[(crc ^ d) & 0xff] ^ (crc >> 8)
     return crc
 
 # formatted print a message header
diff --git a/semonitor.py b/semonitor.py
index 886a54e..a7c2d29 100755
--- a/semonitor.py
+++ b/semonitor.py
@@ -14,6 +14,7 @@
 import se.commands
 import se.network
 import logging
+from builtins import bytes
 
 logger = logging.getLogger(__name__)
 
@@ -32,7 +33,7 @@
 masterEvent = threading.Event()  # event to signal RS485 master release
 
 # program termination
-def terminate(code=0, msg=""):
+def terminate(code=0, msg=b""):
     if code == 0:
         logger.info(msg)
     else:
@@ -42,7 +43,7 @@ def terminate(code=0, msg=""):
 # process the input data
 def readData(args, mode, dataFile, recFile, outFile, keyStr):
     eof = False
-    updateBuf = list('\x00' * UPDATE_SIZE) if args.updatefile else []
+    updateBuf = list(b"\x00" * UPDATE_SIZE) if args.updatefile else []
     if mode.passiveMode:
         # skip data until the start of the first complete message
         (msg, eof) = se.msg.readMsg(dataFile, recFile, mode)  
@@ -55,7 +56,7 @@ def readData(args, mode, dataFile, recFile, outFile, keyStr):
                 se.files.closeData(dataFile, True)
                 dataFile = se.files.openDataSocket(args.ports)
                 eof = False
-        if msg == "\x00" * len(msg):  # ignore messages containing all zeros
+        if msg == b"\x00" * len(msg):  # ignore messages containing all zeros
             logger.data(msg)
         else:
             with threadLock:
@@ -89,15 +90,15 @@ def processMsg(msg, args, mode, dataFile, recFile, outFile, keyStr, updateBuf):
         elif updateBuf and function == se.commands.PROT_CMD_UPGRADE_WRITE:  # firmware update data
             updateBuf[msgData["offset"]:msgData["offset"] + msgData["length"]] = msgData["data"]
         if mode.networkDevice or mode.masterMode:  # send reply
-            replyFunction = ""
+            replyFunction = b""
             if function == se.commands.PROT_CMD_SERVER_POST_DATA:  # performance data
                 # send ack
                 replyFunction = se.commands.PROT_RESP_ACK
-                replyData = ""
+                replyData = b""
             elif function == 0x0503:  # encryption key
                 # send ack
                 replyFunction = se.commands.PROT_RESP_ACK
-                replyData = ""
+                replyData = b""
             elif function == se.commands.PROT_CMD_SERVER_GET_GMT:  # time request
                 # set time
                 replyFunction = se.commands.PROT_RESP_SERVER_GMT
@@ -105,15 +106,15 @@ def processMsg(msg, args, mode, dataFile, recFile, outFile, keyStr, updateBuf):
                     (time.localtime().tm_hour - time.gmtime().tm_hour) * 60 * 60)
             elif function == se.commands.PROT_RESP_POLESTAR_MASTER_GRANT_ACK:  # RS485 master release
                 masterEvent.set()
-            if replyFunction != "":
+            if replyFunction:
                 msg = se.msg.formatMsg(msgSeq, toAddr, fromAddr, replyFunction, replyData)
                 se.msg.sendMsg(dataFile, msg, recFile)
 
 # write firmware image to file
 def writeUpdate(updateBuf, updateFileName):
-    updateBuf = "".join(updateBuf)
+    updateBuf = b"".join(updateBuf)
     logger.info("writing %s", updateFileName)
-    with open(updateFileName, "w") as updateFile:
+    with open(updateFileName, "wb") as updateFile:
         updateFile.write(updateBuf)
 
 # RS485 master commands thread
@@ -224,14 +225,17 @@ def nextSeq():
     else:
         dataFile =  se.files.openInFile(args.datasource)
     # get encryption key
-    keyStr = args.keyfile.read().rstrip("\n") if args.keyfile else None
+    keyStr = args.keyfile.read().rstrip(b"\n") if args.keyfile else None
 
     # open the output files
-    recFile = se.files.openOutFile(args.record, "a" if args.append else "w")
+    recFile = se.files.openOutFile(args.record, "ab" if args.append else "wb")
     if args.outfile == "stdout":
-        outFile = sys.stdout
+        if sys.version_info >= (3,0):
+            outFile = sys.stdout.buffer
+        else:
+            outFile = sys.stdout
     else:
-        outFile = se.files.openOutFile(args.outfile, "a" if args.append else "w")
+        outFile = se.files.openOutFile(args.outfile, "ab" if args.append else "wb")
 
     # figure out what to do based on the mode of operation
     if mode.passiveMode:  # only reading from file or serial device