Fix byte strings (#138)

* Correct Power readings of Nan that are sent in reversed byte order

. an edge case where it appears that the solaredge inverter has a minor
  but irritating bug when encoding data for transmission

* Correct inverter Pmax power readings that should be Nan

. an edge case where it appears that the solaredge inverter has a minor
  but irritating bug when encoding data for transmission, namely that
  readings of Nan that are sent in reversed byte order

* Decode battery id to string instead of bytes.

. The battery ID in a meters_0x0030 message is 12 characters long.
  In Python 3.x struct.unpack returns a byte string when the 's'
  format is used. But byte strings are not accepted as keys for
  dictionaries, so it is necessary to decode the battery ID byte string
  to a proper unicode string to avoid problems later on.

* First attempt at providing a test for the fix_byte_strings changes.

. The 'correct' json file was created under Windows, so there may be
  problems with CR LF endings, when tests are run under a Linux shell.

* Fix value returned from parseData when unknown function encountered.

. The previous return value was raising an exception, even though the
  intent is just to ignore the unknown function part of the message.

* Convert Date and Time values in json output to the US/Pacific timezone.

. test/test.sh, as run by CircleCI, is setup to generate Date and Time
   values as per the US/Pacific timezone.  Hence my target output
  json file must also include Date and Time as per that timezone,
  in order to pass the CI tests.

* Add the .rec and .csv files expected by test/test.sh.

* Save csv and rec files created by running test/test.sh on Ubuntu

se/data.py

@@ -8,6 +8,7 @@
 import se.commands
 from se.dataparams import *
 from se.datadevices import ParseDevice, merge_update
+import codecs
 
 logger = logging.getLogger(__name__)
 
@@ -51,7 +52,7 @@ def parseData(function, data):
     else:
         # unknown function type
         logger.info("Unknown function 0x%04x", function)
-    return ''.join(x.encode('hex') for x in data)
+    return codecs.encode(data, 'hex').decode('ascii')
 
 def parseEnergyStats(data):
     (Eday, Emon, Eyear, Etot, Time1) = struct.unpack("<ffffL", data[0:20])
@@ -155,6 +156,11 @@ def parseDeviceData(data):
         elif seType == 0x0010:  # inverter data
             invDict[seId] = parseInvData(seId, invItems,
                                          data[dataPtr:dataPtr + devLen])
+            # Correct odd case where solaredge inverter sends Nan value in opposite byte order to all other float values
+            # ie solaredge sends b'\xff\xff\x7f\xff' which in little endian format unpacks as - 3.402... * 10 ** 38
+            # but b'\xff\x7f\xff\xff' unpacks as Nan, which is the "correct" value when this byte pattern is seen.
+            if invDict[seId]["Pmax"] < -3 * 10**38:
+                invDict[seId]["Pmax"] = float('nan')
             logDevice("inverter:     ", seType, seId, devLen, invDict[seId])
         elif seType == 0x0011:  # 3 phase inverter data
             invDict[seId] = parseInv3PhData(seId, inv3PhItems,

se/datadevices.py

@@ -227,11 +227,17 @@ def parseDevTable(self, data):
             # I suspect a legacy "bug" somewhere in the solaredge messages, but in the meantime just check the bytes
             # and fix it.
             elif paramInFmt == 'f' and (
-                    data[dataPtr:dataPtr + paramLen] == '\xff\xff\x7f\xff'):
+                    data[dataPtr:dataPtr + paramLen] == b'\xff\xff\x7f\xff'):
                 self[paramName] = float('nan')
             else:
                 self[paramName] = struct.unpack(
                     '<' + paramInFmt, data[dataPtr:dataPtr + paramLen])[0]
+                if 's' in paramInFmt:
+                    # Python3.x requires that we convert byte string to unicode string
+                    # if the value is to be used (later on) as a dictionary key for nested fields
+                    # Required for eg for battery Ids
+                    # Remove trailing 'null' character (ie 0x00) at the same time
+                    self[paramName] = self[paramName].decode('utf-8').strip('\x00')
 
             # Optionally format the field
             if outFormatFn == 'dateTime':