Skip to content

AVRO-1928: Simplify Python float/double encoding #132

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
shalabhc wants to merge 1 commit into from
Closed

AVRO-1928: Simplify Python float/double encoding #132

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 6 additions & 32 deletions lang/py/src/avro/io.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,8 @@ def unpack(self, *args):
return struct.unpack(self.format, *args)
struct_class = SimpleStruct

STRUCT_INT = struct_class('!I') # big-endian unsigned int
STRUCT_LONG = struct_class('!Q') # big-endian unsigned long long
STRUCT_FLOAT = struct_class('!f') # big-endian float
STRUCT_DOUBLE = struct_class('!d') # big-endian double
STRUCT_FLOAT = struct_class('<f') # little-endian float
STRUCT_DOUBLE = struct_class('<d') # little-endian double
STRUCT_CRC32 = struct_class('>I') # big-endian unsigned int

#
Expand Down Expand Up @@ -197,27 +195,15 @@ def read_float(self):
The float is converted into a 32-bit integer using a method equivalent to
Java's floatToIntBits and then encoded in little-endian format.
"""
bits = (((ord(self.read(1)) & 0xffL)) |
((ord(self.read(1)) & 0xffL) << 8) |
((ord(self.read(1)) & 0xffL) << 16) |
((ord(self.read(1)) & 0xffL) << 24))
return STRUCT_FLOAT.unpack(STRUCT_INT.pack(bits))[0]
return STRUCT_FLOAT.unpack(self.read(4))[0]

def read_double(self):
"""
A double is written as 8 bytes.
The double is converted into a 64-bit integer using a method equivalent to
Java's doubleToLongBits and then encoded in little-endian format.
"""
bits = (((ord(self.read(1)) & 0xffL)) |
((ord(self.read(1)) & 0xffL) << 8) |
((ord(self.read(1)) & 0xffL) << 16) |
((ord(self.read(1)) & 0xffL) << 24) |
((ord(self.read(1)) & 0xffL) << 32) |
((ord(self.read(1)) & 0xffL) << 40) |
((ord(self.read(1)) & 0xffL) << 48) |
((ord(self.read(1)) & 0xffL) << 56))
return STRUCT_DOUBLE.unpack(STRUCT_LONG.pack(bits))[0]
return STRUCT_DOUBLE.unpack(self.read(8))[0]

def read_bytes(self):
"""
Expand Down Expand Up @@ -319,27 +305,15 @@ def write_float(self, datum):
The float is converted into a 32-bit integer using a method equivalent to
Java's floatToIntBits and then encoded in little-endian format.
"""
bits = STRUCT_INT.unpack(STRUCT_FLOAT.pack(datum))[0]
self.write(chr((bits) & 0xFF))
self.write(chr((bits >> 8) & 0xFF))
self.write(chr((bits >> 16) & 0xFF))
self.write(chr((bits >> 24) & 0xFF))
self.write(STRUCT_FLOAT.pack(datum))

def write_double(self, datum):
"""
A double is written as 8 bytes.
The double is converted into a 64-bit integer using a method equivalent to
Java's doubleToLongBits and then encoded in little-endian format.
"""
bits = STRUCT_LONG.unpack(STRUCT_DOUBLE.pack(datum))[0]
self.write(chr((bits) & 0xFF))
self.write(chr((bits >> 8) & 0xFF))
self.write(chr((bits >> 16) & 0xFF))
self.write(chr((bits >> 24) & 0xFF))
self.write(chr((bits >> 32) & 0xFF))
self.write(chr((bits >> 40) & 0xFF))
self.write(chr((bits >> 48) & 0xFF))
self.write(chr((bits >> 56) & 0xFF))
self.write(STRUCT_DOUBLE.pack(datum))

def write_bytes(self, datum):
"""
Expand Down
37 changes: 6 additions & 31 deletions lang/py3/avro/io.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,8 @@
LONG_MAX_VALUE = (1 << 63) - 1

STRUCT_INT = struct.Struct('!I') # big-endian unsigned int
STRUCT_LONG = struct.Struct('!Q') # big-endian unsigned long long
STRUCT_FLOAT = struct.Struct('!f') # big-endian float
STRUCT_DOUBLE = struct.Struct('!d') # big-endian double
STRUCT_FLOAT = struct.Struct('<f') # little-endian float
STRUCT_DOUBLE = struct.Struct('<d') # little-endian double
STRUCT_CRC32 = struct.Struct('>I') # big-endian unsigned int


Expand Down Expand Up @@ -211,27 +210,15 @@ def read_float(self):
The float is converted into a 32-bit integer using a method equivalent to
Java's floatToIntBits and then encoded in little-endian format.
"""
bits = (((ord(self.read(1)) & 0xff)) |
((ord(self.read(1)) & 0xff) << 8) |
((ord(self.read(1)) & 0xff) << 16) |
((ord(self.read(1)) & 0xff) << 24))
return STRUCT_FLOAT.unpack(STRUCT_INT.pack(bits))[0]
return STRUCT_FLOAT.unpack(self.read(4))[0]

def read_double(self):
"""
A double is written as 8 bytes.
The double is converted into a 64-bit integer using a method equivalent to
Java's doubleToLongBits and then encoded in little-endian format.
"""
bits = (((ord(self.read(1)) & 0xff)) |
((ord(self.read(1)) & 0xff) << 8) |
((ord(self.read(1)) & 0xff) << 16) |
((ord(self.read(1)) & 0xff) << 24) |
((ord(self.read(1)) & 0xff) << 32) |
((ord(self.read(1)) & 0xff) << 40) |
((ord(self.read(1)) & 0xff) << 48) |
((ord(self.read(1)) & 0xff) << 56))
return STRUCT_DOUBLE.unpack(STRUCT_LONG.pack(bits))[0]
return STRUCT_DOUBLE.unpack(self.read(8))[0]

def read_bytes(self):
"""
Expand Down Expand Up @@ -354,27 +341,15 @@ def write_float(self, datum):
The float is converted into a 32-bit integer using a method equivalent to
Java's floatToIntBits and then encoded in little-endian format.
"""
bits = STRUCT_INT.unpack(STRUCT_FLOAT.pack(datum))[0]
self.WriteByte((bits) & 0xFF)
self.WriteByte((bits >> 8) & 0xFF)
self.WriteByte((bits >> 16) & 0xFF)
self.WriteByte((bits >> 24) & 0xFF)
self.write(STRUCT_FLOAT.pack(datum))

def write_double(self, datum):
"""
A double is written as 8 bytes.
The double is converted into a 64-bit integer using a method equivalent to
Java's doubleToLongBits and then encoded in little-endian format.
"""
bits = STRUCT_LONG.unpack(STRUCT_DOUBLE.pack(datum))[0]
self.WriteByte((bits) & 0xFF)
self.WriteByte((bits >> 8) & 0xFF)
self.WriteByte((bits >> 16) & 0xFF)
self.WriteByte((bits >> 24) & 0xFF)
self.WriteByte((bits >> 32) & 0xFF)
self.WriteByte((bits >> 40) & 0xFF)
self.WriteByte((bits >> 48) & 0xFF)
self.WriteByte((bits >> 56) & 0xFF)
self.write(STRUCT_DOUBLE.pack(datum))

def write_bytes(self, datum):
"""
Expand Down