From 9e9421966ef2fa4fb8c0c003648934998335b0ba Mon Sep 17 00:00:00 2001 From: George Hopkins Date: Sat, 7 Jan 2017 21:21:53 +0100 Subject: [PATCH 1/3] Implement error-correcting OP defragmenter --- README.md | 6 +++ protocols.py | 131 +++++++++++++++++++++++++++++++++++++---------- requirements.txt | 2 + 3 files changed, 112 insertions(+), 27 deletions(-) create mode 100644 requirements.txt diff --git a/README.md b/README.md index 07e0145..5404c21 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,12 @@ Things that are not implemented/supported yet: automatically. This is easy to implement, but it is low on my priority list, as it is very easy to do this manually. +## Dependencies + +To install all dependencies just run: `pip install -r requirements.txt` + + * [zfec](https://pypi.python.org/pypi/zfec) + ## Sample KISS files You can use some [sample KISS files](https://drive.google.com/open?id=0B2pPGQkeEAfdbXFZNThCb1BLMzg) for testing. diff --git a/protocols.py b/protocols.py index 619d214..29ba439 100644 --- a/protocols.py +++ b/protocols.py @@ -26,6 +26,7 @@ import struct +import zfec class OP: """ @@ -45,15 +46,81 @@ def __init__(self, data): Throws ValueError if packet is malformed """ - + header = data[:self.__header_len] if len(header) < self.__header_len: raise ValueError('Malformed OP packet: too short') - self.length, self.fragmentation, self.carousel_id, \ - self.last_fragment, self.fragment_number = struct.unpack('>HBBBB', header) + self.length, self.fragment_type, self.carousel_id, \ + self.last_fragment, self.fragment_index = struct.unpack('>HBBBB', header) self.payload = data[self.__header_len : self.__header_len + self.length - 4] +class PartialLDP: + """ + Fragmented LDP packet + + A LDP packet which has not yet been completely received. + """ + def __init__(self): + self.reset() + + """ + Reset the internal state + """ + def reset(self): + self.__fragments = {} + self.__frag_recv = 0 + self.__fec_recv = 0 + self.frag_size = None + self.frag_count = None + self.fec_count = None + self.next_index = 0 + + """ + Push a data block + + Args: + index (int): the index of the fragment + payload (bytes): the actual data + """ + def push_data(self, index, payload): + if index in self.__fragments: + return + self.__fragments[index] = payload + self.__frag_recv += 1 + + """ + Push a FEC block + + Args: + index (int): the index of the FEC block + payload (bytes): the actual FEC data + """ + def push_fec(self, index, payload): + if not self.frag_count or (self.frag_count + index) in self.__fragments: + return + self.__fragments[self.frag_count + index] = payload + self.__fec_recv += 1 + + """ + Indicates whether a reconstruction is possible + """ + @property + def complete(self): + return self.frag_count and self.__fec_recv + self.__frag_recv >= self.frag_count + + """ + Decode the packet + """ + def decode(self): + if self.__frag_recv == self.frag_count: # No error FEC decoding necessary + return b''.join([self.__fragments[s] for s in range(self.frag_count)]) + k = self.frag_count + n = k + self.fec_count + decoder = zfec.Decoder(k, n) + sharenums = list(self.__fragments.keys()) + return b''.join(decoder.decode([self.__fragments[s] for s in sharenums], sharenums)) + class OPDefragmenter: """ OP defragmenter @@ -66,40 +133,50 @@ def __init__(self): """ Initialize defragmenter """ - self.__payload = bytes() - self.__last_fragment = -1 - self.__previous_fragment = -1 + self.__pending = {} def push(self, packet): """ Push new packet into defragmenter - Returns an the palyoad (bytes) if defragmentation is succesful, + Returns a payload (bytes) if defragmentation is succesful, None otherwise Args: packet (OP): Packet to push """ - if packet.fragment_number != self.__previous_fragment + 1: - # packet lost - self.__init__() - - if packet.fragment_number == 0: - # first fragment - self.__init__() - self.__last_fragment = packet.last_fragment - - if packet.last_fragment == self.__last_fragment and \ - packet.fragment_number == self.__previous_fragment + 1: - # fragment ok - self.__payload = self.__payload + packet.payload - self.__previous_fragment = packet.fragment_number - - if self.__payload and self.__previous_fragment == self.__last_fragment: - # packet complete - payload = self.__payload - self.__init__() - return payload + ldp = self.__pending.get(packet.carousel_id) + if not ldp: + ldp = PartialLDP() + self.__pending[packet.carousel_id] = ldp + + if packet.fragment_type == 0x3c or packet.fragment_type == 0xc3: + if packet.fragment_type == 0x3c and packet.fragment_index == 0: # TODO Verify correctness + return packet.payload + if packet.fragment_index < ldp.next_index: + ldp.reset() + if not ldp.frag_size: + ldp.frag_size = packet.length - 4 + if not ldp.frag_count: + ldp.frag_count = packet.last_fragment + 1 + ldp.next_index = packet.fragment_index + 1 + ldp.push_data(packet.fragment_index, packet.payload + b'\xff' * (ldp.frag_size - len(packet.payload))) + if packet.fragment_type == 0x3c and ldp.complete: + decoded = ldp.decode() + ldp.reset() + return decoded + elif packet.fragment_type == 0x69: + if not ldp.frag_size: + return + if not ldp.fec_count: + ldp.fec_count = packet.last_fragment + 1 + ldp.push_fec(packet.fragment_index, packet.payload) + if ldp.complete: + decoded = ldp.decode() + ldp.reset() + return decoded + else: + print('Unsupported fragment type: {:02x}'.format(packet.fragment_type)) class LDP: """ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..74ecf6d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/george-hopkins/pyutil@python3#egg=pyutil +git+https://github.com/george-hopkins/zfec@python3#egg=zfec From 22f3e2a3574729868a6d26abf7b6a4c25df31a5c Mon Sep 17 00:00:00 2001 From: George Hopkins Date: Sat, 7 Jan 2017 21:30:57 +0100 Subject: [PATCH 2/3] Remove trailing spaces --- files.py | 14 +++++++------- free-outernet.py | 10 +++++----- kiss.py | 12 ++++++------ protocols.py | 10 +++++----- timeservice.py | 6 +++--- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/files.py b/files.py index 6c78c81..6309459 100644 --- a/files.py +++ b/files.py @@ -1,15 +1,15 @@ # Copyright 2016 Daniel Estevez . -# +# # This is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 3, or (at your option) # any later version. -# +# # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License # along with this software; see the file COPYING. If not, write to # the Free Software Foundation, Inc., 51 Franklin Street, @@ -44,7 +44,7 @@ class FileService: using the File() class """ __block_header_len = 6 - + def __init__(self, router, files_path): """ Initialize file service handler @@ -143,7 +143,7 @@ def __try_reconstruct(self, file_id): out.close() del self.__files[file_id] print('[File service] File reconstructed: {}'.format(f.path)) - + class File: """ @@ -194,7 +194,7 @@ def push_fec(self, block, n): if self.__fec_blocks[n]: raise ValueError('File.push_fec(): FEC block already received!') self.__fec_blocks[n] = block - + def reconstruct(self): """ Try to reconstruct the file @@ -213,7 +213,7 @@ def reconstruct(self): fec = bytes().join(self.__fec_blocks) print('Length of FEC data: {} bytes; File size: {} bytes'.format(len(fec), self.size)) print('--------------------------------------------------------------------') - + if None in self.__blocks: print('Some blocks are missing. Cannot reconstruct file {}'.format(self.path)) return diff --git a/free-outernet.py b/free-outernet.py index c0f2360..588d35d 100755 --- a/free-outernet.py +++ b/free-outernet.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # # Copyright 2016 Daniel Estevez . -# +# # This is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 3, or (at your option) @@ -11,7 +11,7 @@ # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License # along with this software; see the file COPYING. If not, write to # the Free Software Foundation, Inc., 51 Franklin Street, @@ -44,7 +44,7 @@ def printMac(mac): return ('%02x:'* 5 + '%02x') % struct.unpack('B'*6, mac) - + def printEthertype(ethertype): return hex(struct.unpack('>H', ethertype)[0]) @@ -105,7 +105,7 @@ def usage(): \t-p, --port=PORT\t\tUDP port to listen (default {}) \t --host=HOST\t\tUDP host to listen (default ::, use 0.0.0.0 for IPv4 only) '''.format(UDP_PORT)) - + def main(): try: @@ -146,7 +146,7 @@ def main(): timeservice.TimeService(router) files.FileService(router, output) - + if kissinput: kissFile = open(kissinput, 'rb') kissDeframer = kiss.KISSDeframer() diff --git a/kiss.py b/kiss.py index 66f99df..5eb68f1 100644 --- a/kiss.py +++ b/kiss.py @@ -1,15 +1,15 @@ # Copyright 2016 Daniel Estevez . -# +# # This is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 3, or (at your option) # any later version. -# +# # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License # along with this software; see the file COPYING. If not, write to # the Free Software Foundation, Inc., 51 Franklin Street, @@ -33,7 +33,7 @@ class KISSDeframer(): __FESC = 0xdb __TFEND = 0xdc __TFESC = 0xdd - + def __init__(self): """ Initialize KISS deframer @@ -53,9 +53,9 @@ def push(self, data): data (bytes): the chunk of bytes to push """ pdus = list() - + self.__kiss.extend(data) - + while self.__kiss: c = self.__kiss.popleft() if c == self.__FEND: diff --git a/protocols.py b/protocols.py index 29ba439..ba3a2e5 100644 --- a/protocols.py +++ b/protocols.py @@ -1,20 +1,20 @@ # Copyright 2016 Daniel Estevez . -# +# # This is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 3, or (at your option) # any later version. -# +# # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License # along with this software; see the file COPYING. If not, write to # the Free Software Foundation, Inc., 51 Franklin Street, # Boston, MA 02110-1301, USA. -# +# """Outernet OP and LDP protocols""" @@ -223,7 +223,7 @@ def __init__(self): Create a new LDP router """ self.__registrations = dict() - + def route(self, packet): """ Push an LDP packet into the router, calling the appropriate diff --git a/timeservice.py b/timeservice.py index bd339a6..b562f63 100644 --- a/timeservice.py +++ b/timeservice.py @@ -1,15 +1,15 @@ # Copyright 2016 Daniel Estevez . -# +# # This is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 3, or (at your option) # any later version. -# +# # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License # along with this software; see the file COPYING. If not, write to # the Free Software Foundation, Inc., 51 Franklin Street, From f4d641179209dd50b97e4fd4877e0da75d87cc63 Mon Sep 17 00:00:00 2001 From: George Hopkins Date: Sat, 7 Jan 2017 23:34:00 +0100 Subject: [PATCH 3/3] Validate LDP packet checksum --- README.md | 1 + protocols.py | 7 +++++-- requirements.txt | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5404c21..0c06ffe 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ Things that are not implemented/supported yet: To install all dependencies just run: `pip install -r requirements.txt` + * [crcmod](https://pypi.python.org/pypi/crcmod) * [zfec](https://pypi.python.org/pypi/zfec) ## Sample KISS files diff --git a/protocols.py b/protocols.py index ba3a2e5..076b13a 100644 --- a/protocols.py +++ b/protocols.py @@ -27,6 +27,7 @@ import struct import zfec +from crcmod.predefined import PredefinedCrc class OP: """ @@ -205,8 +206,10 @@ def __init__(self, data): if self.length > len(data): raise ValueError('Malformed LDP packet: invalid length') - self.checksum = data[self.length-self.__checksum_len:self.length] - # TODO implement checksum handling + crc = PredefinedCrc('crc-32-mpeg') + crc.update(data[:self.length]) + if crc.crcValue != 0: + raise ValueError('Malformed LDP packet: invalid checksum') self.payload = data[self.__header_len:self.length-self.__checksum_len] diff --git a/requirements.txt b/requirements.txt index 74ecf6d..d142992 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ +crcmod>=1.7,<2 git+https://github.com/george-hopkins/pyutil@python3#egg=pyutil git+https://github.com/george-hopkins/zfec@python3#egg=zfec