Implement error-correcting OP defragmenter

README.md

@@ -46,6 +46,13 @@ Things that are not implemented/supported yet:
    automatically. This is easy to implement, but it is low on my priority list,
    as it is very easy to do this manually.
 
+## Dependencies
+
+To install all dependencies just run: `pip install -r requirements.txt`
+
+ * [crcmod](https://pypi.python.org/pypi/crcmod)
+ * [zfec](https://pypi.python.org/pypi/zfec)
+
 ## Sample KISS files
 
 You can use some [sample KISS files](https://drive.google.com/open?id=0B2pPGQkeEAfdbXFZNThCb1BLMzg) for testing.

files.py

@@ -1,15 +1,15 @@
 # Copyright 2016 Daniel Estevez <daniel@destevez.net>.
-# 
+#
 # This is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3, or (at your option)
 # any later version.
-# 
+#
 # This software is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
 # along with this software; see the file COPYING.  If not, write to
 # the Free Software Foundation, Inc., 51 Franklin Street,
@@ -44,7 +44,7 @@ class FileService:
     using the File() class
     """
     __block_header_len = 6
-    
+
     def __init__(self, router, files_path):
         """
         Initialize file service handler
@@ -143,7 +143,7 @@ def __try_reconstruct(self, file_id):
         out.close()
         del self.__files[file_id]
         print('[File service] File reconstructed: {}'.format(f.path))
-        
+
 
 class File:
     """
@@ -194,7 +194,7 @@ def push_fec(self, block, n):
         if self.__fec_blocks[n]:
             raise ValueError('File.push_fec(): FEC block already received!')
         self.__fec_blocks[n] = block
-    
+
     def reconstruct(self):
         """
         Try to reconstruct the file
@@ -213,7 +213,7 @@ def reconstruct(self):
                 fec = bytes().join(self.__fec_blocks)
                 print('Length of FEC data: {} bytes; File size: {} bytes'.format(len(fec), self.size))
             print('--------------------------------------------------------------------')
-        
+
         if None in self.__blocks:
             print('Some blocks are missing. Cannot reconstruct file {}'.format(self.path))
             return

free-outernet.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 #
 # Copyright 2016 Daniel Estevez <daniel@destevez.net>.
-# 
+#
 # This is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3, or (at your option)
@@ -11,7 +11,7 @@
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
 # along with this software; see the file COPYING.  If not, write to
 # the Free Software Foundation, Inc., 51 Franklin Street,
@@ -44,7 +44,7 @@
 
 def printMac(mac):
     return ('%02x:'* 5 + '%02x') % struct.unpack('B'*6, mac)
-    
+
 def printEthertype(ethertype):
     return hex(struct.unpack('>H', ethertype)[0])
 
@@ -105,7 +105,7 @@ def usage():
 \t-p, --port=PORT\t\tUDP port to listen (default {})
 \t    --host=HOST\t\tUDP host to listen (default ::, use 0.0.0.0 for IPv4 only)
 '''.format(UDP_PORT))
-    
+
 
 def main():
     try:
@@ -146,7 +146,7 @@ def main():
 
     timeservice.TimeService(router)
     files.FileService(router, output)
-    
+
     if kissinput:
         kissFile = open(kissinput, 'rb')
         kissDeframer = kiss.KISSDeframer()

kiss.py

@@ -1,15 +1,15 @@
 # Copyright 2016 Daniel Estevez <daniel@destevez.net>.
-# 
+#
 # This is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3, or (at your option)
 # any later version.
-# 
+#
 # This software is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
 # along with this software; see the file COPYING.  If not, write to
 # the Free Software Foundation, Inc., 51 Franklin Street,
@@ -33,7 +33,7 @@ class KISSDeframer():
     __FESC = 0xdb
     __TFEND = 0xdc
     __TFESC = 0xdd
-    
+
     def __init__(self):
         """
         Initialize KISS deframer
@@ -53,9 +53,9 @@ def push(self, data):
           data (bytes): the chunk of bytes to push
         """
         pdus = list()
-        
+
         self.__kiss.extend(data)
-        
+
         while self.__kiss:
             c = self.__kiss.popleft()
             if c == self.__FEND:

protocols.py

@@ -1,20 +1,20 @@
 # Copyright 2016 Daniel Estevez <daniel@destevez.net>.
-# 
+#
 # This is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3, or (at your option)
 # any later version.
-# 
+#
 # This software is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
 # along with this software; see the file COPYING.  If not, write to
 # the Free Software Foundation, Inc., 51 Franklin Street,
 # Boston, MA 02110-1301, USA.
-# 
+#
 
 """Outernet OP and LDP protocols"""
 
@@ -26,6 +26,8 @@
 
 
 import struct
+import zfec
+from crcmod.predefined import PredefinedCrc
 
 class OP:
     """
@@ -45,15 +47,81 @@ def __init__(self, data):
 
         Throws ValueError if packet is malformed
         """
-        
+
         header = data[:self.__header_len]
         if len(header) < self.__header_len:
             raise ValueError('Malformed OP packet: too short')
 
-        self.length, self.fragmentation, self.carousel_id, \
-          self.last_fragment, self.fragment_number = struct.unpack('>HBBBB', header)
+        self.length, self.fragment_type, self.carousel_id, \
+          self.last_fragment, self.fragment_index = struct.unpack('>HBBBB', header)
         self.payload = data[self.__header_len : self.__header_len + self.length - 4]
 
+class PartialLDP:
+    """
+    Fragmented LDP packet
+
+    A LDP packet which has not yet been completely received.
+    """
+    def __init__(self):
+        self.reset()
+
+    """
+    Reset the internal state
+    """
+    def reset(self):
+        self.__fragments = {}
+        self.__frag_recv = 0
+        self.__fec_recv = 0
+        self.frag_size = None
+        self.frag_count = None
+        self.fec_count = None
+        self.next_index = 0
+
+    """
+    Push a data block
+
+    Args:
+      index (int): the index of the fragment
+      payload (bytes): the actual data
+    """
+    def push_data(self, index, payload):
+        if index in self.__fragments:
+            return
+        self.__fragments[index] = payload
+        self.__frag_recv += 1
+
+    """
+    Push a FEC block
+
+    Args:
+      index (int): the index of the FEC block
+      payload (bytes): the actual FEC data
+    """
+    def push_fec(self, index, payload):
+        if not self.frag_count or (self.frag_count + index) in self.__fragments:
+            return
+        self.__fragments[self.frag_count + index] = payload
+        self.__fec_recv += 1
+
+    """
+    Indicates whether a reconstruction is possible
+    """
+    @property
+    def complete(self):
+        return self.frag_count and self.__fec_recv + self.__frag_recv >= self.frag_count
+
+    """
+    Decode the packet
+    """
+    def decode(self):
+        if self.__frag_recv == self.frag_count: # No error FEC decoding necessary
+            return b''.join([self.__fragments[s] for s in range(self.frag_count)])
+        k = self.frag_count
+        n = k + self.fec_count
+        decoder = zfec.Decoder(k, n)
+        sharenums = list(self.__fragments.keys())
+        return b''.join(decoder.decode([self.__fragments[s] for s in sharenums], sharenums))
+
 class OPDefragmenter:
     """
     OP defragmenter
@@ -66,40 +134,50 @@ def __init__(self):
         """
         Initialize defragmenter
         """
-        self.__payload = bytes()
-        self.__last_fragment = -1
-        self.__previous_fragment = -1
+        self.__pending = {}
 
     def push(self, packet):
         """
         Push new packet into defragmenter
 
-        Returns an the palyoad (bytes) if defragmentation is succesful,
+        Returns a payload (bytes) if defragmentation is succesful,
         None otherwise
 
         Args:
           packet (OP): Packet to push
         """
-        if packet.fragment_number != self.__previous_fragment + 1:
-            # packet lost
-            self.__init__()
-
-        if packet.fragment_number == 0:
-            # first fragment
-            self.__init__()
-            self.__last_fragment = packet.last_fragment
-
-        if packet.last_fragment == self.__last_fragment and \
-            packet.fragment_number == self.__previous_fragment + 1:
-            # fragment ok
-            self.__payload = self.__payload + packet.payload
-            self.__previous_fragment = packet.fragment_number
-
-        if self.__payload and self.__previous_fragment == self.__last_fragment:
-            # packet complete
-            payload = self.__payload
-            self.__init__()
-            return payload
+        ldp = self.__pending.get(packet.carousel_id)
+        if not ldp:
+            ldp = PartialLDP()
+            self.__pending[packet.carousel_id] = ldp
+
+        if packet.fragment_type == 0x3c or packet.fragment_type == 0xc3:
+            if packet.fragment_type == 0x3c and packet.fragment_index == 0: # TODO Verify correctness
+                return packet.payload
+            if packet.fragment_index < ldp.next_index:
+                ldp.reset()
+            if not ldp.frag_size:
+                ldp.frag_size = packet.length - 4
+            if not ldp.frag_count:
+                ldp.frag_count = packet.last_fragment + 1
+            ldp.next_index = packet.fragment_index + 1
+            ldp.push_data(packet.fragment_index, packet.payload + b'\xff' * (ldp.frag_size - len(packet.payload)))
+            if packet.fragment_type == 0x3c and ldp.complete:
+                decoded = ldp.decode()
+                ldp.reset()
+                return decoded
+        elif packet.fragment_type == 0x69:
+            if not ldp.frag_size:
+                return
+            if not ldp.fec_count:
+                ldp.fec_count = packet.last_fragment + 1
+            ldp.push_fec(packet.fragment_index, packet.payload)
+            if ldp.complete:
+                decoded = ldp.decode()
+                ldp.reset()
+                return decoded
+        else:
+            print('Unsupported fragment type: {:02x}'.format(packet.fragment_type))
 
 class LDP:
     """
@@ -128,8 +206,10 @@ def __init__(self, data):
         if self.length > len(data):
             raise ValueError('Malformed LDP packet: invalid length')
 
-        self.checksum = data[self.length-self.__checksum_len:self.length]
-        # TODO implement checksum handling
+        crc = PredefinedCrc('crc-32-mpeg')
+        crc.update(data[:self.length])
+        if crc.crcValue != 0:
+            raise ValueError('Malformed LDP packet: invalid checksum')
 
         self.payload = data[self.__header_len:self.length-self.__checksum_len]
 
@@ -146,7 +226,7 @@ def __init__(self):
         Create a new LDP router
         """
         self.__registrations = dict()
-        
+
     def route(self, packet):
         """
         Push an LDP packet into the router, calling the appropriate

requirements.txt

@@ -0,0 +1,3 @@
+crcmod>=1.7,<2
+git+https://github.com/george-hopkins/pyutil@python3#egg=pyutil
+git+https://github.com/george-hopkins/zfec@python3#egg=zfec

timeservice.py

@@ -1,15 +1,15 @@
 # Copyright 2016 Daniel Estevez <daniel@destevez.net>.
-# 
+#
 # This is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3, or (at your option)
 # any later version.
-# 
+#
 # This software is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
 # along with this software; see the file COPYING.  If not, write to
 # the Free Software Foundation, Inc., 51 Franklin Street,