Trivial changes from Libo Song

Miscellaneous format improvements, a few debug messages which are now
commented out. I think that's all.

pcap.py

@@ -13,15 +13,15 @@ class TCPFlowAccumulator:
     connections, or flows. It does this by organizing packets into a
     dictionary indexed by their socket (the tuple
     ((srcip, sport), (dstip,dport)), possibly the other way around).
-    
+
     Members:
     flowdict = {socket: tcp.Flow}, the list of tcp.Flow's organized by socket
     '''
     def __init__(self, pcap_reader):
         '''
         scans the pcap_reader for TCP packets, and adds them to the tcp.Flow
         they belong to, based on their socket
-        
+
         Args:
         pcap_reader = pcaputil.ModifiedReader
         errors = list of errors encountered during parsing pcap.
@@ -48,12 +48,19 @@ def __init__(self, pcap_reader):
                         if isinstance(ip.data, dpkt.tcp.TCP):
                             # then it's a TCP packet
                             # process it
-                            tcppkt = tcp.Packet(pkt[0], pkt[1], eth, ip, ip.data)
+                            tcppkt = tcp.Packet(pkt[0], pkt[1], eth, ip,
+                                                ip.data)
+##                            print ("LSONG_DEBUG ",
+##                                   __file__, tcppkt.seq_start, tcppkt.seq_end,
+##                                   tcppkt.seq_end - tcppkt.seq_start,
+##                                   len(tcppkt.tcp.data))
                             self.process_packet(tcppkt) # organize by socket
+                    # TODO(lsong): UDP packet for DNS lookup.
                 except dpkt.Error as e:
                     self.errors.append((pkt, e, debug_pkt_count))
         except dpkt.dpkt.NeedData as e:
-            log.warning('A packet in the pcap file was too short, debug_pkt_count=%d' % debug_pkt_count)
+            log.warning('A packet in the pcap file was too short, '
+                        'debug_pkt_count=%d' % debug_pkt_count)
             self.errors.append((None, e))
         # finish all tcp flows
         map(tcp.Flow.finish, self.flowdict.itervalues())
@@ -67,6 +74,13 @@ def process_packet(self, pkt):
         src, dst = pkt.socket
         #ok, NOW add it
         #print 'processing packet: ', pkt
+        srcip, srcport = src  # LSONG
+        dstip, dstport = dst  # LSONG
+        if (srcport == 5223 or dstport == 5223):  # LSONG
+            # hpvirtgrp  #LSONG
+            print "%s LSONG_DEBUG: hpvirtgrp" %(__file__)  # LSONG
+            return  # LSONG
+
         if (src, dst) in self.flowdict:
             #print '  adding as ', (src, dst)
             self.flowdict[(src,dst)].add(pkt)
@@ -161,4 +175,4 @@ def WriteTCPFlowsFromFile(filename):
     os.mkdir(output_dir)
     # write out data
     for i, f in enumerate(flows.flowdict.itervalues()):
-        f.writeout_data(os.path.join(output_dir, str(i)))
+        f.writeout_data(os.path.join(output_dir, str(i)))

tcp/__init__.py

@@ -29,10 +29,14 @@ def detect_handshake(packets):
     if syn.tcp.flags & dpkt.tcp.TH_SYN and not syn.tcp.flags & dpkt.tcp.TH_ACK:
         # have syn
         fwd_seq = syn.seq # start_seq is the seq field of the segment
-        if synack.flags & dpkt.tcp.TH_SYN and synack.flags & dpkt.tcp.TH_ACK and synack.ack == fwd_seq + 1:
+        if (synack.flags & dpkt.tcp.TH_SYN and
+            synack.flags & dpkt.tcp.TH_ACK and
+            synack.ack == fwd_seq + 1):
             # have synack
             rev_seq = synack.seq
-            if ack.flags & dpkt.tcp.TH_ACK and ack.ack == rev_seq + 1 and ack.seq == fwd_seq + 1:
+            if (ack.flags & dpkt.tcp.TH_ACK and
+                ack.ack == rev_seq + 1 and
+                ack.seq == fwd_seq + 1):
                 # have ack
                 return True
-    return False
+    return False

tcp/chunk.py

@@ -37,10 +37,12 @@ def merge(self, new, new_seq_callback = None):
         Note that (True, (False, False)) is a valid value, which indicates that
         the new data was completely inside the existing data
         '''
-        if new.data: # if we have actual data yet (maybe false if there was no init packet)
+        # if we have actual data yet (maybe false if there was no init packet)
+        if new.data:
             # assume self.seq_* are also valid
             if self.data:
-                return self.inner_merge((new.seq_start, new.seq_end), new.data, new_seq_callback)
+                return self.inner_merge((new.seq_start, new.seq_end),
+                                        new.data, new_seq_callback)
             else:
                 # if they have data and we don't, just steal theirs
                 self.data = new.data
@@ -71,13 +73,16 @@ def inner_merge(self, newseq, newdata, callback):
         added_front_data = False
         added_back_data = False
         # front data?
-        if seq.lt(newseq[0], self.seq_start) and seq.lte(self.seq_start, newseq[1]):
+        if (seq.lt(newseq[0], self.seq_start) and
+            seq.lte(self.seq_start, newseq[1])):
             new_data_length = seq.subtract(self.seq[0], newseq[0])
-            self.data = newdata[:new_data_length] + self.data # slice out new data, stick it on the front
+            # slice out new data, stick it on the front
+            self.data = newdata[:new_data_length] + self.data
             self.seq_start = newseq[0]
             # notifications
             overlapped = True
             added_front_data = True
+            #print ("LSONG_DEBGU ", __file__, "overlapped, front_data")
             if callback:
                 callback(newseq[0])
         # back data?
@@ -88,11 +93,16 @@ def inner_merge(self, newseq, newdata, callback):
             # notifications
             overlapped = True
             added_back_data = True
+            #print ("LSONG_DEBGU ", __file__, "overlapped, back_data")
             if callback:
-                back_seq_start = newseq[1] - new_data_length # the first seq number of new data in the back
+                # the first seq number of new data in the back
+                back_seq_start = newseq[1] - new_data_length
                 callback(back_seq_start)
         # completely inside?
-        if seq.lte(self.seq_start, newseq[0]) and seq.lte(newseq[1], self.seq_end):
+        if (seq.lte(self.seq_start, newseq[0]) and
+            seq.lte(newseq[1], self.seq_end)):
             overlapped = True
+            #print ("LSONG_DEBGU ", __file__, "overlapped, inside")
+            # Nothing to do with the data?
         # done
         return (overlapped, (added_front_data, added_back_data))

tcp/direction.py

@@ -4,7 +4,7 @@
 class Direction:
     '''
     Represents data moving in one direction in a TCP flow.
-    
+
     Members:
     * chunks = [tcp.Chunk], sorted by seq_start
     * flow = tcp.Flow, the flow to which the direction belongs
@@ -15,7 +15,7 @@ class Direction:
     def __init__(self, flow):
         '''
         Sets things up for adding packets.
-        
+
         Args:
         flow = tcp.Flow
         '''
@@ -24,14 +24,16 @@ def __init__(self, flow):
         self.closed_cleanly = False # until proven true
         self.chunks = []
         self.flow = flow
-        self.seq_start= None # the seq number of the first byte of data, valid after finish() if self.data is valid
+        # the seq number of the first byte of data,
+        # valid after finish() if self.data is valid
+        self.seq_start= None
     def add(self, pkt):
         '''
         Merge the packet into the first chunk it overlaps with. If data was
         added to the end of a chunk, attempts to merge the next chunk (if there
         is one). This way, it is ensured that everything is as fully merged as
         it can be with the current data.
-        
+
         Args:
         pkt = tcp.Packet
         '''
@@ -42,15 +44,17 @@ def add(self, pkt):
         merged = False
         for i in range(len(self.chunks)):
             chunk = self.chunks[i]
-            overlapped, result = chunk.merge(pkt, self.create_merge_callback(pkt))
+            overlapped, result = chunk.merge(pkt,
+                                             self.create_merge_callback(pkt))
             if overlapped: # if the data overlapped
                 # if data was added on the back and there is a chunk after this
                 if result[1] and i < (len(self.chunks)-1):
                     # try to merge with the next chunk as well
                     # in case that packet bridged the gap
                     overlapped2, result2 = chunk.merge(self.chunks[i+1])
                     if overlapped2: # if that merge worked
-                        assert( (not result2[0]) and (result2[1])) # data should only be added to back
+                        # data should only be added to back
+                        assert( (not result2[0]) and (result2[1]))
                         del self.chunks[i+1] # remove the now-redundant chunk
                 merged = True
                 break # skip further chunks
@@ -81,16 +85,20 @@ def calculate_final_arrivals(self):
         '''
         self.final_arrival_data = []
         peak_time = 0.0
-        for vertex in self.arrival_data: # final arrival vertex always coincides with arrival vertex
+        # final arrival vertex always coincides with an arrival vertex
+        for vertex in self.arrival_data:
             if vertex[1].ts > peak_time:
                 peak_time = vertex[1].ts
                 self.final_arrival_data.append((vertex[0], vertex[1].ts))
-        self.final_arrival_data = SortedCollection(self.final_arrival_data, key=lambda v: v[0])
+        self.final_arrival_data = SortedCollection(
+            self.final_arrival_data,
+            key=lambda v: v[0]
+        )
 
     def new_chunk(self, pkt):
         '''
-        creates a new tcp.Chunk for the pkt to live in. Only called if an attempt
-        has been made to merge the packet with all existing chunks.
+        creates a new tcp.Chunk for the pkt to live in. Only called if an
+        attempt has been made to merge the packet with all existing chunks.
         '''
         chunk = tcp.Chunk()
         chunk.merge(pkt, self.create_merge_callback(pkt))
@@ -119,8 +127,8 @@ def byte_to_seq(self, byte):
     def seq_arrival(self, seq_num):
         '''
         returns the packet in which the specified sequence number first arrived.
-        self.arrival_data must be a SortedCollection at this point; self.finish()
-        must have been called.
+        self.arrival_data must be a SortedCollection at this point;
+        self.finish() must have been called.
         '''
         if self.arrival_data:
             return self.arrival_data.find_le(seq_num)[1]

tcp/flow.py

@@ -33,10 +33,12 @@ def add(self, pkt):
         if len(self.packets): # if we have received packets before...
             if self.packets[-1].ts > pkt.ts: # if this one is out of order...
                 # error out
-                raise ValueError("packet added to TCPFlow out of chronological order")
+                raise ValueError("packet added to TCPFlow out of "
+                                 "chronological order")
         self.packets.append(pkt)
         # look out for handshake
-        # add it to the appropriate direction, if we've found or given up on finding handshake
+        # add it to the appropriate direction, if we've found or given up on
+        # finding handshake
         if self.handshake is not None:
             self.merge_pkt(pkt)
         else: # if handshake is None, we're still looking for a handshake
@@ -45,8 +47,10 @@ def add(self, pkt):
                 self.handshake = False
                 self.socket = self.packets[0].socket
                 self.flush_packets() # merge all stored packets
+                print "LSONG_DEBUG %s: cannot detect handshake." % (__file__)
             # check last three packets
-            elif tcp.detect_handshake(self.packets[-3:]): # function handles packets < 3 case
+            elif tcp.detect_handshake(self.packets[-3:]):
+                # function handles packets < 3 case
                 self.handshake = tuple(self.packets[-3:])
                 self.socket = self.handshake[0].socket
                 self.flush_packets()

tcp/packet.py

@@ -58,10 +58,3 @@ def __repr__(self):
             self.tcp.ack,
             friendly_data(self.tcp.data)[:60]
         )
-    def overlaps(self, other):
-        return (self.seq_start <= other.seq_start and \
-                other.seq_start < self.seq_end) \
-                              or \
-               (self.seq_start < other.seq_end and \
-                other.seq_end <= self.seq_end)
-