performance issues

[outcast]

Pleas take a look at the code below. This was my best guess. I am sure I am doing something wrong here but switching from pymongo to txmongo has caused a 95% decrease in performance. I am wondering if there is way to use the txmongo.MonoConnection object global so I only need to connect to mongo once. Sorry I am still a bit of a noob to python, twisted and mongo. but basic idea is store udp packet for later processing.

---

from twisted.internet.protocol import DatagramProtocol
from twisted.internet import reactor,defer
import txmongo
import struct
import string

x = 0

class Collector(DatagramProtocol):
def init(self):
print "Initializing Collector";

@defer.inlineCallbacks
def datagramReceived(self, data, (host, port)):
    #print "Recieved Packet"
    ch = yield txmongo.MongoConnection()
    dbh = ch.flowpackets.packets
    #print "received %r from %s:%d  type:%s" % (data, host, port,type(data))
    #self.transport.write(data, (host, port))
    global x
    packet = dict()
    packet = { 'packet': data.encode('hex')}
    dbh.insert(packet)
    x += 1

reactor.listenUDP(2055, Collector())
print "Starting Reactor"
reactor.run()
print "Packet Count: ", x

[fiorix]

Perhaps that's because it's making a new connection to mongo on each udp received.
Consider something like the below, which will set up a mongo connection on demand on the first udp:

class Collector(DatagramProtocol):
def init(self):
self.db = None

@defer.inlineCallbacks                                                       
def datagramReceived(self, data, (host, port)):                              
    if self.db is None:                                                      
        self.db = yield txmongo.MongoConnection()                            

    # then use self.db for everything                                        
    # ...                                                                    

On 2012-05-29, at 1:23 PM, James Jones wrote:

from twisted.internet.protocol import DatagramProtocol
from twisted.internet import reactor,defer
import txmongo
import struct
import string

x = 0

class Collector(DatagramProtocol):
def init(self):
print "Initializing Collector";

@defer.inlineCallbacks
def datagramReceived(self, data, (host, port)):
#print "Recieved Packet"
ch = yield txmongo.MongoConnection()
dbh = ch.flowpackets.packets
#print "received %r from %s:%d type:%s" % (data, host, port,type(data))
#self.transport.write(data, (host, port))
global x
packet = dict()
packet = { 'packet': data.encode('hex')}
dbh.insert(packet)
x += 1

reactor.listenUDP(2055, Collector())
print "Starting Reactor"
reactor.run()
print "Packet Count: ", x

Ship, ahoy! Hast seen the White Whale?

• Cap'n Ahab

[outcast]

Sounds like a plan will try that.