Merge pull request #24 from sammyrulez/master

Better memory management (Thanks sammyrulez - will also merge in your mongodb commit as part of this pull request but back it out of master since MongoDB isn't used elsewhere in Mining the Social Web)

python_code/mailboxes__jsonify_mbox.py

@@ -12,6 +12,11 @@
     import json
 
 MBOX = sys.argv[1]
+OUT_FILE = None
+try:
+    OUT_FILE = sys.argv[2]
+except Exception, e:
+    pass
 
 def cleanContent(msg):
 
@@ -54,13 +59,17 @@ def jsonifyMessage(msg):
     finally:
         return json_msg
 
-# Note: opening in binary mode is recommended
-mbox = mailbox.UnixMailbox(open(MBOX, 'rb'), email.message_from_file)  
-json_msgs = []
-while 1:
-    msg = mbox.next()
-    if msg is None:
-        break
-    json_msgs.append(jsonifyMessage(msg))
-
-print json.dumps(json_msgs, indent=4)
+ #Note: opening in binary mode is recommended
+
+ mbox = mailbox.UnixMailbox(open(MBOX, 'rb'), email.message_from_file)  
+ def gen_json_msgs(m_box):
+    while 1:
+        msg = m_box.next()
+        if msg is None:
+            break
+        yield jsonifyMessage(msg)
+
+if OUT_FILE:
+    json.dump(gen_json_msgs(mbox),open(OUT_FILE, 'wb'), indent=4)
+else:
+    print json.dumps(gen_json_msgs(mbox), indent=4)

python_code/mailboxes__load_json_mbox.py

@@ -2,16 +2,23 @@
 
 import sys
 import os
-import couchdb
+import pymongo
+from pymongo import Connection
+
 try:
     import jsonlib2 as json
 except ImportError:
     import json
 
 JSON_MBOX = sys.argv[1]  # i.e. enron.mbox.json
-DB = os.path.basename(JSON_MBOX).split('.')[0]
+DB_NAME = os.path.basename(JSON_MBOX).split('.')[0]
+
+connection = Connection('localhost', 27017)
+db = connection[DB_NAME]
 
-server = couchdb.Server('http://localhost:5984')
-db = server.create(DB)
 docs = json.loads(open(JSON_MBOX).read())
-db.update(docs, all_or_nothing=True)
+
+collection = db['messages']
+for doc in docs:
+    print str(doc)
+	collection.insert(doc)