-
Notifications
You must be signed in to change notification settings - Fork 1
/
recv.py
74 lines (63 loc) · 2.33 KB
/
recv.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
import requests
import time
import json
import zmq
from multiprocessing import Process
from logbook import FileHandler, catch_exceptions
from sachintweets.models import connect, MongoException
from twitter import username, password
from os import remove, getpid
####################################### Log book setup #########################
log_handler = FileHandler('recv.log')
log_handler.push_application()
###################################### Constants ###############################
LOCK_FILE = 'recv.lock'
###################################### Mongodb connection ######################
try:
db = connect()
if db:
tweet = db.tweet
except Exception as e:
log_handler.write(e.message)
#################################### ZERO MQ PULLER ############################
context = zmq.Context()
socket = context.socket(zmq.SUB)
socket.connect("tcp://*:6789")
socket.setsockopt(zmq.SUBSCRIBE, "")
def store_live_tweets():
while True:
try:
d = json.loads(socket.recv())
if tweet:
tweet.insert({'text': d['text'],\
'location': d['user']['location'], \
'uid': d['user']['id'], 'tid': d['id'],\
'created_at': d['created_at'],\
'username': d['user']['name'],\
'retweet_count': d['retweet_count'],\
'screen_name': d['user']['screen_name'],\
'profile_image_url': d['user']['profile_image_url']})
print "%s ===added to db===", time.ctime()
except Exception as e:
log_handler.write(e.message)
if __name__ == '__main__':
with catch_exceptions():
try:
with open(LOCK_FILE, 'w') as f:
f.write(str(getpid()))
store_live_tweets()
except IndexError as e:
log_handler.write(e.message)
except KeyboardInterrupt:
log_handler.write('keyboard interrupt\n')
finally:
try:
with open(LOCK_FILE, 'r') as f:
#check file exists, I felt this one is better than
#os.path.isfile
# http://stackoverflow.com/questions/82831/\
#how-do-i-check-if-a-file-exists-using-python
pass
remove(LOCK_FILE)
except:
pass