-
Notifications
You must be signed in to change notification settings - Fork 36
/
get_metadata.py
152 lines (130 loc) · 5.42 KB
/
get_metadata.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# /$$$$$$ /$$ /$$ /$$ /$$ /$$$$$$$$
# /$$__ $$| $$$ /$$$| $$$ /$$$|__ $$__/
# | $$ \__/| $$$$ /$$$$| $$$$ /$$$$ | $$
# | $$$$$$ | $$ $$/$$ $$| $$ $$/$$ $$ | $$
# \____ $$| $$ $$$| $$| $$ $$$| $$ | $$
# /$$ \ $$| $$\ $ | $$| $$\ $ | $$ | $$
# | $$$$$$/| $$ \/ | $$| $$ \/ | $$ | $$
# \______/ |__/ |__/|__/ |__/ |__/
#
#
# Developed during Biomedical Hackathon 6 - http://blah6.linkedannotation.org/
# Authors: Ramya Tekumalla, Javad Asl, Juan M. Banda
# Contributors: Kevin B. Cohen, Joanthan Lucero
import tweepy
import json
import math
import glob
import csv
import zipfile
import zlib
import argparse
import os
import os.path as osp
import pandas as pd
from tweepy import TweepError
from time import sleep
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-o", "--outputfile", help="Output file name with extension")
parser.add_argument("-i", "--inputfile", help="Input file name with extension")
parser.add_argument("-c", "--idcolumn", help="tweet id column in the input file, string name")
args = parser.parse_args()
if args.inputfile is None or args.outputfile is None:
parser.error("please add necessary arguments")
with open('api_keys.json') as f:
keys = json.load(f)
auth = tweepy.OAuthHandler(keys['consumer_key'], keys['consumer_secret'])
auth.set_access_token(keys['access_token'], keys['access_token_secret'])
api = tweepy.API(auth)
output_file = args.outputfile
output_file_noformat = output_file.split(".",maxsplit=1)[0]
print(output_file)
output_file = '{}'.format(output_file)
output_file_short = '{}_short.json'.format(output_file_noformat)
compression = zipfile.ZIP_DEFLATED
ids = []
if '.tsv' in args.inputfile:
inputfile_data = pd.read_csv(args.inputfile, sep='\t')
print('tab seperated file, using \\t delimiter')
elif '.csv' in args.inputfile:
inputfile_data = pd.read_csv(args.inputfile)
if not isinstance(args.idcolumn, type(None)):
inputfile_data = inputfile_data.set_index(args.idcolumn)
else:
inputfile_data = inputfile_data.set_index('tweet_id')
ids = list(inputfile_data.index)
print('total ids: {}'.format(len(ids)))
start = 0
end = 100
limit = len(ids)
i = int(math.ceil(float(limit) / 100))
last_tweet = None
if osp.isfile(args.outputfile):
with open(output_file, 'rb') as f:
#may be a large file, seeking without iterating
f.seek(-2, os.SEEK_END)
while f.read(1) != b'\n':
f.seek(-2, os.SEEK_CUR)
last_line = f.readline().decode()
last_tweet = json.loads(last_line)
start = ids.index(last_tweet['id'])
end = start+100
i = int(math.ceil(float(limit-start) / 100))
print('metadata collection complete')
print('creating master json file')
try:
with open(output_file, 'a') as outfile:
for go in range(i):
print('currently getting {} - {}'.format(start, end))
sleep(6) # needed to prevent hitting API rate limit
id_batch = ids[start:end]
start += 100
end += 100
tweets = api.statuses_lookup(id_batch)
for tweet in tweets:
json.dump(tweet._json, outfile)
outfile.write('\n')
except:
print('exception: continuing to zip the file')
print('creating ziped master json file')
zf = zipfile.ZipFile('{}.zip'.format(output_file_noformat), mode='w')
zf.write(output_file, compress_type=compression)
zf.close()
def is_retweet(entry):
return 'retweeted_status' in entry.keys()
def get_source(entry):
if '<' in entry["source"]:
return entry["source"].split('>')[1].split('<')[0]
else:
return entry["source"]
print('creating minimized json master file')
with open(output_file_short, 'w') as outfile:
with open(output_file) as json_data:
for tweet in json_data:
data = json.loads(tweet)
t = {
"created_at": data["created_at"],
"text": data["text"],
"in_reply_to_screen_name": data["in_reply_to_screen_name"],
"retweet_count": data["retweet_count"],
"favorite_count": data["favorite_count"],
"source": get_source(data),
"id_str": data["id_str"],
"is_retweet": is_retweet(data)
}
json.dump(t, outfile)
outfile.write('\n')
f = csv.writer(open('{}.csv'.format(output_file_noformat), 'w'))
print('creating CSV version of minimized json master file')
fields = ["favorite_count", "source", "text", "in_reply_to_screen_name", "is_retweet", "created_at", "retweet_count", "id_str"]
f.writerow(fields)
with open(output_file_short) as master_file:
for tweet in master_file:
data = json.loads(tweet)
f.writerow([data["favorite_count"], data["source"], data["text"].encode('utf-8'), data["in_reply_to_screen_name"], data["is_retweet"], data["created_at"], data["retweet_count"], data["id_str"].encode('utf-8')])
# main invoked here
main()