Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100755 265 lines (216 sloc) 8.533 kb
31479fc Tool to sync flair on site with a local csv file.
Logan Hanks authored
1 #!/usr/bin/python
2
dfb9d0f Clean things up and add comments.
Logan Hanks authored
3 """Tool for syncing flair on reddit with a local csv file.
4
5 The first line of the csv file will be ignored (assumed to be a header).
6
7 Example:
8
9 $ cat test.csv
10 user,text,css
11 intortus,zzr600,kawasaki
12 $ ./flairsync.py -c intortus.cookie example test.csv
13 Parsing csv file: test.csv ...
14 Connecting to http://www.reddit.com ...
15 reddit username: intortus
16 reddit password:
17 Fetching current flair from site ...
18 Computing differences ...
19
20 modifications:
21 intortus -> 'zzr600' kawasaki
22
23 deletions:
24
25
26 apply the above changes? [y/N] y
27 posting flair for users 1-1/1
28 Done!
29 $ cat null.csv
30 user,text,css
31 $ ./flairsync.py -c intortus.cookie example null.csv
32 Parsing csv file: null.csv ...
33 Connecting to http://www.reddit.com ...
34 Fetching current flair from site ...
35 Computing differences ...
36
37 modifications:
38
39
40 deletions:
41 intortus
42
43 apply the above changes? [y/N] y
44 posting flair for users 1-1/1
45 Done!
46 """
47
31479fc Tool to sync flair on site with a local csv file.
Logan Hanks authored
48 import argparse
49 import csv
afb1066 Add crude http digest auth support.
Logan Hanks authored
50 import getpass
31479fc Tool to sync flair on site with a local csv file.
Logan Hanks authored
51 import htmlentitydefs
01f4ad9 Add logging, API updates.
Logan Hanks authored
52 import logging
53 import os
31479fc Tool to sync flair on site with a local csv file.
Logan Hanks authored
54 import re
55 import sys
01f4ad9 Add logging, API updates.
Logan Hanks authored
56 import time
31479fc Tool to sync flair on site with a local csv file.
Logan Hanks authored
57
58 import redditclient
59
60 def parse_args():
61 parser = argparse.ArgumentParser(
62 description='sync subreddit flair with with a csv file')
63
64 parser.add_argument('subreddit', metavar='SUBREDDIT')
65 parser.add_argument('csvfile', metavar='CSVFILE')
66
afb1066 Add crude http digest auth support.
Logan Hanks authored
67 parser.add_argument('-A', '--http_auth', default=False, const=True,
68 action='store_const',
69 help='set if HTTP basic authentication is needed')
31479fc Tool to sync flair on site with a local csv file.
Logan Hanks authored
70 parser.add_argument('-b', '--batch_size', type=int, default=100,
71 help='number of users to read at a time from the site')
72 parser.add_argument('-c', '--cookie_file',
73 help='if given, save session cookie in this file')
dfb9d0f Clean things up and add comments.
Logan Hanks authored
74 parser.add_argument('-H', '--host', default='http://www.reddit.com',
31479fc Tool to sync flair on site with a local csv file.
Logan Hanks authored
75 help='URL of reddit API server')
10e3c73 Add support for syncing flair templates.
Logan Hanks authored
76 parser.add_argument('-t', '--templates', default=False, const=True,
77 action='store_const',
78 help='sync flair templates instead of flair')
dfb9d0f Clean things up and add comments.
Logan Hanks authored
79 parser.add_argument('-v', '--verbose', default=False, const=True,
80 action='store_const',
81 help='emit more verbose logging')
31479fc Tool to sync flair on site with a local csv file.
Logan Hanks authored
82 return parser.parse_args()
83
84 def ynprompt(prompt):
85 return raw_input(prompt).lower().startswith('y')
86
afb1066 Add crude http digest auth support.
Logan Hanks authored
87 def log_in(host, cookie_file, use_http_auth):
3466639 Add User-Agent to flairsync requests.
Logan Hanks authored
88 options = dict(user_agent='flairsync')
afb1066 Add crude http digest auth support.
Logan Hanks authored
89 if use_http_auth:
90 http_user = raw_input('HTTP auth username: ')
91 http_password = getpass.getpass('HTTP auth password: ')
3466639 Add User-Agent to flairsync requests.
Logan Hanks authored
92 options.update(
93 _http_user=http_user, _http_password=http_password)
afb1066 Add crude http digest auth support.
Logan Hanks authored
94 client = redditclient.RedditClient(host, cookie_file, **options)
31479fc Tool to sync flair on site with a local csv file.
Logan Hanks authored
95 while not client.log_in():
96 print 'login failed'
97 return client
98
99 def flair_from_csv(path):
100 f = csv.reader(file(path))
101 # skip header row
102 f.next()
01f4ad9 Add logging, API updates.
Logan Hanks authored
103 return dict((r[0], (r[1], r[2])) for r in f if r[1] or r[2])
31479fc Tool to sync flair on site with a local csv file.
Logan Hanks authored
104
105 def flair_from_reddit(client, subreddit, batch_size):
106 def u(html):
107 if html is None:
108 return None
109 def decode_entity(m):
110 text = m.group(0)
111 if text.startswith('&#'):
112 try:
113 if text.startswith('&#x'):
114 return unichr(int(text[3:-1]), 16)
115 else:
116 return unichr(int(text[2:-1]))
117 except ValueError:
118 pass # ignore
119 else:
120 try:
121 return unichr(htmlentitydefs.name2codepoint[text[1:-1]])
122 except KeyError:
123 pass # ignore
124 # fallthrough; on error just return original text
125 return text
126 return re.sub(r'&#?\w+;', decode_entity, html)
127
128 return dict((u(r[0]), (u(r[1]), u(r[2])))
129 for r in client.flair_list(subreddit, batch_size=batch_size))
130
131 def diff_flair(left, right):
01f4ad9 Add logging, API updates.
Logan Hanks authored
132 """returns: (modifications, deletions)"""
31479fc Tool to sync flair on site with a local csv file.
Logan Hanks authored
133 left_users = frozenset(left)
134 right_users = frozenset(right)
135 common_users = left_users & right_users
136 added_users = right_users - left_users
137 removed_users = left_users - right_users
01f4ad9 Add logging, API updates.
Logan Hanks authored
138 modifications = [(u, right[u][0], right[u][1])
139 for u in common_users if left[u] != right[u]]
140 modifications.extend((u, right[u][0], right[u][1]) for u in added_users)
141 return modifications, removed_users
142
143 def configure_logging():
144 class LoggingFormatter(logging.Formatter):
145 def formatTime(self, record, datefmt=None):
146 timestamp = time.strftime(datefmt)
147 return timestamp % dict(ms=(1000 * record.created) % 1000)
148
149 logger = logging.getLogger()
150 logger.setLevel(logging.INFO)
151 ch = logging.StreamHandler()
152 ch.setLevel(logging.INFO)
153 formatter = LoggingFormatter(
154 '%(levelname).1s%(asctime)s: %(message)s',
155 '%m%d %H:%M:%S.%%(ms)03d')
156 ch.setFormatter(formatter)
157 logger.addHandler(ch)
31479fc Tool to sync flair on site with a local csv file.
Logan Hanks authored
158
dfb9d0f Clean things up and add comments.
Logan Hanks authored
159 def summarize_batch_result(lines, result):
160 failed_entries = [e for e in result if not e['ok']]
161 if failed_entries:
162 print 'WARNING: %d entr%s skipped:' % (
163 len(failed_entries),
164 'y was' if len(failed_entries) == 1 else 'ies were')
165
166 for l, e in zip(lines, result):
167 if not e['ok']:
168 print ' entry: %s' % ','.join(l)
169 for m in e['errors'].itervalues():
170 print ' - %s' % m
171
172 # print out any warnings for items that were committed
173 for l, e in zip(lines, result):
174 if e['ok'] and e['warnings']:
175 for m in e['warnings'].itervalues():
176 print 'NOTE: entry %s: %s' % (','.join(l), m)
177
10e3c73 Add support for syncing flair templates.
Logan Hanks authored
178 def sync_flair(config):
31479fc Tool to sync flair on site with a local csv file.
Logan Hanks authored
179 print 'Parsing csv file: %s ...' % config.csvfile
180 csv_flair = flair_from_csv(config.csvfile)
181
182 print 'Connecting to %s ...' % config.host
afb1066 Add crude http digest auth support.
Logan Hanks authored
183 client = log_in(config.host, config.cookie_file, config.http_auth)
31479fc Tool to sync flair on site with a local csv file.
Logan Hanks authored
184
185 print 'Fetching current flair from site ...'
186 reddit_flair = flair_from_reddit(client, config.subreddit,
187 config.batch_size)
188
189 print 'Computing differences ...'
01f4ad9 Add logging, API updates.
Logan Hanks authored
190 modifications, deletions = diff_flair(reddit_flair, csv_flair)
31479fc Tool to sync flair on site with a local csv file.
Logan Hanks authored
191
192 print '\nmodifications:'
193 print '\n'.join([' %s -> %r %s' % mod for mod in modifications])
194 print '\ndeletions:'
195 print '\n'.join([' %s' % u for u in deletions])
196
01f4ad9 Add logging, API updates.
Logan Hanks authored
197 if ((modifications or deletions)
31479fc Tool to sync flair on site with a local csv file.
Logan Hanks authored
198 and ynprompt('\napply the above changes? [y/N] ')):
199
01f4ad9 Add logging, API updates.
Logan Hanks authored
200 new_flair = modifications + [(u, '', '') for u in deletions]
31479fc Tool to sync flair on site with a local csv file.
Logan Hanks authored
201
01f4ad9 Add logging, API updates.
Logan Hanks authored
202 for i in xrange(0, len(new_flair), 100):
203 print 'posting flair for users %d-%d/%d' % (
dfb9d0f Clean things up and add comments.
Logan Hanks authored
204 i + 1, min(len(new_flair), i + 100), len(new_flair))
01f4ad9 Add logging, API updates.
Logan Hanks authored
205 result = client.flaircsv(config.subreddit, new_flair[i:i+100])
dfb9d0f Clean things up and add comments.
Logan Hanks authored
206 logging.info('flaircsv result: %r', result)
207 summarize_batch_result(new_flair, result)
31479fc Tool to sync flair on site with a local csv file.
Logan Hanks authored
208
209 print 'Done!'
210
3c9d630 Fix the text-editable boolean of flair templates.
Logan Hanks authored
211 def csv_bool(value):
212 # interpret a string as a boolean in the same way that the server does for
213 # CGI parameters
214 value = value.lower()
215 if not value or value[0] in 'fn' or value == 'off':
216 return False
217 return True
218
10e3c73 Add support for syncing flair templates.
Logan Hanks authored
219 def templates_from_csv(path):
220 f = csv.reader(file(path))
221 # skip header row
222 f.next()
3c9d630 Fix the text-editable boolean of flair templates.
Logan Hanks authored
223 return [(r[0], r[1], csv_bool(r[2])) for r in f]
10e3c73 Add support for syncing flair templates.
Logan Hanks authored
224
225 def sync_templates(config):
226 print 'Parsing csv file: %s ...' % config.csvfile
227 csv_templates = templates_from_csv(config.csvfile)
228
229 print 'Connecting to %s ...' % config.host
230 client = log_in(config.host, config.cookie_file, config.http_auth)
231
232 print 'Clearing flair templates ...'
233 client._post(client._url('/api/clearflairtemplates', sr=config.subreddit))
234
235 for text, css_class, text_editable in csv_templates:
236 print 'Adding flair templates: %r, %r, %r' % (
237 text, css_class, text_editable)
238 count = 0
239 while count < 3:
240 try:
241 client._post(client._url('/api/flairtemplate',
242 sr=config.subreddit),
243 text=text, css_class=css_class,
3c9d630 Fix the text-editable boolean of flair templates.
Logan Hanks authored
244 text_editable='on' if text_editable else 'off')
10e3c73 Add support for syncing flair templates.
Logan Hanks authored
245 except:
246 count += 1
247 else:
248 break
249
250 print 'Done!'
251
252
253 def main():
254 config = parse_args()
255 if config.verbose:
256 configure_logging()
257 if config.templates:
258 return sync_templates(config)
259 else:
260 return sync_flair(config)
261
262
31479fc Tool to sync flair on site with a local csv file.
Logan Hanks authored
263 if __name__ == '__main__':
264 main()
Something went wrong with that request. Please try again.