Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Create clis package

  • Loading branch information...
commit 7d01d9cb10e4cc915d149832c70ef034e7003c70 1 parent 93b9089
Yu-Jie Lin authored
6 README.md
Source Rendered
@@ -16,11 +16,9 @@ clis remembers (not all source types) last one it shows you, so after you restar
16 16
17 17 # Dependencies
18 18
19   -All are included in repo.
20   -
21   - * [pyratemp](http://www.simple-is-better.org/template/pyratemp.html)
  19 + * [pyratemp](http://www.simple-is-better.org/template/pyratemp.html) (included)
22 20 * [FeedParser](http://feedparser.org/)
23   - * [Official FriendFeed API library](http://code.google.com/p/friendfeed-api/)
  21 + * [Official FriendFeed API library](http://code.google.com/p/friendfeed-api/) (included)
24 22 * [python-oauth2](http://github.com/simplegeo/python-oauth2) for Twitter sources
25 23
26 24 # Supported Sources
0  src/clis_cfg-sample.py → samples/clis.cfg.py
File renamed without changes
1,657 src/clis.py
... ... @@ -1,1659 +1,10 @@
1   -#!/usr/bin/python
2   -# -*- coding: utf-8 -*-
3   -# clis - CLI Stream Reader
4   -# Copyright 2009, 2010, Yu-Jie Lin
5   -# GPLv3
  1 +#!/usr/bin/env python
6 2
7   -from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
8   -from datetime import datetime, timedelta, tzinfo
9   -from optparse import OptionParser
10   -from os import path
11   -from StringIO import StringIO
12   -import __builtin__
13   -import elementtree.ElementTree as ET
14   -import imp
15   -import json
16   -import new
17   -import os
18   -import re
19   -import select
20   -import shelve
21   -import signal
22   -import socket
23   -import stat
24   -import sys
25   -import termios
26   -import threading
27   -import time
28   -import traceback
29   -import tty
30   -import urllib
31   -import urllib2
32   -import urlparse
33   -socket.setdefaulttimeout(10)
34 3
35   -from pyratemp import Template as tpl
36   -
37   -import feedparser as fp
38   -import friendfeed as ff
39   -import oauth2 as oauth
40   -
41   -
42   -###############
43   -# Twitter OAuth
44   -
45   -request_token_url = 'https://api.twitter.com/oauth/request_token'
46   -access_token_url = 'https://api.twitter.com/oauth/access_token'
47   -authorize_url = 'https://api.twitter.com/oauth/authorize'
48   -
49   -###########
50   -# Utilities
51   -
52   -TPL_P_ERR = tpl('@!ansi.fiwhite!@@!ansi.bired!@ERROR: @!error!@@!ansi.breset!@@!ansi.freset!@', escape=None)
53   -TPL_P_DBG = tpl('@!ansi.fiwhite!@@!ansi.biblue!@DEBUG: @!msg!@@!ansi.breset!@@!ansi.freset!@', escape=None)
54   -
55   -
56   -def p(msg):
57   -
58   - sys.stdout.write(msg)
59   - sys.stdout.flush()
60   -
61   -
62   -def p_clr(msg):
63   -
64   - for k, v in ANSI.__dict__.iteritems():
65   - if not k.startswith('_'):
66   - msg = msg.replace(v, '')
67   -
68   - sys.stdout.write('\b \b' * len(msg))
69   - sys.stdout.flush()
70   -
71   -
72   -def p_err(error):
73   -
74   - p(TPL_P_ERR(ansi=ANSI, error=error))
75   -
76   -
77   -def p_dbg(msg, newline=True):
78   -
79   - if DEBUG:
80   - p(TPL_P_DBG(ansi=ANSI, msg=msg))
81   - if newline:
82   - p('\n')
83   -
84   -
85   -def safe_update(func):
86   -
87   - def deco(*args, **kwds):
88   - try:
89   - func(*args, **kwds)
90   - except select.error:
91   - pass
92   - except (socket.error, socket.timeout, urllib2.HTTPError, urllib2.URLError), e:
93   - p_err('%s\n' % repr(e))
94   - except Exception:
95   - p_err('\n')
96   - traceback.print_exc()
97   - return deco
98   -
99   -
100   -def ftime(d, fmt):
101   -
102   - if d:
103   - return time.strftime(fmt, d.timetuple())
104   - else:
105   - return '!NODATE!'
106   -
107   -
108   -lurls = {}
109   -lurls['count'] = 0
110   -# TODO limit the numbers
111   -def surl(url):
112   -
113   - global options
114   -
115   - def fmt_url(id):
116   -
117   - if options.local_port == 80:
118   - new_url = 'http://%s/%d' % (options.local_server, id)
119   - else:
120   - new_url = 'http://%s:%d/%d' % (options.local_server, options.local_port, id)
121   - return new_url
122   -
123   - if not options.local_shortening:
124   - return url
125   -
126   - # Alredy shortened?
127   - for id, val in lurls.iteritems():
128   - if val == url:
129   - return fmt_url(id)
130   -
131   - new_url = fmt_url(lurls['count'])
132   -
133   - if len(new_url) >= len(url):
134   - return url
135   -
136   - # Shorter, store it
137   - lurls[lurls['count']] = url
138   - lurls['count'] += 1
139   - return new_url
140   -
141   -
142   -def unescape(s):
143   -
144   - s = s.replace("&lt;", "<")
145   - s = s.replace("&gt;", ">")
146   - s = s.replace("&quot;", '"')
147   - s = s.replace("&amp;", "&")
148   - return s
149   -
150   -
151   -def remove_additional_space(s):
152   -
153   - return re.sub('( |\n|\t)+', ' ', s)
154   -
155   -
156   -# A very simple version of tags stripping, it might be buggy
157   -RE_STRIP_TAGS = re.compile(ur'<.*?>', re.IGNORECASE | re.MULTILINE | re.DOTALL | re.UNICODE)
158   -
159   -def strip_tags(s):
160   -
161   - while True:
162   - s1 = RE_STRIP_TAGS.sub('', s)
163   - if s1 == s:
164   - break
165   - s = s1
166   - return s1
167   -
168   -
169   -##################
170   -# ANSI escape code
171   -
172   -class ANSI:
173   -
174   - # http://en.wikipedia.org/wiki/ANSI_escape_code
175   - reset = '\033[0m'
176   - bold = '\033[1m'
177   - underline = '\033[4m'
178   - blink = '\033[5m'
179   - no_underline = '\033[24m'
180   - no_blink = '\033[25m'
181   - fblack = '\033[30m'
182   - fred = '\033[31m'
183   - fgreen = '\033[32m'
184   - fyellow = '\033[33m'
185   - fblue = '\033[34m'
186   - fmagenta = '\033[35m'
187   - fcyan = '\033[36m'
188   - fwhite = '\033[37m'
189   - freset = '\033[39m'
190   - bblack = '\033[40m'
191   - bred = '\033[41m'
192   - bgreen = '\033[42m'
193   - byellow = '\033[43m'
194   - bblue = '\033[44m'
195   - bmagenta = '\033[45m'
196   - bcyan = '\033[46m'
197   - bwhite = '\033[47m'
198   - breset = '\033[49m'
199   - fiblack = '\033[90m'
200   - fired = '\033[91m'
201   - figreen = '\033[92m'
202   - fiyellow = '\033[93m'
203   - fiblue = '\033[94m'
204   - fimagenta = '\033[95m'
205   - ficyan = '\033[96m'
206   - fiwhite = '\033[97m'
207   - biblack = '\033[100m'
208   - bired = '\033[101m'
209   - bigreen = '\033[102m'
210   - biyellow = '\033[103m'
211   - biblue = '\033[104m'
212   - bimagenta = '\033[105m'
213   - bicyan = '\033[106m'
214   - biwhite = '\033[107m'
215   -
216   -
217   -common_tpl_opts = {
218   - 'ansi': ANSI,
219   - 'ftime': ftime,
220   - 'surl': surl, 'lurls': lurls,
221   - 'unescape': unescape,
222   - 'remove_additional_space': remove_additional_space,
223   - 'strip_tags': strip_tags,
224   - }
225   -
226   -
227   -##########
228   -# Timezone
229   -
230   -ZERO = timedelta(0)
231   -class UTC(tzinfo):
232   -
233   - def utcoffset(self, dt):
234   - return ZERO
235   -
236   - def tzname(self, dt):
237   - return "UTC"
238   -
239   - def dst(self, dt):
240   - return ZERO
241   -
242   -
243   -class LOCAL_TZ(tzinfo):
244   -
245   - def utcoffset(self, dt):
246   - return timedelta(seconds=-1*time.altzone)
247   -
248   - def tzname(self, dt):
249   - return "UTC"
250   -
251   - def dst(self, dt):
252   - # FIXME not sure if it is in seconds
253   - return timedelta(seconds=-1*time.daylight)
254   -
255   -
256   -utc = UTC()
257   -local_tz = LOCAL_TZ()
258   -
259   -####################################
260   -# xml2dict (modified by me)
261   -# http://code.google.com/p/xml2dict/
262   -
263   -class XML2Dict(object):
264   -
265   - @classmethod
266   - def _parse_node(cls, node):
267   - node_tree = object_dict()
268   - # Save attrs and text, hope there will not be a child with same name
269   - if node.text:
270   - node_tree.value = node.text
271   - for (k,v) in node.attrib.items():
272   - k,v = cls._namespace_split(k, object_dict({'value':v}))
273   - node_tree[k] = v
274   - #Save childrens
275   - for child in node.getchildren():
276   - tag, tree = cls._namespace_split(child.tag, cls._parse_node(child))
277   - if tag not in node_tree: # the first time, so store it in dict
278   - node_tree[tag] = tree
279   - continue
280   - old = node_tree[tag]
281   - if not isinstance(old, list):
282   - node_tree.pop(tag)
283   - node_tree[tag] = [old] # multi times, so change old dict to a list
284   - node_tree[tag].append(tree) # add the new one
285   -
286   - return node_tree
287   -
288   - @classmethod
289   - def _namespace_split(cls, tag, value):
290   - """
291   - Split the tag '{http://cs.sfsu.edu/csc867/myscheduler}patients'
292   - ns = http://cs.sfsu.edu/csc867/myscheduler
293   - name = patients
294   - """
295   - result = re.compile("\{(.*)\}(.*)").search(tag)
296   - if result:
297   - value.namespace, tag = result.groups()
298   - return (tag, value)
299   -
300   - @classmethod
301   - def parse(cls, file):
302   - """parse a xml file to a dict"""
303   - f = open(file, 'r')
304   - return cls.fromstring(f.read())
305   -
306   - @classmethod
307   - def fromstring(cls, s):
308   - """parse a string"""
309   - t = ET.fromstring(s)
310   - root_tag, root_tree = cls._namespace_split(t.tag, cls._parse_node(t))
311   - return object_dict({root_tag: root_tree})
312   -
313   -
314   -class object_dict(dict):
315   - """object view of dict, you can
316   - >>> a = object_dict()
317   - >>> a.fish = 'fish'
318   - >>> a['fish']
319   - 'fish'
320   - >>> a['water'] = 'water'
321   - >>> a.water
322   - 'water'
323   - >>> a.test = {'value': 1}
324   - >>> a.test2 = object_dict({'name': 'test2', 'value': 2})
325   - >>> a.test, a.test2.name, a.test2.value
326   - (1, 'test2', 2)
327   - """
328   - def __init__(self, initd=None):
329   - if initd is None:
330   - initd = {}
331   - dict.__init__(self, initd)
332   -
333   - def __getattr__(self, item):
334   - d = self.__getitem__(item)
335   - # if value is the only key in object, you can omit it
336   - if isinstance(d, dict) and 'value' in d and len(d) == 1:
337   - return d['value']
338   - else:
339   - return d
340   -
341   - def __setattr__(self, item, value):
342   - self.__setitem__(item, value)
343   -
344   -####################
345   -# non-blocking stdin
346   -
347   -def getch():
348   -
349   - global p_stdin
350   -
351   - if p_stdin.poll(1000):
352   - return sys.stdin.read(1)
353   - else:
354   - return None
355   -
356   -def ttywidth():
357   -
358   - f = os.popen('tput cols', 'r')
359   - width = int(f.read())
360   - f.close()
361   - return width
362   -
363   -
364   -def update_width(signum, frame):
365   -
366   - global width
367   -
368   - width = ttywidth()
369   -
370   -
371   -def sigexit(signum, frame):
372   -
373   - if 'session' in __builtin__.__dict__:
374   - session.close()
375   - if fd is not None and old_settings is not None:
376   - termios.tcsetattr(fd, termios.TCSANOW, old_settings)
377   - p('\033[?25h')
378   -
379   -
380   -class STDOUT_R:
381   -
382   - @staticmethod
383   - def write(s):
384   -
385   - s = s.replace('\n', '\r\n')
386   - sys.__stdout__.write(s.encode('utf-8'))
387   -
388   - @staticmethod
389   - def flush():
390   -
391   - return sys.__stdout__.flush()
392   -
393   -
394   -class STDERR_R:
395   -
396   - @staticmethod
397   - def write(s, *args, **kwds):
398   -
399   - s = s.replace('\n', '\r\n')
400   - sys.__stderr__.write(s.encode('utf-8'))
401   -
402   - @staticmethod
403   - def flush():
404   -
405   - return sys.__stderr__.flush()
406   -
407   -#########
408   -# Session
409   -
410   -def open_session(loc):
411   -
412   - s_path = path.expanduser('~/.clis/session')
413   - if not path.exists(s_path):
414   - os.makedirs(s_path, 0700)
415   - filename = '%s/%s' % (s_path, hash(loc))
416   - __builtin__.session = shelve.open(filename, writeback=True)
417   - session['config'] = loc
418   - session.__dict__['last_sync'] = time.time()
419   - session.__dict__['interval'] = 60
420   -
421   - def do_sync(self, sources):
422   -
423   - if time.time() < self.interval + self.last_sync:
424   - return
425   - self.last_sync = time.time()
426   -
427   - msg = 'Saving session data...'
428   - p(msg)
429   - self.sync()
430   - p_clr(msg)
431   -
432   - session.__dict__['do_sync'] = new.instancemethod(do_sync, session, dict)
433   - p_dbg('Session file %s opened' % filename)
434   -
435   -##############
436   -# Source class
437   -
438   -class Source(object):
439   -
440   - TYPE = 'unknown'
441   - TPL_ACCESS = tpl('@!ansi.fired!@Accessing [@!src_name!@] @!src_id!@...@!ansi.freset!@', escape=None)
442   - CHECK_LIST_SIZE = 20
443   -
444   - def __init__(self, src):
445   -
446   - self.last_accessed = 0
447   - self.include = src.get('include', [])
448   - self.exclude = src.get('exclude', [])
449   - self.highlight = src.get('highlight', [])
450   - self.hide_id = src.get('hide_id', False)
451   -
452   - self.RE_INCLUDE = {}
453   - for key, includes in self.include:
454   - self.RE_INCLUDE[key] = re.compile(u'(' + u'|'.join(includes) + u')', re.I | re.U)
455   - self.RE_EXCLUDE = {}
456   - for key, excludes in self.exclude:
457   - self.RE_EXCLUDE[key] = re.compile(u'(' + u'|'.join(excludes) + u')', re.I | re.U)
458   -
459   - def _init_session(self):
460   -
461   - session_id = '%s:%s' % (self.TYPE, self.src_id)
462   - if session_id not in session:
463   - # New source, need to initialize
464   - p_dbg('New source: [%s]' % session_id)
465   - session[session_id] = {}
466   - self.session = session[session_id]
467   - self.session_id = session_id
468   -
469   - # The following two functions simplely use last entry's id to check, usually
470   - # helpful source can query with parameter like since_id or starting date.
471   - def _load_last_id(self):
472   -
473   - if 'last_id' in self.session:
474   - self.last_id = self.session['last_id']
475   - p_dbg('Session [%s] last_id = "%s"' % (self.session_id, self.last_id))
476   - else:
477   - self._update_last_id(None)
478   -
479   - def _update_last_id(self, last_id):
480   -
481   - self.last_id = last_id
482   - self.session['last_id'] = last_id
483   - session[self.session_id] = self.session
484   - p_dbg('Updating [%s] last_id to %s' % (self.session_id, session[self.session_id]['last_id']))
485   -
486   - # The following three functions store a list of entries' id and updated, then
487   - # use the list to compare. The length of the list should be larger than the
488   - # amount of entries in feed. Default is 20.
489   - def is_new_item(self, entry):
490   - '''Check if entry is new and also update check_list if it is new'''
491   - e_id = self.get_entry_id(entry)
492   - e_updated = self.get_entry_updated(entry)
493   - if e_id in self.check_list:
494   - if e_updated <= self.check_list[e_id]:
495   - return False
496   - self.check_list[e_id] = e_updated
497   - return True
498   -
499   - def _load_check_list(self):
500   -
501   - if 'check_list' in self.session:
502   - self.check_list = self.session['check_list']
503   - p_dbg('Session [%s] checklist loaded' % self.session_id)
504   - else:
505   - self.check_list = {}
506   - p_dbg('Session [%s] checklist initialized' % self.session_id)
507   -
508   - def _update_check_list(self):
509   - '''Limit items in check_list'''
510   -
511   - lst = self.check_list.items()
512   - lst.sort(key=lambda x: x[1], reverse=True)
513   - # Limit the size
514   - lst = lst[:self.CHECK_LIST_SIZE]
515   - self.check_list = dict(lst)
516   - self.session['check_list'] = self.check_list
517   - session[self.session_id] = self.session
518   - p_dbg('Updated [%s] check_list' % self.session_id)
519   -
520   - @staticmethod
521   - def get_entry_id(entry):
522   -
523   - if 'guid' in entry:
524   - return entry['guid']
525   - if 'id' in entry:
526   - return entry['id']
527   - return entry['title'] + entry['link']
528   -
529   - @staticmethod
530   - def get_entry_updated(entry):
531   - '''Decide the last updated of the entry
532   - The date object must be a datetime. If none date is found, then it assign
533   - the current local time to key updated.'''
534   - dates = []
535   - for key in ['updated', 'published', 'created']:
536   - if key in entry and entry[key]:
537   - dates += [entry[key]]
538   - if not dates:
539   - entry['updated'] = datetime.utcnow().replace(tzinfo=utc)
540   - return entry['updated']
541   - return max(dates)
542   -
543   - def datetimeize(self, entry):
544   - '''Convert all date to datetime in localtime'''
545   - for key in ['updated', 'published', 'created', 'expired']:
546   - if key in entry and entry[key]:
547   - # XXX
548   - try:
549   - entry[key] = self.to_localtime(entry[key])
550   - except Exception, e:
551   - print entry
552   - raise e
553   -
554   - @staticmethod
555   - def to_localtime(d):
556   - '''Convert UTC datetime to localtime datetime'''
557   - return datetime(*d[:6]).replace(tzinfo=utc).astimezone(local_tz)
558   -
559   - def is_included(self, entry):
560   -
561   - for key in self.RE_INCLUDE.keys():
562   - try:
563   - # FIXME Dangerous
564   - value = eval('entry%s' % key)
565   - if key == '["tags"]':
566   - # Specially for feed class
567   - for tag in value:
568   - if self.RE_INCLUDE[key].search(tag['term']):
569   - p_dbg('Included %s: Category %s' % (key, tag['term']))
570   - return True
571   - elif self.RE_INCLUDE[key].search(value):
572   - # The value of key is not a list
573   - p_dbg('Included %s: %s' % (key, value))
574   - return True
575   - except Exception, e:
576   - p_err('[%s][is_included] %s' % (self.session_id, repr(e)))
577   - raise e
578   - return False
579   -
580   - def is_excluded(self, entry):
581   -
582   - for key in self.RE_EXCLUDE.keys():
583   - try:
584   - # FIXME Dangerous
585   - value = eval('entry%s' % key)
586   - if key == '["tags"]':
587   - # Specially for feed class
588   - for tag in value:
589   - if self.RE_EXCLUDE[key].search(tag['term']):
590   - p_dbg('Excluded %s: Category %s' % (key, tag['term']))
591   - return True
592   - elif self.RE_EXCLUDE[key].search(value):
593   - # The value of key is not a list
594   - p_dbg('Excluded %s: %s' % (key, value))
595   - return True
596   - except Exception, e:
597   - p_err('[%s][is_excluded] %s' % (self.session_id, repr(e)))
598   - raise e
599   - return False
600   -
601   - def process_highlight(self, entry):
602   -
603   - if not self.highlight:
604   - return
605   -
606   - for key, highlights in self.highlight:
607   - try:
608   - # FIXME Dangerous
609   - value = eval('entry%s' % key)
610   - r_hl = re.compile(u'(' + u'|'.join(highlights) + u')', re.I | re.U)
611   - new_value = r_hl.sub(unicode(ANSI.fired) + ur'\1' + unicode(ANSI.freset), value)
612   - exec u'entry%s = u"""%s"""' % (key, new_value.replace(u'"', ur'\"'))
613   - except Exception, e:
614   - p_err('[%s] %s' % (self.session_id, repr(e)))
615   - raise e
616   -
617   - @safe_update
618   - def update(self):
619   -
620   - if time.time() < self.interval + self.last_accessed:
621   - return
622   - self.last_accessed = time.time()
623   -
624   - if self.hide_id:
625   - msg = self.TPL_ACCESS(ansi=ANSI, src_name=self.src_name, src_id='*Source ID is Hidden*')
626   - else:
627   - msg = self.TPL_ACCESS(ansi=ANSI, src_name=self.src_name, src_id=self.src_id)
628   - p(msg)
629   - feed = self.get_list()
630   - p_clr(msg)
631   - if not feed['entries']:
632   - return
633   -
634   - entries = []
635   - if self.CHECK_LIST_SIZE < len(feed['entries'] * 2):
636   - self.CHECK_LIST_SIZE = len(feed['entries'] * 2)
637   - p_dbg('Changed CHECK_LIST_SIZE to %d' % self.CHECK_LIST_SIZE)
638   -
639   - # Get entries after last_id
640   - for entry in feed['entries']:
641   - self.datetimeize(entry)
642   - if not self.is_new_item(entry):
643   - continue
644   - entries += [entry]
645   - # Update last_id
646   - if entries:
647   - self._update_check_list()
648   -
649   - entries.reverse()
650   - for entry in entries:
651   - p_dbg('ID: %s' % self.get_entry_id(entry))
652   - if self.RE_INCLUDE and not self.is_included(entry):
653   - continue
654   - if self.is_excluded(entry):
655   - continue
656   - # XXX
657   - try:
658   - self.process_highlight(entry)
659   - print self.output(entry=entry, src_name=self.src_name, **common_tpl_opts)
660   - if hasattr(self, 'say'):
661   - self.sayit(self.say(entry=entry, src_name=self.src_name, **common_tpl_opts))
662   - except Exception, e:
663   - print entry
664   - raise e
665   -
666   - def sayit(self, text):
667   -
668   - # XXX !!!Experimental!!! Should have no quotation mark in text
669   - # TODO: Use pipe and/or speechd
670   - #os.system('echo "%s" | festival --tts &' % text.replace('"', ''))
671   - os.system('echo "%s" | festival --tts' % text.replace('"', ''))
672   -
673   -
674   -class Twitter(Source):
675   -
676   - TYPE = 'twitter'
677   - REQUEST_URI = 'http://api.twitter.com/1/statuses/friends_timeline.json'
678   -
679   - def __init__(self, src):
680   -
681   - super(Twitter, self).__init__(src)
682   -
683   - self.username = src['username']
684   - self.consumer_key = src['consumer_key']
685   - self.consumer_secret = src['consumer_secret']
686   -
687   - self.src_id = self.username
688   - self.src_name = src.get('src_name', 'Twitter')
689   - self.interval = src.get('interval', 90)
690   - self.output = tpl(src.get('output', '@!ansi.fgreen!@@!ftime(status["created_at"], "%H:%M:%S")!@@!ansi.freset!@ [@!src_name!@] @!ansi.fyellow!@@!status["user"]["screen_name"]!@@!ansi.freset!@: @!unescape(status["text"])!@ @!ansi.fmagenta!@@!surl(status["tweet_link"])!@@!ansi.freset!@'), escape=None)
691   -
692   - self._init_session()
693   - self.create_connection()
694   - self._load_last_id()
695   -
696   - def create_connection(self):
697   -
698   - self.consumer = oauth.Consumer(self.consumer_key, self.consumer_secret)
699   - if 'access_token' not in self.session:
700   - self.get_access_token()
701   -
702   - self.token = oauth.Token(self.session['access_token']['oauth_token'],
703   - self.session['access_token']['oauth_token_secret'])
704   - self.client = oauth.Client(self.consumer, self.token)
705   -
706   - # Test if access token is working
707   - resp, content = self.client.request('https://api.twitter.com/1/account/verify_credentials.json', 'GET')
708   - if resp['status'] == '401':
709   - p_err('Something is wrong with access token, getting again...\n')
710   - self.get_access_token()
711   - self.token = oauth.Token(self.session['access_token']['oauth_token'],
712   - self.session['access_token']['oauth_token_secret'])
713   - self.client = oauth.Client(self.consumer, self.token)
714   -
715   - def get_access_token(self):
716   -
717   - p('\nGetting Access Token from Twitter...\n\n')
718   -
719   - client = oauth.Client(self.consumer)
720   -
721   -# Step 1: Get a request token. This is a temporary token that is used for
722   -# having the user authorize an access token and to sign the request to obtain
723   -# said access token.
724   -
725   - resp, content = client.request(request_token_url, "GET")
726   - if resp['status'] != '200':
727   - p_err(repr(resp))
728   - p_err(repr(content))
729   - raise Exception("Invalid response %s." % resp['status'])
730   -
731   - request_token = dict(urlparse.parse_qsl(content))
732   -
733   - p("Request Token:\n")
734   - p(" - oauth_token = %s\n" % request_token['oauth_token'])
735   - p(" - oauth_token_secret = %s\n" % request_token['oauth_token_secret'])
736   - p('\n')
737   -
738   -# Step 2: Redirect to the provider. Since this is a CLI script we do not
739   -# redirect. In a web application you would redirect the user to the URL
740   -# below.
741   -
742   - p("Go to the following link in your browser:\n")
743   - p("%s?oauth_token=%s\n" % (authorize_url, request_token['oauth_token']))
744   - p('\n')
745   -
746   -# After the user has granted access to you, the consumer, the provider will
747   -# redirect you to whatever URL you have told them to redirect to. You can
748   -# usually define this in the oauth_callback argument as well.
749   -
750   - accepted = 'n'
751   - while accepted.lower() == 'n':
752   - accepted = raw_input('Have you authorized me? (y/n) ')
753   - oauth_verifier = raw_input('What is the PIN? ')
754   -
755   -# Step 3: Once the consumer has redirected the user back to the oauth_callback
756   -# URL you can request the access token the user has approved. You use the
757   -# request token to sign this request. After this is done you throw away the
758   -# request token and use the access token returned. You should store this
759   -# access token somewhere safe, like a database, for future use.
760   - token = oauth.Token(request_token['oauth_token'],
761   - request_token['oauth_token_secret'])
762   - token.set_verifier(oauth_verifier)
763   - client = oauth.Client(self.consumer, token)
764   -
765   - resp, content = client.request(access_token_url, "POST")
766   - access_token = dict(urlparse.parse_qsl(content))
767   -
768   - p("Access Token:\n")
769   - p(" - oauth_token = %s\n" % access_token['oauth_token'])
770   - p(" - oauth_token_secret = %s\n" % access_token['oauth_token_secret'])
771   - p('\n')
772   - p("You may now access protected resources using the access tokens above.\n" )
773   - p('\n')
774   -
775   - self.session['access_token'] = access_token
776   - self.access_token = access_token
777   -
778   - @staticmethod
779   - def to_localtime(d):
780   -
781   - return datetime(*fp._parse_date(d)[:6]).replace(tzinfo=utc).astimezone(local_tz)
782   -
783   - def get_list(self):
784   -
785   - request_uri = self.REQUEST_URI
786   - if self.last_id:
787   - request_uri += '?since_id=%s' % self.last_id
788   - try:
789   - resp, content = self.client.request(request_uri, 'GET')
790   - if resp['status'] != '200':
791   - p_err("Invalid response %s.\n" % resp['status'])
792   - return
793   - return json.loads(content)
794   - except AttributeError, e:
795   - if repr(e) == """AttributeError("'NoneType' object has no attribute 'makefile'",)""":
796   - # XXX http://code.google.com/p/httplib2/issues/detail?id=62
797   - # Force to reconnect
798   - p_err("AttributeError: 'NoneType' object has no attribute 'makefile'\n")
799   - self.create_connection()
800   - return
801   - else:
802   - raise e
803   -
804   - @safe_update
805   - def update(self):
806   -
807   - if time.time() < self.interval + self.last_accessed:
808   - return
809   - self.last_accessed = time.time()
810   -
811   - msg = self.TPL_ACCESS(ansi=ANSI, src_name=self.src_name, src_id=self.src_id)
812   - p(msg)
813   - statuses = self.get_list()
814   - p_clr(msg)
815   -
816   - if not statuses:
817   - return
818   - self._update_last_id(statuses[0]['id'])
819   -
820   - statuses.reverse()
821   - for status in statuses:
822   - p_dbg('ID: %s' % status['id'])
823   - if self.is_excluded(status):
824   - continue
825   - status['tweet_link'] = 'http://twitter.com/%s/status/%s' % (status['user']['screen_name'], status['id'])
826   - status['created_at'] = self.to_localtime(status['created_at'])
827   - self.process_highlight(status)
828   - print self.output(status=status, src_name=self.src_name, **common_tpl_opts)
829   -
830   -
831   -# FIXME Merge into class Twitter
832   -class TwitterMentions(Twitter):
833   -
834   - TYPE = 'twittermentions'
835   - REQUEST_URI = 'http://api.twitter.com/1/statuses/mentions.json'
836   -
837   - def __init__(self, src):
838   -
839   - super(Twitter, self).__init__(src)
840   -
841   - self.username = src['username']
842   - self.consumer_key = src['consumer_key']
843   - self.consumer_secret = src['consumer_secret']
844   -
845   - self.src_id = self.username
846   - self.src_name = src.get('src_name', 'TwitterMentions')
847   - self.interval = src.get('interval', 90)
848   - self.output = tpl(src.get('output', '@!ansi.fgreen!@@!ftime(status["created_at"], "%H:%M:%S")!@@!ansi.freset!@ [@!src_name!@] @!ansi.fyellow!@@!status["user"]["screen_name"]!@@!ansi.freset!@: @!unescape(status["text"])!@ @!ansi.fmagenta!@@!surl(status["tweet_link"])!@@!ansi.freset!@'), escape=None)
849   -
850   - self._init_session()
851   - self.create_connection()
852   - self._load_last_id()
853   -
854   -
855   -class FriendFeed(Source):
856   -
857   - TYPE = 'friendfeed'
858   -
859   - def __init__(self, src):
860   -
861   - super(FriendFeed, self).__init__(src)
862   -
863   - self.token = None
864   - self.nickname = src['nickname']
865   - self.api = ff.FriendFeed(self.nickname, src['remote_key'])
866   - self.src_id = self.nickname
867   - self.src_name = src.get('src_name', 'FriendFeed')
868   - self.interval = src.get('interval', 60)
869   - self.output = tpl(src.get('output', '@!ansi.fgreen!@@!ftime(entry["updated"], "%H:%M:%S")!@@!ansi.freset!@ [@!src_name!@] @!ansi.fyellow!@@!entry["user"]["nickname"]!@@!ansi.freset!@:<!--(if "room" in entry)--> @!ansi.fiyellow!@[@!entry["room"]["name"]!@]@!ansi.freset!@<!--(end)--> @!ansi.fcyan!@@!entry["title"]!@@!ansi.freset!@ @!ansi.fmagenta!@@!surl(entry["_link"])!@@!ansi.freset!@'), escape=None)
870   - self.output_like = tpl(src.get('output_like', '@!ansi.fgreen!@@!ftime(like["date"], "%H:%M:%S")!@@!ansi.freset!@ [@!src_name!@] @!ansi.fyellow!@@!like["user"]["nickname"]!@@!ansi.freset!@ @!ansi.fired!@♥@!ansi.freset!@ @!ansi.fcyan!@@!entry["title"]!@@!ansi.freset!@ @!ansi.fmagenta!@@!surl(entry["_link"])!@@!ansi.freset!@'), escape=None)
871   - self.output_comment = tpl(src.get('output_comment', '@!ansi.fgreen!@@!ftime(comment["date"], "%H:%M:%S")!@@!ansi.freset!@ [@!src_name!@] @!ansi.fyellow!@@!comment["user"]["nickname"]!@@!ansi.freset!@ ✎ @!ansi.fcyan!@@!entry["title"]!@@!ansi.freset!@: @!comment["body"]!@ @!ansi.fmagenta!@@!surl(entry["_link"])!@@!ansi.freset!@'), escape=None)
872   - self.show_like = src.get('show_like', True)
873   - self.show_comment = src.get('show_comment', True)
874   - self.show_hidden = src.get('show_hidden', False)
875   -
876   - @staticmethod
877   - def to_localtime(d):
878   -
879   - return d.replace(tzinfo=utc).astimezone(local_tz)
880   -
881   - @safe_update
882   - def update(self):
883   -
884   - if time.time() < self.interval + self.last_accessed:
885   - return
886   - self.last_accessed = time.time()
887   -
888   - msg = self.TPL_ACCESS(ansi=ANSI, src_name=self.src_name, src_id=self.src_id)
889   - p(msg)
890   - if not self.token:
891   - self.token = self.api.fetch_updates()['update']['token']
892   -
893   - home = self.api.fetch_updates_home(token=self.token, timeout=0)
894   - p_clr(msg)
895   -
896   - self.token = home['update']['token']
897   -
898   - entries = home['entries']
899   - for entry in entries:
900   - if entry['hidden'] and not self.show_hidden:
901   - continue
902   - entry['_link'] = 'http://friendfeed.com/e/' + entry["id"]
903   - if entry['is_new']:
904   - entry['updated'] = self.to_localtime(entry['updated'])
905   - print self.output(entry=entry, src_name=self.src_name, **common_tpl_opts)
906   -
907   - if self.show_like:
908   - for like in entry['likes']:
909   - if like['is_new']:
910   - like['date'] = self.to_localtime(like['date'])
911   - print self.output_like(like=like, entry=entry, src_name=self.src_name, **common_tpl_opts)
912   -
913   - if self.show_comment:
914   - for comment in entry['comments']:
915   - if comment['is_new']:
916   - comment['date'] = self.to_localtime(comment['date'])
917   - print self.output_comment(comment=comment, entry=entry, src_name=self.src_name, **common_tpl_opts)
918   -
919   -
920   -class Feed(Source):
921   -
922   - TYPE = 'feed'
923   -
924   - def __init__(self, src):
925   -
926   - super(Feed, self).__init__(src)
927   -
928   - self.feed = src['feed']
929   - # Used as key to store session data
930   - self.src_id = self.feed
931   - self.src_name = src.get('src_name', 'Feed')
932   - self.interval = src.get('interval', 60)
933   - self.output = tpl(src.get('output', '@!ansi.fgreen!@@!ftime(entry["updated"], "%H:%M:%S")!@@!ansi.freset!@ [@!src_name!@] @!entry["title"]!@ @!ansi.fmagenta!@@!surl(entry.link)!@@!ansi.fmagenta!@@!ansi.freset!@@!ansi.freset!@'), escape=None)
934   - # XXX
935   - if 'say' in src:
936   - self.say = tpl(src['say'])
937   -
938   - self._init_session()
939   - self._load_check_list()
940   -
941   - def datetimeize(self, entry):
942   - '''Replace date with parsed date then do Source.datetimeize'''
943   - for key in ['updated', 'published', 'created', 'expired']:
944   - if key + '_parsed' in entry:
945   - entry[key] = entry[key + '_parsed']
946   - del entry[key + '_parsed']
947   - Source.datetimeize(self, entry)
948   -
949   - def get_list(self):
950   -
951   - return fp.parse(self.feed)
952   -
953   -
954   -class Craigslist(Feed):
955   -
956   - TYPE = 'cl'
957   - ID_RE = re.compile('.*?(\d+)\.html')
958   -
959   - def __init__(self, src):
960   -
961   - super(Feed, self).__init__(src)
962   -
963   - self.feed = src['feed']
964   - # Used as key to store session data
965   - self.src_id = self.feed
966   - self.src_name = src.get('src_name', self.TYPE)
967   - self.interval = src.get('interval', 60)
968   - self.output = tpl(src.get('output', '@!ansi.fgreen!@@!ftime(entry["updated"], "%H:%M:%S")!@@!ansi.freset!@ [@!src_name!@] @!entry["title"]!@ @!ansi.fmagenta!@@!surl(entry.link)!@@!ansi.fmagenta!@@!ansi.freset!@@!ansi.freset!@'), escape=None)
969   - # XXX
970   - if 'say' in src:
971   - self.say = tpl(src['say'])
972   -
973   - self._init_session()
974   - self._load_check_list()
975   - if isinstance(self.check_list, dict):
976   - self.check_list = 0
977   - # Use it to store check_list for comparison of a list of entries
978   - self._check_list = None
979   -
980   - def _update_check_list(self):
981   - # It is not a list but a single int
982   - self._check_list = None
983   - self.session['check_list'] = self.check_list
984   - session[self.session_id] = self.session
985   - p_dbg('Updated [%s] check_list' % self.session_id)
986   -
987   - @classmethod
988   - def get_entry_id(cls, entry):
989   -
990   - m = cls.ID_RE.match(entry['guid'])
991   - if not m:
992   - raise ValueError('Craiglist should have guid')
993   -
994   - return int(m.group(1))
995   -
996   - def is_new_item(self, entry):
997   - '''Check if entry is new and also update check_list if it is new'''
998   - e_id = self.get_entry_id(entry)
999   - if self._check_list is None:
1000   - self._check_list = self.check_list
1001   - if e_id > self._check_list:
1002   - if e_id > self.check_list:
1003   - self.check_list = e_id
1004   - return True
1005   - return False
1006   -
1007   - def get_list(self):
1008   -
1009   - feed = super(Craigslist, self).get_list()
1010   -
1011   - new_entries = []
1012   - for entry in feed['entries']:
1013   - # Some entries didn't have updated or published, they are very old
1014   - # entries, need to be filtered out.
1015   - if entry.published:
1016   - new_entries.append(entry)
1017   -
1018   - feed['entries'] = new_entries
1019   - return feed
1020   -
1021   -
1022   -class FlickrContacts(Craigslist):
1023   -
1024   - TYPE = 'frck'
1025   -
1026   - @classmethod
1027   - def get_entry_id(cls, entry):
1028   -
1029   - return super(Feed, cls).get_entry_id(entry)
1030   -
1031   -
1032   -class StackOverflowNewOnly(Craigslist):
1033   -
1034   - TYPE = 'sono'
1035   - ID_RE = re.compile('.*?(\d+)/[a-zA-Z0-9-]+')
1036   -
1037   - @classmethod
1038   - def get_entry_id(cls, entry):
1039   -
1040   - m = cls.ID_RE.match(entry['id'])
1041   - if not m:
1042   - raise ValueError('StackOverflow should have guid')
1043   -
1044   - return int(m.group(1))
1045   -
1046   -
1047   -class TwitterSearch(Feed):
1048   -
1049   - TYPE = 'twittersearch'
1050   - SEARCH_URL = 'http://search.twitter.com/search.atom'
1051   - # For reseting only
1052   - PUBLIC_URL = 'http://twitter.com/statuses/public_timeline.json'
1053   - RE_LINK = re.compile(u'(.*?)<a href="(.*?)">(.*?)</a>(.*)', re.DOTALL)
1054   -
1055   - def __init__(self, src):
1056   -
1057   - # Skip feed
1058   - super(Feed, self).__init__(src)
1059   -
1060   - self.src_name = src.get('src_name', 'TwitterSearch')
1061   - self.interval = src.get('interval', 60)
1062   - self.output = tpl(src.get('output', '@!ansi.fgreen!@@!ftime(entry["published"], "%H:%M:%S")!@@!ansi.freset!@ [@!src_name!@] @!ansi.fyellow!@@!entry["author"]["screen_name"]!@@!ansi.freset!@: @!remove_additional_space(entry["title"])!@ @!ansi.fmagenta!@@!surl(entry["link"])!@@!ansi.freset!@'), escape=None)
1063   - self.q = src['q']
1064   - self.lang = src.get('lang', 'en')
1065   - self.src_id = '%s:%s' % (self.lang, self.q)
1066   - self.rpp = src.get('rpp', 15)
1067   -
1068   - self._init_session()
1069   - # Do not load last_id, so we can have a clean-run searching
1070   - # self._load_last_id()
1071   - self.last_id = None
1072   - self.update(suppress=True)
1073   -
1074   - def get_list(self):
1075   -
1076   - parameters = {'q': self.q, 'lang': self.lang, 'rpp': self.rpp, 'result_type': 'recent'}
1077   - if self.last_id:
1078   - parameters['since_id'] = self.last_id
1079   - feed = fp.parse(self.SEARCH_URL + '?' + urllib.urlencode(parameters))
1080   - try:
1081   - if 'status' not in feed:
1082   - p_err('No key status in feed: %s\n' % feed['bozo_exception'])
1083   - return
1084   - if feed['status'] == 403 or feed['status'] == 404:
1085   - p_err('Got 403 or 404\n')
1086   - return
1087   - elif feed['status'] == 503:
1088   - p_err('HTTP Status 503\n')
1089   - return
1090   - except AttributeError, e:
1091   - p_dbg(repr(feed))
1092   - raise e
1093   -
1094   - if not feed['feed']:
1095   - # The feed (since_id) is expired, feed.entries is [], so just return
1096   - return feed
1097   - # XXX
1098   - try:
1099   - for link in feed.feed.links:
1100   - if link.rel == 'refresh':
1101   - self._update_last_id(link.href.rsplit('=', 1)[1])
1102   - break
1103   - except AttributeError, e:
1104   - print feed
1105   - raise e
1106   -
1107   - new_entries = []
1108   - for entry in feed['entries']:
1109   - screen_name, name = entry['author'].split(' ', 1)
1110   - entry['author'] = {'screen_name': screen_name, 'name': name[1:-1]}
1111   - if self.is_excluded(entry):
1112   - continue
1113   - new_entries += [entry]
1114   - feed['entries'] = new_entries
1115   -
1116   - return feed
1117   -
1118   - @safe_update
1119   - def update(self, suppress=False):
1120   - # suppress is for first seach of session, user may not want to read lots of
1121   - # tweets when they just run clis.py
1122   -
1123   - if time.time() < self.interval + self.last_accessed:
1124   - return
1125   - self.last_accessed = time.time()
1126   -
1127   - msg = self.TPL_ACCESS(ansi=ANSI, src_name=self.src_name, src_id=self.src_id)
1128   - p(msg)
1129   - feed = self.get_list()
1130   - if feed is None:
1131   - return
1132   - p_clr(msg)
1133   - if suppress:
1134   - return
1135   -
1136   - if not feed['entries']:
1137   - return
1138   -
1139   - entries = feed['entries']
1140   - # Get entries after last_id
1141   - for entry in entries:
1142   - self.datetimeize(entry)
1143   -
1144   - entries.reverse()
1145   - for entry in entries:
1146   - p_dbg('ID: %s' % self.get_entry_id(entry))
1147   - # XXX make this a method and move to base class, and also should support re
1148   - if self.is_excluded(entry):
1149   - continue
1150   - self.process_highlight(entry)
1151   - print self.output(entry=entry, src_name=self.src_name, **common_tpl_opts)
1152   -
1153   -
1154   -# http://stackoverflow.com/questions/52880/google-reader-api-unread-count
1155   -class GoogleBase(Feed):
1156   -
1157   - def __init__(self, src):
1158   -
1159   - # Skip Feed
1160   - super(Feed, self).__init__(src)
1161   -
1162   - auth_url = 'https://www.google.com/accounts/ClientLogin'
1163   - self.email = src['email']
1164   - auth_req_data = urllib.urlencode({
1165   - 'accountType': 'GOOGLE',
1166   - 'Email': self.email,
1167   - 'Passwd': src['password'],
1168   - 'service': 'reader',
1169   - 'source': 'YJL-clis-0',
1170   - })
1171   - auth_req = urllib2.Request(auth_url, data=auth_req_data)
1172   - auth_resp = urllib2.urlopen(auth_req)
1173   - auth_resp_content = auth_resp.read()
1174   - p_dbg(auth_resp_content)
1175   - auth_resp_dict = dict(x.split('=') for x in auth_resp_content.split('\n') if x)
1176   - p_dbg(auth_resp_dict)
1177   - self.Auth = auth_resp_dict["Auth"]
1178   -
1179   - def get(self, url, header=None):
1180   -
1181   - if header is None:
1182   - header = {}
1183   -
1184   - header['Authorization'] = 'GoogleLogin auth=%s' % self.Auth
1185   -
1186   - req = urllib2.Request(url, None, header)
1187   - f = urllib2.urlopen(req)
1188   - content = f.read()
1189   - f.close()
1190   - return content
1191   -
1192   -
1193   -class GoogleMail(Feed):
1194   -
1195   - TYPE = 'gmail'
1196   -
1197   - def __init__(self, src):
1198   -
1199   - # Skip Feed
1200   - super(Feed, self).__init__(src)
1201   -
1202   - self.email = src['email']
1203   - self.password = src['password']
1204   - self.src_id = self.email
1205   - self.src_name = src.get('src_name', 'Gmail')
1206   - self.interval = src.get('interval', 60)
1207   - self.output = tpl(src.get('output', '@!ansi.fgreen!@@!ftime(entry["updated"], "%H:%M:%S")!@@!ansi.freset!@ @!ansi.fred!@[@!src_name!@]@!ansi.freset!@ @!ansi.fyellow!@@!entry["author"]!@@!ansi.freset!@: @!entry["title"]!@ @!ansi.fmagenta!@@!surl(entry["link"])!@@!ansi.freset!@'), escape=None)
1208   -
1209   - self._init_session()
1210   - self._load_check_list()
1211   -
1212   - def get_list(self):
1213   -
1214   - # FIXME Check if we can use ClientLogin, don't like password being stored
1215   - feed = fp.parse('https://%s:%s@mail.google.com/mail/feed/atom' % (urllib.quote(self.email), urllib.quote(self.password)))
1216   - return feed
1217   -
1218   -
1219   -class GoogleReader(GoogleBase):
1220   -
1221   - TYPE = 'greader'
1222   - # Google Reader has this attribute to entry entitiy, this is a reliable
1223   - # source to check if item is new.
1224   - RE_CRAWL_TIME = re.compile(r'gr:crawl-timestamp-msec="(\d+)"')
1225   -
1226   - def __init__(self, src):
1227   -
1228   - super(GoogleReader, self).__init__(src)
1229   -
1230   - self.src_id = self.email
1231   - self.src_name = src.get('src_name', 'GR')
1232   - self.interval = src.get('interval', 60)
1233   - self.output = tpl(src.get('output', '@!ansi.fgreen!@@!ftime(entry["updated"], "%H:%M:%S")!@@!ansi.freset!@ [@!src_name!@] @!ansi.fyellow!@@!entry["source"]["title"]!@@!ansi.freset!@: @!entry["title"]!@ @!ansi.fmagenta!@@!surl(entry["link"])!@@!ansi.freset!@'), escape=None)
1234   -
1235   - self._init_session()
1236   - self._load_check_list()
1237   -
1238   - def get_list(self):
1239   - '''Retrieve Google Reader feed, and replace published date with crawl
1240   - time'''
1241   - # Get the last 50 items