Permalink
Browse files

Initial commit. Working but needs cleanup.

  • Loading branch information...
inactivist committed Nov 9, 2012
0 parents commit b88f1a47b65ef4d5771ab2543ff4f23c25943d48
@@ -0,0 +1,7 @@
+.project
+.pydevproject
+dev.ini
+*.cfg
+*.pyc
+
+
21 LICENSE
@@ -0,0 +1,21 @@
+MIT License
+Copyright (c) 2012 Exodus Development, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,55 @@
+Twitter Streamer is a command-line utility to dump [Twitter streaming API][1] data
+to stdout. It uses the [statuses/filter][3] method.
+
+NOTE: Output formats are lame and will change at some point soon.
+
+ usage: streamer.py [-h] [-c CONFIG_FILE] [-l LOG_LEVEL] [-v] [-r REPORT_LAG]
+ [-u USER_LANG] [-i] [-f FIELD]
+ track [track ...]
+
+The *track* parameters provides [track search terms][2] for the Twitter
+statuses/filter API. Commas denote an *or* relationship, while spaces
+denote an *and* relationship.
+
+You can provide multiple *track* parameters, which will expand the search terms.
+
+Please refer to the [track][2] documentation for specific limitations and
+usage examples.
+
+## Examples ##
+Filter statuses containing both *car* **and** *dog*:
+ python streamer.py "car dog"
+
+Filter statuses containing either *boat* **or** *bike*:
+ python streamer.py "boat,bike"
+
+Filter statuses containing (*water* **and** *drink*) *or* (*eat* **and** *lunch*):
+ python streamer.py "water drink" "eat lunch"
+
+
+## Output Format ##
+The normal output mode is to dump the raw stream data text for each received
+stream element, followed by a newline.
+
+Each element is a well-formed JSON object, but at present the
+stream elements are not separated by commas, nor is the stream wrapped in a JSON
+list. Therefore, if you expect to parse the output of this program as JSON
+data, you will need to preprocess it in order to be well-formed JSON. You should
+be able to split the output on newlines, inserting commas between records.
+
+## Experimental Features ##
+
+The *-f* or *--field=* option allows a comma-separated list of output fields.
+The field values will be emitted in the order listed in the given *-f* or *--field=*
+parameter. The output is not formatted in any other way.
+
+Example:
+ python streamer.py -f created_at,text elections
+
+Example results:
+ 2012-11-09 20:26:47 Volatility the Likely Outcome of Elections http://t.co/trmmSpXp #Barron's
+ 2012-11-09 20:26:50 @WHLive then why the president ordered Boeing to release the layoff news AFTER the elections?
+
+[1]: https://dev.twitter.com/docs/streaming-apis
+[2]: https://dev.twitter.com/docs/streaming-apis/parameters#track
+[3]: https://dev.twitter.com/docs/api/1.1/post/statuses/filter
@@ -0,0 +1,3 @@
+* Improve character encoding support.
+* Better output formatting options.
+* Unit tests.
@@ -0,0 +1,3 @@
+simplejson
+tweepy
+
@@ -0,0 +1,2 @@
+__version__ = "0.0.1"
+
@@ -0,0 +1,20 @@
+"""Config utilities and variables.
+"""
+import ConfigParser
+
+
+class DictConfigParser(ConfigParser.ConfigParser):
+ """Provides dictionary access to ConfigParser items.
+
+ Inspired by http://stackoverflow.com/a/3220891/1309774
+ """
+ def __init__(self):
+ ConfigParser.ConfigParser.__init__(self)
+ self.optionxform = str
+
+ def as_dict(self):
+ d = dict(self._sections)
+ for k in d:
+ d[k] = dict(self._defaults, **d[k])
+ d[k].pop('__name__', None)
+ return d
@@ -0,0 +1,6 @@
+[twitter_api]
+# You must use the keys provided by Twitter for your account.
+consumer_key: YOUR_CONSUMER_KEY
+consumer_secret: YOUR_CONSUMER_SECRET
+access_token_key: YOUR_ACCESS_TOKEN_KEY
+access_token_secret: YOUR_ACCESS_TOKEN_SECRET
No changes.
@@ -0,0 +1,48 @@
+"""
+ Tweepy streaming API StreamListener message recognizers.
+
+ These recognize various types of incoming data based on predefined rules.
+
+ TODO: Probably don't need classes. Consider refactoring.
+"""
+import tweepy
+import simplejson as json
+
+
+class MessageRecognizer(object):
+ def __init__(self, handler_method):
+ self.handler = handler_method
+
+ def match(self, stream_data):
+ """Return True if this recognizer matches the incoming stream data."""
+ return False
+
+ def handle_message(self, stream_data):
+ "Handle the incoming message. Return False to stop stream processing."
+ return self.handler(stream_data)
+
+
+class MatchAnyRecognizer(MessageRecognizer):
+ """Match any stream_data."""
+ def match(self, stream_data):
+ return True
+
+
+class DataStartsWithRecognizer(MessageRecognizer):
+ def __init__(self, handler_method, starts_with):
+ self.starts_with = starts_with
+ super(DataStartsWithRecognizer, self).__init__(handler_method)
+
+ def match(self, stream_data):
+ return stream_data.startswith(self.starts_with)
+
+
+class DataContainsRecognizer(MessageRecognizer):
+ """A simple string-in-message recognizer. Matches if a string is anywhere
+ in the stream_data body."""
+ def __init__(self, handler_method, match_string):
+ self.contains_string = match_string
+ super(DataContainsRecognizer, self).__init__(handler_method)
+
+ def match(self, stream_data):
+ return self.contains_string in stream_data
@@ -0,0 +1,8 @@
+import pstats
+SORT = 'cumulative'
+#SORT = 'time'
+LIMIT = 50
+p = pstats.Stats('profile2.dat')
+p.sort_stats(SORT).print_stats(LIMIT) #.print_callees(LIMIT)
+
+
@@ -0,0 +1,3 @@
+#!/bin/bash
+# Print Twitter server time and local system time for comparison
+lynx --head --dump https://api.twitter.com/1/help/test.json | grep Date: ; echo Date: `date -u +"%a, %d %b %Y %T %z"` '(local system)'
Oops, something went wrong.

0 comments on commit b88f1a4

Please sign in to comment.