Permalink
Browse files

Twitter entities added

  • Loading branch information...
1 parent 2bdd17e commit d737566907544f2160328b1b2062b7ebbfe3aae5 @webisteme committed May 10, 2012
View
@@ -42,7 +42,10 @@ PunkMoney has two parts: a tracker for finding, interpreting and storing #PunkMo
Web interface (Django):
-(1) Create a MySQL database (UTF-8 charset)
+(1) Create a MySQL database (UTF-8 charset):
+
+CREATE DATABASE punkmoney CHARACTER SET utf8 COLLATE utf8_general_ci;
+
(2) Create your settings in /web/settings_template.py, then rename to settings.py. Be sure to add a template path (an absolute path to the template directory,) and your MySQL database credentials.
(3) Run python manage.py syncdb to create the necessary tables
(4) Run python manage.py runserver to check it's set up correctly.
View
@@ -28,7 +28,7 @@ def run(self):
try:
self.updateExpired()
except Exception, e:
- self.logError("Updatine expired failed: %s" % e)
+ self.logError("Updating expired failed: %s" % e)
# If more than 25 hits remaining, harvest new tweets
View
@@ -10,8 +10,12 @@
from config import TW_CONSUMER_KEY, TW_CONSUMER_SECRET, TW_ACCESS_KEY, TW_ACCESS_SECRET, HASHTAG, ALT_HASHTAG
import tweepy
-
from datetime import datetime
+import sys
+import urllib2
+from urllib import urlencode
+import simplejson
+from pprint import pprint
class Harvester(Connection):
@@ -44,22 +48,43 @@ def harvestNew(self):
# Save to DB
i = 0
for tweet in tweets:
-
+
# Double check tweet isn't duplicate
query = "SELECT tweet_id FROM tracker_tweets WHERE tweet_id = %s" % tweet.id
if self.getSingleValue(query) is None:
- # Check for a reply_to_id
- t = self.TW.get_status(tweet.id)
- reply_to_id = t.in_reply_to_status_id
-
+ # Get tweet metadata
+ t = self.getStatus(tweet.id)
+
+ #pprint(t)
+
+ # Extract first img_url
+ img_url = None
+ for media_entity in t.get('entities', {}).get('media', []):
+ if media_entity.get('type', None) == 'photo':
+ img_url = media_entity.get('media_url', None)
+
+ # Extract first url
+ url = None; display_url = None
+ for url_entity in t.get('entities', {}).get('urls', []):
+ url = url_entity.get('expanded_url', None)
+ display_url = url_entity.get('display_url', None)
+
+ # Extract hashtags
+ tags = {}; k = 0
+ for hashtag in t.get('entities', {}).get('hashtags', []):
+ tags[k] = hashtag.get('text')
+ k = k + 1
+
+ reply_to_id = t.get('in_reply_to_status_id', None)
+
# Save data
self.logInfo("Saving tweet %s to database" % tweet.id)
- query = "INSERT INTO tracker_tweets (timestamp, tweet_id, author, content, reply_to_id) VALUES (%s, %s, %s, %s, %s)"
- params = (tweet.created_at, tweet.id, tweet.from_user.lower(), tweet.text, reply_to_id)
-
+ query = "INSERT INTO tracker_tweets (timestamp, tweet_id, author, content, reply_to_id, url, display_url, img_url) VALUES (%s, %s, %s, %s, %s, %s, %s, %s)"
+ params = (tweet.created_at, tweet.id, tweet.from_user.lower(), tweet.text, reply_to_id, url, display_url, img_url)
+
self.queryDB(query, params)
i = i + 1
@@ -88,3 +113,38 @@ def connectTwitter(self):
else:
return api
+ # getTweets()
+ # Gets data tweet data directly from the Twitter API
+ def getStatus(self, tweet_id):
+ try:
+ url = 'http://api.twitter.com/1/statuses/show/%s.json' % tweet_id
+
+ # Set params
+ params = {
+ 'include_entities' : '1',
+ }
+
+ query_url = url + '?' + urlencode(params)
+ raw_data = urllib2.urlopen(query_url)
+
+ data = {}
+ for d in raw_data:
+ data = simplejson.loads(d)
+
+ except Exception, e:
+ raise Exception("Error connecting to Twitter API: %s" % e)
+ else:
+ return data
+
+
+
+
+
+
+
+
+
+
+
+
+
View
@@ -61,6 +61,11 @@ def parseNew(self):
self.setParsed(tweet['tweet_id'], '-')
raise Exception("Tweet was not recognised")
+ # strip urls from text
+ r = re.search("(.*)(?P<url>https?://[^\s]+)(.*)", tweet['content'], re.IGNORECASE)
+ if r:
+ tweet['content'] = r.group(1) + ' ' + r.group(3)
+
except Exception, e:
self.logWarning("Parsing tweet %s failed: %s" % (tweet['tweet_id'], e))
self.setParsed(tweet['tweet_id'], '-')
View
@@ -52,18 +52,36 @@ <h3 style='display:inline'><span id="promise">{{ note.promise }}</span></h3>
{% endifequal %}
-
- <p>Issuer: <b><a href="/user/{{ note.issuer }}">{{ note.issuer }}</a></b> <br>
+ <hr class="space">
+
<p>Created: <b>{{ note.created }}</b><br>
Expiry: <b>{{ note.expiry }}</b></p>
-
+
+ {% if url %}
+ Link: <a href="{{ url }}"><b>{{ display_url }}</b></a><br>
+ {% endif %}
+
+
</div>
<!-- RIGHT COLUMN -->
<div id="right" class="span-7 prepend-1 last">
+
+ {% if img_url %}
+
+ {% if url %}
+ <a href="{{ url }}" target="_blank"><img src="{{ img_url }}" height="100" align="centre"></a>
+ {% else %}
+ <a href="{{ img_url }}" target="_blank"><img src="{{ img_url }}" height="100" align="centre"></a>
+ {% endif %}
+
+ <hr class="space">
+
+ {% endif %}
+
<h3><i>"{{ content }}"</i></h3>
<b><a href="/user/{{ note.issuer }}">{{ note.issuer }}</a></b>
View
@@ -66,8 +66,6 @@ <h3 style='display:inline'><span id="expiration">{{ note.promise }}</span></h3>
Condition: <b>{{ note.conditional }}</b>
{% endif %}
-
- <hr class="space">
<p>Created: <b>{{ note.created }}</b><br>
Expiry: <b>{{ note.expiry }}</b></p>
@@ -80,6 +78,18 @@ <h3 style='display:inline'><span id="expiration">{{ note.promise }}</span></h3>
<div id="right" class="span-7 prepend-1 last">
+ {% if img_url %}
+
+ {% if url %}
+ <a href="{{ url }}" target="_blank"><img src="{{ img_url }}" height="100" align="centre"></a>
+ {% else %}
+ <a href="{{ img_url }}" target="_blank"><img src="{{ img_url }}" height="100" align="centre"></a>
+ {% endif %}
+
+ <hr class="space">
+
+ {% endif %}
+
<h3><i>"{{ content }}"</i></h3>
<b><a href="/user/{{ note.issuer }}">{{ note.issuer }}</a></b>
@@ -90,11 +100,8 @@ <h3 style='display:inline'><span id="expiration">{{ note.promise }}</span></h3>
{% ifequal note.status 0 %}
-
-
<a href="https://twitter.com/intent/tweet?in_reply_to={{ note.id }}&status=@{{note.issuer}}%20thanks%20%23punkmoney"><span id="redemption">[Redeemed]</span></a> &nbsp;
-
{% if note.transferable %}
<a href="https://twitter.com/intent/tweet?in_reply_to={{ note.id }}&status=@{{note.issuer}}%20transfer%20@USERNAME%20%23punkmoney"><span id="transfer">[Transfer]</span></a>
View
@@ -54,36 +54,40 @@ <h3 style='display:inline'><span id="promise">{{ note.promise }}</span></h3>
<!-- <hr class="space"> -->
- <p>Issuer: <b><a href="/user/{{ note.issuer }}">{{ note.issuer }}</a></b> <br>
+ <p>Issuer: <b><a href="/user/{{ note.issuer }}">{{ note.issuer }}</a></b></p>
<!-- CONDITION -->
- {% if note.conditional %}
-
- Condition: <b>{{ note.conditional }}</b>
-
+ {% if url %}
+ Url: <a href="{{ url }}">{{ display_url }}</a>
{% endif %}
-
<p>Created: <b>{{ note.created }}</b><br>
Expiry: <b>{{ note.expiry }}</b></p>
+
+ {% if note.conditional %}
+ <p>Condition: <b>{{ note.conditional }}</b></p>
+ {% endif %}
</div>
-
-
-
-
-
-
-
-
-
<!-- RIGHT COLUMN -->
<div id="right" class="span-7 prepend-1 last">
+ {% if img_url %}
+
+ {% if url %}
+ <a href="{{ url }}" target="_blank"><img src="{{ img_url }}" height="100" align="centre"></a>
+ {% else %}
+ <a href="{{ img_url }}" target="_blank"><img src="{{ img_url }}" height="100" align="centre"></a>
+ {% endif %}
+
+ <hr class="space">
+
+ {% endif %}
+
<h3><i>"{{ content }}"</i></h3>
<b><a href="/user/{{ note.issuer }}">{{ note.issuer }}</a></b>
View
@@ -72,6 +72,18 @@ <h3 style='display:inline'><span id="promise">{{ note.promise }}</span></h3>
<div id="right" class="span-7 prepend-1 last">
+ {% if img_url %}
+
+ {% if url %}
+ <a href="{{ url }}" target="_blank"><img src="{{ img_url }}" height="100" align="centre"></a>
+ {% else %}
+ <a href="{{ img_url }}" target="_blank"><img src="{{ img_url }}" height="100" align="centre"></a>
+ {% endif %}
+
+ <hr class="space">
+
+ {% endif %}
+
<h3><i>"{{ content }}"</i></h3>
<b><a href="/user/{{ note.issuer }}">{{ note.issuer }}</a></b>
View
@@ -39,7 +39,10 @@ class tweets(models.Model):
author = models.CharField(max_length=90, blank=True)
content = models.CharField(max_length=420, blank=True)
reply_to_id = models.BigIntegerField(null=True, blank=True)
- parsed = models.CharField(max_length=1, null=True, blank=True) #! fix
+ parsed = models.CharField(max_length=1, null=True, blank=True)
+ url = models.CharField(max_length=420, null=True, blank=True)
+ display_url = models.CharField(max_length=420, null=True, blank=True)
+ img_url = models.CharField(max_length=420, null=True, blank=True)
class Meta:
db_table = u'tracker_tweets'
View
@@ -188,7 +188,10 @@ def getnote(request, noteid):
'events':new_events,
'note':note,
'content':tweet.content,
- 'id':id
+ 'url' : tweet.url,
+ 'display_url' : tweet.display_url,
+ 'id':id,
+ 'img_url' : tweet.img_url,
}
if note.type == 0:

0 comments on commit d737566

Please sign in to comment.