Permalink
Browse files

Run 0.4-add-feed-meta.py with this checkin.

Added Webpage model. Adding ability to add static chunks of text or
HTML before and after the stream and profile widget. Setting
filename for logging. Consolidated tidy_up and limiting entries
to PATCHOULI_TLDR during display.
  • Loading branch information...
ozten committed Jun 27, 2010
1 parent 502967e commit 2d3ba1e7d2e8789838a45ebef1fe7ca4ffb5163c
View
@@ -181,10 +181,8 @@ def cron_fetch_feeds():
log.exception(e)
finally:
lock.close()
- else:
- log.debug("Without caching")
log.info("Finished run in %f seconds for %d new entries" % ((time.time() - start), new_entry_count))
return 'Finished importing %d items' % new_entry_count
if __name__ == '__main__':
- cron_fetch_feeds()
+ cron_fetch_feeds()
@@ -0,0 +1,24 @@
+from django.core.management import setup_environ
+import settings
+setup_environ(settings)
+
+import logging
+logging.basicConfig(level = logging.DEBUG,
+ format = '%(asctime)s %(levelname)s %(process)d %(message)s', )
+log = logging.getLogger()
+
+from lifestream.models import Stream
+from lifestream.models import Webpage
+import lifestream.models
+
+import patchouli_auth.preferences
+
+streams = Stream.objects.all()
+for stream in streams:
+ log.info("Checking page exists for %s owned by %s" % (stream.name, stream.user.username))
+ try:
+ Webpage.objects.get(name=stream.name, user=stream.user)
+ except Webpage.DoesNotExist:
+ webpage = Webpage(name=stream.name, user=stream.user, config='{}')
+ patchouli_auth.preferences.savePageOrStreamProperties(
+ webpage, patchouli_auth.preferences.getPageProperties(webpage))
@@ -3,10 +3,11 @@
import lxml.etree
import lxml.html.soupparser
+from django.conf import settings
+
from bleach import Bleach
bleach = Bleach()
-
def tidy_up(entry, log):
# TODO Security, mostly using bleach to linkify and cleanup (tidy style)
html_tags = ['a', 'abbr', 'b', 'blockquote', 'br',
@@ -37,16 +38,15 @@ def tidy_up(entry, log):
'span': basic_attrs,
'p': basic_attrs,
- }
+ }
try:
+ # Bugfix wrap content in <div> and then pop it out, othewise 'foo <span>bar</span>' will fail
+ htmlElement = lxml.html.soupparser.fromstring("<div>%s</div>" % entry[0:settings.PATCHOULI_TLDR])
+ elements = ''.join([lxml.html.tostring(el) for el in htmlElement.getchildren()])
- htmlElement = lxml.html.soupparser.fromstring(entry)
- if htmlElement.getchildren():
- elements = ''.join([lxml.html.tostring(el) for el in htmlElement.getchildren()])
- else:
- elements = entry
+ # <div> - 5 </div> - 6 characters
return bleach.linkify(
- bleach.clean(elements, tags=html_tags, attributes=attrs))
+ bleach.clean(elements[5:-6], tags=html_tags, attributes=attrs))
except Exception, x:
log.error("Ouch, unable to linkify or clean _%s_\nError: %s" % (entry, x))
log.exception(x)
@@ -59,15 +59,22 @@ def prepare_entry(entryJSON, log):
elif 'description' in entryJSON:
content = entryJSON['description']
else:
- #log.debug('unreadable... ' + str(entryJSON))
+ log.debug('unreadable... ' + str(entryJSON))
pass
title = tidy_up(entryJSON['title'], log)
content = tidy_up(content, log)
+
+ # Generic image in feed?
+ image = None
+ if 'links' in entryJSON:
+ for link in entryJSON['links']:
+ if link['rel'] == 'image':
+ image = link['href']
tags = []
if 'tags' in entryJSON:
for tag in entryJSON['tags']:
if 'term' in tag:
tags.append({'tag': tag['term'], 'name': tag['term']})
- return {'entry': content, 'tags': tags, 'title': title, 'permalink': entryJSON['link'], 'raw': entryJSON}
+ return {'entry': content, 'tags': tags, 'title': title, 'permalink': entryJSON['link'], 'raw': entryJSON, 'image': image}
@@ -1,46 +1,4 @@
-from bleach import Bleach
-bleach = Bleach()
-
-import re
-
-def tidy_up(entry, log):
- # TODO Security, mostly using bleach to linkify and cleanup (tidy style)
- html_tags = ['a', 'abbr', 'b', 'blockquote', 'br',
- 'cite', 'code', 'dd', 'dl', 'div', 'dt',
- 'em', 'font', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
- 'i', 'img', 'hr',
- 'math', 'mi', 'mo', 'mn', 'mfrac', 'mrow', 'msqrt', 'msup',
- 'pre', 'span', 'strong',
- 'svg', 'path', 'line', 'circle',
- 'strike', 'strong', 'sub'
- 'table', 'caption', 'thead', 'tfoot', 'tbody', 'tr', 'td', 'th', 'colgroup', 'col',
- 'tt', 'var',
- 'ul', 'li', 'ol', 'p', 'q']
-
- a_attrs = ['href', 'rel', 'title']
- img_attrs = ['align', 'alt', 'border', 'height','src', 'width']
- basic_attrs = ['class', 'dir', 'lang', 'title']
-
- [x.extend(basic_attrs) for x in (a_attrs, img_attrs)]
-
- attrs = {
- 'a': a_attrs,
- 'img': img_attrs,
-
- 'abbr': basic_attrs,
- 'acronym': basic_attrs,
- 'div': basic_attrs,
- 'span': basic_attrs,
- 'p': basic_attrs,
-
- }
- try:
- return bleach.linkify(
- bleach.clean(entry, tags=html_tags, attributes=attrs))
- except Exception, x:
- log.error("Ouch, unable to linkify or clean _%s_" % entry)
- log.exception(x)
- return entry
+from lifestream.generic.hooks import tidy_up
def prepare_entry(entryJSON, log):
content = ''
@@ -54,13 +12,8 @@ def prepare_entry(entryJSON, log):
image = None
if 'links' in entryJSON:
for link in entryJSON['links']:
- log.info(link)
if link['rel'] == 'image':
image = link['href']
- else:
- log.info("No photo")
- else:
- log.info("No links")
title = tidy_up(entryJSON['title'], log)
content = tidy_up(content, log)
@@ -1,7 +1,6 @@
import re
-from bleach import Bleach
-bleach = Bleach()
+from lifestream.generic.hooks import tidy_up
def prepare_entry(entryJSON, log):
""" Given entryJSON, output a dictionary of variables for use in
@@ -56,7 +55,7 @@ def prepare_entry(entryJSON, log):
status = re.sub('@(\w+)', r'<a href="http://identi.ca/\1">@\1</a>', status)
try:
- status = bleach.linkify(status.replace('\n', '<br />'))
+ status = tidy_up(status.replace('\n', '<br />'), log)
except Exception, x:
log.error("Ouch, unable to linkify _%s_" % status)
log.exception(x)
View
@@ -4,8 +4,22 @@
import django.forms.models
from django.contrib.auth.models import User
+class Webpage(models.Model):
+ """ A webpage is composed of Streams, page widgets,
+ and other funage.
+ """
+ user = models.ForeignKey(User)
+ name = models.CharField(max_length=140)
+ title = models.CharField(max_length=140)
+ # PageConfig
+ config = models.TextField()
+ created_date = models.DateTimeField(auto_now_add=True,
+ verbose_name='Created On')
+ updated_date = models.DateTimeField(auto_now=True,
+ verbose_name='Last Modified')
+
class Stream(models.Model):
- """ A Stream is like a 'page'. A stream is combosed of Feeds. """
+ """ A stream is composed of Feeds. """
user = models.ForeignKey(User)
name = models.CharField(max_length=140)
# StreamConfig
@@ -19,7 +33,6 @@ class Stream(models.Model):
# ALTER TABLE lifestream_stream ADD COLUMN `config` longtext NOT NULL;
# ALTER TABLE lifestream_stream ADD COLUMN `edit_list` longtext NOT NULL;
-
def __unicode__(self):
return self.name
@@ -1,5 +1,4 @@
-from bleach import Bleach
-bleach = Bleach()
+from lifestream.generic.hooks import tidy_up
def prepare_entry(entryJSON, log):
content = ''
@@ -12,7 +11,7 @@ def prepare_entry(entryJSON, log):
else:
content = 'unreadable... ' + str(entryJSON)
try:
- content = bleach.linkify(content)
+ content = tidy_up(content, log)
except Exception, x:
log.error("Ouch, unable to linkify _%s_" % content)
log.exception(x)
@@ -1,8 +1,4 @@
-import re
-
-from bleach import Bleach
-bleach = Bleach()
-
+from lifestream.generic.hooks import tidy_up
def prepare_entry(entryJSON, log):
""" Given entryJSON, output a dictionary of variables for use in
templates/twitter_com/entry.html
@@ -75,7 +71,7 @@ def prepare_entry(entryJSON, log):
tweet = re.sub('#(\w+)', r'<a href="http://twitter.com/search?q=%23\1">#\1</a>', tweet)
tweet = re.sub('@(\w+)', r'<a href="http://twitter.com/\1">@\1</a>', tweet)
try:
- tweet = bleach.linkify(tweet.replace('\n', '<br />'))
+ tweet = tidy_up(tweet.replace('\n', '<br />'), log)
except Exception, x:
log.error("Ouch, unable to linkify _%s_ caught" % tweet)
log.exception(x)
View
@@ -5,6 +5,7 @@
import jsonpickle
import simplejson as json
+from django.conf import settings
import django.utils.encoding
import django.template
import django.template.loaders
@@ -19,7 +20,7 @@
from patchouli.plugins.hostname_css_class import HostnameCssPlugin
from patchouli.plugins.social_identities import SocialIdentityFromTagsPlugin
-logging.basicConfig( level = logging.DEBUG, format = '%(asctime)s %(levelname)s %(message)s', )
+logging.basicConfig(filename=settings.LOG_FILENAME, level = logging.DEBUG, format = '%(asctime)s %(levelname)s %(message)s', )
log = logging.getLogger()
def profile(request, username):
@@ -41,30 +42,35 @@ def js_embed_stream(request, username, streamname):
def stream(request, username, streamname):
ctx = django.template.RequestContext(request)
ctx.autoescape=False #TODO Need more thinking around best way to handle this...
-
pageVars = common_stream(request, username, streamname)
-
+
return render_to_response('lifestream/profile.html',
pageVars,
context_instance=ctx,
)
def common_stream(request, username, streamname):
username = username.lower()
- user = User.objects.get(username=username)
+ user = User.objects.get(username=username)
+
rawEntries = (lifestream.models.Entry.objects.order_by('-last_published_date')
.filter(feed__user=user,
feed__streams__name__exact = streamname,
- visible=True))[:50]
+ visible=True))[:150]
+
entries = []
plugins = []
if username == 'ozten':
plugins = [SocialIdentityFromTagsPlugin()]
plugins.append(HostnameCssPlugin(log))
+
renderedEntries = render_entries(request, rawEntries, plugins)
- profile = renderProfile(request, user, plugins)
-
+ webpage = lifestream.models.Webpage.objects.get(name=streamname, user=user)
+ webpage_properties = patchouli_auth.preferences.getPageProperties(webpage)
+
+ profile = renderProfile(request, user, plugins, webpage_properties)
+
preferences = patchouli_auth.preferences.getPreferences(user)
if 'default' == preferences['javascript_url']:
@@ -76,15 +82,15 @@ def common_stream(request, username, streamname):
css_url = '/static/css/stylo.css'
else:
css_url = preferences['css_url']
-
return {'entries': renderedEntries,
'profile': profile,
'css_url': css_url,
'javascript_url': js_url,
'processing_js': preferences['processing_js'],
'stream_name': streamname,
'user': user,
- 'username': username}
+ 'username': username,
+ 'page_props': webpage_properties,}
def render_entries(request, rawEntries, plugins=[]):
""" plugins - list of functions to be run once for each entry's variables """
@@ -107,7 +113,7 @@ def render_entries(request, rawEntries, plugins=[]):
[p.post_observe_stream_entries() for p in plugins]
return renderedEntries
-def renderProfile(request, user, plugins):
+def renderProfile(request, user, plugins, webpage_properties):
sourcesResults = lifestream.models.Feed.objects.order_by('url').filter(user=user)
sources = [{'title': s.title, 'url':s.url} for s in sourcesResults]
@@ -124,7 +130,8 @@ def renderProfile(request, user, plugins):
'show_fn': show_fn,
'username': user.username,
'preferences': json.loads(user.get_profile().properties),
- 'sources': sources}
+ 'sources': sources,
+ 'page_props': webpage_properties,}
[data.update(plugin.template_variables()) for plugin in plugins]
t = django.template.loader.select_template(('foo', 'lifestream/profile_blurb.html'))
@@ -1,3 +1,5 @@
+# coding=utf-8
+
import simplejson
import patchouli_auth.models
@@ -30,4 +32,32 @@ def savePreferences(user, properties):
profile.save()
except patchouli_auth.models.UserProfile.DoesNotExist, e:
# Violates a pre-condition that getPreferences will be called atleast once before savePreferences is called..
- pass
+ pass
+
+def getPageProperties(page):
+ """ Loads the properties for a Page.
+ This includes migrating any settings
+ for new code """
+ after_profile_html_area = """
+<!-- Give props to Robert Podgórski -->
+<h5>Firefoxzilla protects the city</h5>
+<div>Background imagery By <a href="http://creative.mozilla.org/people/blackmoondev">Blackmoondev</a></div>
+ """
+ pageProps = {
+ 'before_stream_html_area': 'before stream',
+ 'after_stream_html_area': 'after stream',
+ 'show_profile_blurb': True,
+ 'show_follow_me_links': True,
+ 'before_profile_html_area': 'before profile',
+ 'after_profile_html_area': after_profile_html_area,
+ }
+
+ existingProps = simplejson.loads(page.config)
+ pageProps.update(existingProps)
+ return pageProps
+
+def savePageOrStreamProperties(model, properties):
+ """ Given a Page or Stream model, persists the
+ properties """
+ model.config = simplejson.dumps(properties)
+ model.save()
Oops, something went wrong.

0 comments on commit 2d3ba1e

Please sign in to comment.