merged latest

README.txt

@@ -42,7 +42,7 @@ dedicated hosting.  I run it with apache2 and mod_wsgi on ubuntu 8.04.
 
 Install ubuntu packages (and dependencies) for:
 
-    python ipython python-setuptools python-simplejson
+    python ipython python-setuptools python-simplejson python-feedparser
     postgresql-8.3 postgresql-client-8.3 python-psycopg2
     sun-java6-jdk solr-tomcat5.5
     libapache2-mod-wsgi 
@@ -72,7 +72,7 @@ it's right when you can do this and get 'utf8':
 
 Install non-.deb python dependencies:
 
-    from tarball:   Django 1.2.1
+    from tarball:   Django 1.2.4
     w/easy_install: solrpy 0.9 (and iso8601, if import from zodb install)
 
 Set up solr config.  Move the original /etc/solr/conf/schema.xml

base/admin.py

@@ -39,17 +39,10 @@ class EntryTagAdmin(admin.ModelAdmin):
     list_display = ['id', 'entry', 'tag', 'sequence_num']
     search_fields = ['tag']
 
-#class EntryWebhookAdmin(admin.ModelAdmin):
-#    list_display = ['id', 'entry', 'url', 'date_first_attempt',
-#        'num_attempts', 'date_latest_attempt']
-#    list_filter = ['date_first_attempt', 'date_latest_attempt',
-#        'num_attempts']
-
 admin.site.register(m.GroupProfile, GroupProfileAdmin)
 admin.site.register(m.UserProfile, UserProfileAdmin)
 admin.site.register(m.Filter, FilterAdmin)
 admin.site.register(m.Tag, TagAdmin)
 admin.site.register(m.Url, UrlAdmin)
 admin.site.register(m.Entry, EntryAdmin)
 admin.site.register(m.EntryTag, EntryTagAdmin)
-#admin.site.register(m.EntryWebhook, EntryWebhookAdmin)

base/models.py

@@ -47,7 +47,6 @@ class UserProfile (m.Model):
     token = m.CharField(blank=True, max_length=32)
     tz = m.CharField(blank=True, max_length=6)
     group_invites = m.ManyToManyField(Group, related_name='invitees', blank=True)
-    webhook_url = m.URLField(blank=True, verify_exists=True)
     date_modified = m.DateTimeField(auto_now=True)
 
     def solr_reindex (self):
@@ -86,8 +85,7 @@ def solr_reindex (self):
 class UserProfileForm (ModelForm):
     class Meta:
         model = UserProfile
-        fields = ['url', 'is_private', 'default_to_private_entry',
-            'webhook_url']
+        fields = ['url', 'is_private', 'default_to_private_entry']
 
 
 class Filter (m.Model):
@@ -316,29 +314,4 @@ def delete(self, solr_delete=True):
         if solr_delete:
             self.solr_delete()
         super(Entry, self).delete()
-
-
-#class EntryWebhook (m.Model):
-#    entry = m.ForeignKey(Entry, unique=True)
-#    url = m.URLField()
-#    num_attempts = m.SmallIntegerField(default=0)
-#    date_first_attempt = m.DateTimeField(auto_now_add=True)
-#    date_latest_attempt = m.DateTimeField(auto_now=True, db_index=True)
-#    latest_attempt_status = m.CharField(max_length=3, blank=True)
-#
-#    class Meta:
-#        verbose_name = "Webhook POST URL"
 
-
-## This handler is wired up to Entry's post_save to create a new
-## EntryWebhook.  Done here with a signal instead of on Entry.save()
-## because we only want to do it when the Entry is first created.
-#def create_entry_webhook_handler(sender, **kwargs):
-#    if kwargs['created']:
-#        instance = kwargs['instance']
-#        webhook_url = instance.user.get_profile().webhook_url
-#        if webhook_url:
-#            entry_webhook = EntryWebhook(entry=kwargs['instance'],
-#                url=webhook_url)
-#            entry_webhook.save()
-#post_save.connect(create_entry_webhook_handler, sender=Entry)

base/templates/prefs.html

@@ -28,7 +28,6 @@ <h2>Update your preferences</h2>
     by default -- even if your account isn't private.  
 </p>
 
-
 <table class='gentable'>
     <thead>
     </thead>
@@ -48,6 +47,7 @@ <h2>Update your preferences</h2>
                 <th>Links private by default</th>
                 <td>{{ profile_form.default_to_private_entry }}</td>
             </tr>
+            <tr>
             <tr>
                 <td>&nbsp;</td>
                 <td>
@@ -64,3 +64,4 @@ <h2>Update your preferences</h2>
 
 
 {% endblock content %}
+

base/tests.py

@@ -34,6 +34,7 @@ def test_add_entry(self):
         client = Client()
         self.assertTrue(client.login(username='unalog', password='unalog'))
         response = client.post('/entry/new', self.test_entry)
+        self.assertEqual(response.status_code, 302)
         self.assertEqual(response['location'], 
                 'http://testserver/entry/1/edit/')
         entry = m.Entry.objects.get(id=1)
@@ -54,9 +55,9 @@ def test_add_entry_json(self):
         entry_json = json.dumps(self.test_entry)
         response = client.post('/entry/new', entry_json, 
                 content_type='application/json')
-        self.assertEqual(response.status_code, 302)
-        self.assertEqual(response['location'], 'http://testserver/entry/1/edit/')
-        entry = m.Entry.objects.get(id=1)
+        self.assertEqual(response.status_code, 201)
+        self.assertEqual(response['location'], 'http://testserver/entry/2/')
+        entry = m.Entry.objects.get(id=2)
         self.assertEqual(entry.url.value, 'http://example.com/')
         self.assertEqual(entry.title, 'hey example.com!')
         self.assertEqual(entry.is_private, False)
@@ -75,9 +76,9 @@ def test_add_entry_json_with_created(self):
         entry_json = json.dumps(test_entry)
         response = client.post('/entry/new', entry_json, 
                 content_type='application/json')
-        self.assertEqual(response.status_code, 302)
-        self.assertEqual(response['location'], 'http://testserver/entry/1/edit/')
-        entry = m.Entry.objects.get(id=1)
+        self.assertEqual(response.status_code, 201)
+        self.assertEqual(response['location'], 'http://testserver/entry/3/')
+        entry = m.Entry.objects.get(id=3)
         self.assertEqual(entry.date_created, 
             datetime.datetime(1985, 4, 12, 23, 20, 50))
 

base/views.py

@@ -241,7 +241,14 @@ def entry_new (request):
             new_entry.save()
             new_entry.add_tags(tags_orig)
 
+            if payload == "application/json":
+                url = reverse('entry', args=[new_entry.id])
+                if was_created:
+                    return HttpResponseCreated(url)
+                else:
+                    return HttpResponseRedirect(url)
             request.user.message_set.create(message='Saved!')
+
             # If they had to confirm this, they don't need an edit screen again
             if submit == 'Save anyway':
                 return HttpResponseRedirect(reverse('index'))
@@ -774,3 +781,10 @@ def search_feed (request):
         except:
             page = None
     return render_to_response('index.html', context)
+
+class HttpResponseCreated(HttpResponse):
+    status_code = 201
+
+    def __init__(self, location):
+        HttpResponse.__init__(self)
+        self["Location"] = location

scripts/d2u.py

@@ -1,7 +1,10 @@
 #!/usr/bin/env python
 
 """
-This is a delcious to unalog import tool. 
+This is a delicious to unalog import tool. You'll need to export your
+delicious links and save them as a file ... it should be html. Then
+you run this script with your unalog username and password with the 
+delicious export file:
 
   ./d2u.py --username me --password secret delicious-20101216.htm
 
@@ -19,17 +22,23 @@
 import json
 import optparse
 import os
+import socket
+import sys
 import time
-import urllib
+import urllib2
 
 from html5lib import HTMLParser, treebuilders
 from lxml import etree
 
+socket.setdefaulttimeout(10)
+
 opt_parser = optparse.OptionParser()
 opt_parser.add_option('-u', '--username', dest='username')
 opt_parser.add_option('-p', '--password', dest='password')
 opt_parser.add_option('-n', '--unalog', dest='unalog', 
                       default="http://unalog.com")
+opt_parser.add_option('-s', '--skip', type=int, dest='skip', default=0)
+opt_parser.add_option('-c', '--check', action='store_true', dest='check')
 
 opts, args = opt_parser.parse_args()
 
@@ -53,10 +62,14 @@
 h.add_credentials(opts.username, opts.password)
 unalog = opts.unalog.rstrip("/") + "/entry/new"
 
-status = {}
+summary = {}
 count = 0
 
 for dt in doc.findall(".//{%s}dt" % xhtml):
+    count += 1
+    if opts.skip > 1 and count < opts.skip:
+        continue
+
     # get the bookmark from the dt
     a = dt.find('{%s}a' % xhtml)
     b  = a.attrib
@@ -72,10 +85,34 @@
     t = time.localtime(int(b["add_date"]))
     t = time.strftime('%Y-%m-%dT%H:%M:%S%z', t)
 
-    # get the content at the url
+    # get the content at the url only if it looks like html or text
     url = b["href"]
-    resp, content = h.request(url, "GET")
-    status[resp.status] = status.get(resp.status, 0) + 1
+    try:
+        resp = urllib2.urlopen(url)
+        content = resp.read()
+        status = resp.code 
+        summary[status] = summary.get(status, 0) + 1
+        content_type = resp.headers['content-type']
+        if 'html' in content_type or 'text' in content_type:
+            content = content.decode('utf-8', 'replace')
+        else:
+            content = None
+    except urllib2.HTTPError, e:
+        content = None
+        status = e.code
+    except urllib2.URLError, e:
+        content = None
+        status = e.reason
+    except Exception, e:
+        content = None
+        status = str(type(e))
+
+    summary[status] = summary.get(status, 0) + 1
+    print "\t".join([url, str(status)])
+
+    # don't bother posting to unalog if we're just checking
+    if opts.check:
+        continue
 
     # build the bookmark entry
     entry = {
@@ -85,16 +122,17 @@
         "private": b["private"] == 1,
         "date_created": t,
         "comment": comment,
-        #"content": content
+        "content": content
     }
 
     # send the bookmark to unalog as json
-    print url
     resp, content = h.request(unalog, "POST", body=json.dumps(entry),
             headers={"content-type": "application/json"})
 
-    count += 1
-    if count % 30 == 0:
-        break
+    if resp.status not in [200, 201, 302]:
+        print "post to unalog failed! %s" % url
+
+
 
-print status
+print "imported %s bookmarks" % count
+print "response summary: %s" % summary