First commit

.gitignore

@@ -0,0 +1,14 @@
+chef/node.json
+fabfile.py
+build/*
+project/static/*
+project/media/archive/*
+project/django.log
+development.db
+media/archive/*
+project/settings_prod.py
+project/settings_dev.py
+chef/cookbooks/palewire/files/default/id_rsa
+chef/cookbooks/palewire/files/default/id_rsa.pub
+chef/cookbooks/palewire/files/default/authorized_keys
+*.pyc

archive/admin.py

@@ -0,0 +1,30 @@
+from django.contrib import admin
+from archive.models import Site, Update, Screenshot, Champion
+
+
+class SiteAdmin(admin.ModelAdmin):
+    list_display = ("name", "url", "status", "on_the_homepage")
+    list_filter = ("status",)
+    prepopulated_fields = {"slug": ("name",)}
+    list_editable = ('status', 'on_the_homepage')
+
+admin.site.register(Site, SiteAdmin)
+
+
+class UpdateAdmin(admin.ModelAdmin):
+    list_display = ('start',)
+    date_hierarchy = 'start'
+
+admin.site.register(Update, UpdateAdmin)
+
+
+class ScreenshotAdmin(admin.ModelAdmin):
+    list_display = ("site", "update",)
+
+admin.site.register(Screenshot, ScreenshotAdmin)
+
+
+class ChampionAdmin(admin.ModelAdmin):
+    pass
+
+admin.site.register(Champion, ChampionAdmin)

archive/fixtures/sources.json

@@ -0,0 +1,244 @@
+[
+   {
+      "pk":16,
+      "model":"archive.site",
+      "fields":{
+         "url":"http://aljazeera.net/",
+         "status":"active",
+         "description":"",
+         "name":"Al Jazeera",
+         "slug":"al-jazeera"
+      }
+   },
+   {
+      "pk":14,
+      "model":"archive.site",
+      "fields":{
+         "url":"http://aol.com/",
+         "status":"active",
+         "description":"",
+         "name":"Aol",
+         "slug":"aol"
+      }
+   },
+   {
+      "pk":4,
+      "model":"archive.site",
+      "fields":{
+         "url":"http://bbc.co.uk/",
+         "status":"active",
+         "description":"",
+         "name":"BBC",
+         "slug":"bbc"
+      }
+   },
+   {
+      "pk":1,
+      "model":"archive.site",
+      "fields":{
+         "url":"http://cnn.com/",
+         "status":"active",
+         "description":"",
+         "name":"CNN",
+         "slug":"cnn"
+      }
+   },
+   {
+      "pk":20,
+      "model":"archive.site",
+      "fields":{
+         "url":"http://chicagotribune.com/",
+         "status":"active",
+         "description":"",
+         "name":"Chicago Tribune",
+         "slug":"chicago-tribune"
+      }
+   },
+   {
+      "pk":3,
+      "model":"archive.site",
+      "fields":{
+         "url":"http://www.drudgereport.com/",
+         "status":"active",
+         "description":"",
+         "name":"Drudge Report, The",
+         "slug":"drudge-report"
+      }
+   },
+   {
+      "pk":9,
+      "model":"archive.site",
+      "fields":{
+         "url":"http://elpais.com/",
+         "status":"active",
+         "description":"",
+         "name":"El Pais",
+         "slug":"el-pais"
+      }
+   },
+   {
+      "pk":11,
+      "model":"archive.site",
+      "fields":{
+         "url":"http://guardian.co.uk/",
+         "status":"active",
+         "description":"",
+         "name":"Guardian, The",
+         "slug":"guardian"
+      }
+   },
+   {
+      "pk":7,
+      "model":"archive.site",
+      "fields":{
+         "url":"http://lemonde.fr/",
+         "status":"active",
+         "description":"",
+         "name":"Le Monde",
+         "slug":"le-monde"
+      }
+   },
+   {
+      "pk":2,
+      "model":"archive.site",
+      "fields":{
+         "url":"http://latimes.com/",
+         "status":"active",
+         "description":"",
+         "name":"Los Angeles Times",
+         "slug":"los-angeles-times"
+      }
+   },
+   {
+      "pk":6,
+      "model":"archive.site",
+      "fields":{
+         "url":"http://npr.org/",
+         "status":"active",
+         "description":"",
+         "name":"NPR",
+         "slug":"npr"
+      }
+   },
+   {
+      "pk":10,
+      "model":"archive.site",
+      "fields":{
+         "url":"http://nytimes.com/",
+         "status":"active",
+         "description":"",
+         "name":"New York Times",
+         "slug":"new-york-times"
+      }
+   },
+   {
+      "pk":8,
+      "model":"archive.site",
+      "fields":{
+         "url":"http://reuters.com/",
+         "status":"active",
+         "description":"",
+         "name":"Reuters",
+         "slug":"reuters"
+      }
+   },
+   {
+      "pk":22,
+      "model":"archive.site",
+      "fields":{
+         "url":"http://svt.se/",
+         "status":"active",
+         "description":"",
+         "name":"SVT",
+         "slug":"svt"
+      }
+   },
+   {
+      "pk":19,
+      "model":"archive.site",
+      "fields":{
+         "url":"http://www.spiegel.de/",
+         "status":"active",
+         "description":"",
+         "name":"Spiegel",
+         "slug":"spiegel"
+      }
+   },
+   {
+      "pk":21,
+      "model":"archive.site",
+      "fields":{
+         "url":"http://timesofindia.indiatimes.com/",
+         "status":"active",
+         "description":"",
+         "name":"Times of India",
+         "slug":"times-india"
+      }
+   },
+   {
+      "pk":15,
+      "model":"archive.site",
+      "fields":{
+         "url":"http://usatoday.com/",
+         "status":"active",
+         "description":"",
+         "name":"USA Today",
+         "slug":"usa-today"
+      }
+   },
+   {
+      "pk":12,
+      "model":"archive.site",
+      "fields":{
+         "url":"http://wsj.com/",
+         "status":"active",
+         "description":"",
+         "name":"Wall Street Journal",
+         "slug":"wall-street-journal"
+      }
+   },
+   {
+      "pk":5,
+      "model":"archive.site",
+      "fields":{
+         "url":"http://washingtonpost.com/",
+         "status":"active",
+         "description":"",
+         "name":"Washington Post",
+         "slug":"washington-post"
+      }
+   },
+   {
+      "pk":18,
+      "model":"archive.site",
+      "fields":{
+         "url":"http://www.xinhuanet.com/",
+         "status":"active",
+         "description":"",
+         "name":"Xinhua",
+         "slug":"xinhua"
+      }
+   },
+   {
+      "pk":13,
+      "model":"archive.site",
+      "fields":{
+         "url":"http://yahoo.com/",
+         "status":"active",
+         "description":"",
+         "name":"Yahoo!",
+         "slug":"yahoo"
+      }
+   },
+   {
+      "pk":17,
+      "model":"archive.site",
+      "fields":{
+         "url":"http://www.todayszaman.com/",
+         "status":"active",
+         "description":"",
+         "name":"Zaman",
+         "slug":"zaman"
+      }
+   }
+]

archive/management/commands/buildserver.py

@@ -0,0 +1,19 @@
+from project import urls
+from django.conf import settings
+from django.conf.urls.defaults import *
+from django.core.management.commands import runserver
+
+
+class Command(runserver.Command):
+    help = "Starts a variation of Django's runserver designed to serve the static files you've built."
+
+    def handle(self, *args, **kwds):
+        urls.urlpatterns = patterns("bakery.static_views",
+            url(r"^(.*)$", "serve", {
+                "document_root": settings.BUILD_DIR,
+                'show_indexes': True,
+                'default': 'index.html'
+                }),
+        )
+        runserver.Command.handle(self, *args, **kwds)
+

archive/management/commands/testscrape.py

@@ -0,0 +1,10 @@
+from django.core.management.base import BaseCommand, CommandError
+from archive.tasks import get_html
+from archive.models import Site, Screenshot
+
+
+class Command(BaseCommand):
+
+    def handle(self, *args, **options): 
+        obj = Site.objects.get(name="Yahoo!")
+        get_html(Screenshot.objects.filter(site=obj)[0].id)

archive/management/commands/updatearchive.py

@@ -0,0 +1,47 @@
+import logging
+from archive import views
+from archive.tasks import images
+from optparse import make_option
+from django.utils import timezone
+from django.core import management
+logger = logging.getLogger(__name__)
+from archive.models import Update, Site
+from django.core.management.base import BaseCommand, CommandError
+
+
+custom_options = (
+    make_option(
+        "--publish",
+        action="store_true",
+        dest="publish",
+        default=False,
+        help="Build and publish results to S3"
+    ),
+)
+
+
+class Command(BaseCommand):
+    help = 'Archive screenshots for all sites'
+    option_list = BaseCommand.option_list + custom_options
+    cmd = "s3cmd put --acl-public %(source)s s3://%(target)s"
+
+    def handle(self, *args, **options): 
+        update = Update.objects.create(
+            start=timezone.now(),
+        )
+        for site in Site.objects.filter(status='active'):
+            try:
+                images.get_screenshot(site.id, update.id)
+            except Exception, e:
+                logger.error(e)
+        if options.get("publish"):
+             logger.debug("Publishing data to S3")
+             views.Index().build_method()
+             update.build()
+             views.DateDetail().build_object(timezone.localtime(update.start).date())
+             views.TagDetail().build_method()
+             views.CryForHelp().build_method()
+             management.call_command("publish", interactive=False, verbosity=0)
+             logger.debug("Finished")
+
+