Permalink
Browse files

initial

  • Loading branch information...
0 parents commit 07510cc030693613d06c2a37ae6c9fe606025c4a @mtigas committed Nov 4, 2011
@@ -0,0 +1,31 @@
+.svn/
+*.pyc
+*.pyo
+.DS_Store
+Thumbs.db
+*.swp
+*.lock
+*~
+*.tmp
+*.bak
+access.log
+error.log
+*.pid
+local_settings.py
+.pid
+.fseventsd/
+.Trashes/
+
+Build
+build
+dist
+._*
+*.egg-info
+
+*.mode1v3
+*.pbxuser
+*.xcworkspace
+xcuserdata
+
+var/html/
+var/db/
@@ -0,0 +1,126 @@
+
+
+**EARLY WIP, PROBABLY TOTALLY BROKEN, DO NOT USE, ET CETERA, ET CETERA.**
+
+# django-medusa
+
+Allows rendering a Django-powered website into a static website a la *Jekyll*,
+*Movable Type*, or other static page generation CMSes or frameworks.
+**django-medusa** is designed to be as simple as possible and allow the
+easy(ish) conversion of existing dynamic Django-powered websites -- nearly any
+existing Django site installation (not relying on highly-dynamic content) can
+be converted into a static generator which mirror's that site's output.
+
+Given a "renderer" that defines a set of URLs (see below), this uses Django's
+built-in `TestClient` to render out those views to either disk or Amazon S3.
+
+At the moment, this likely does not scale to extremely large websites.
+
+Optionally utilizes the `multiprocessing` library to speed up the rendering
+process by rendering many views at once.
+
+## Renderer classes
+
+Simply subclassing the `StaticSiteRenderer` class and defining `get_paths`
+works:
+
+ from django_medusa.renderers import StaticSiteRenderer
+
+ class HomeRenderer(StaticSiteRenderer):
+ def get_paths(self):
+ return frozenset([
+ "/",
+ "/about/",
+ "/sitemap.xml",
+ ])
+
+ renderers = [HomeRenderer, ]
+
+A more complex example:
+
+ from django_medusa.renderers import StaticSiteRenderer
+ from myproject.blog.models import BlogPost
+
+
+ class BlogPostsRenderer(StaticSiteRenderer):
+ def get_paths(self):
+ paths = ["/blog/", ]
+
+ items = BlogPost.objects.filter(is_live=True).order_by('-pubdate')
+ for item in items:
+ paths.append(item.get_absolute_url())
+
+ return paths
+
+ renderers = [BlogPostsRenderer, ]
+
+Or even:
+
+ from django_medusa.renderers import StaticSiteRenderer
+ from myproject.blog.models import BlogPost
+ from django.core.urlresolvers import reverse
+
+
+ class BlogPostsRenderer(StaticSiteRenderer):
+ def get_paths(self):
+ # A "set" so we can throw items in blindly and be guaranteed that
+ # we don't end up with dupes.
+ paths = set(["/blog/", ])
+
+ items = BlogPost.objects.filter(is_live=True).order_by('-pubdate')
+ for item in items:
+ # BlogPost detail view
+ paths.add(item.get_absolute_url())
+
+ # The generic date-based list views.
+ paths.add(reverse('blog:archive_day', args=(
+ item.pubdate.year, item.pubdate.month, item.pubdate.day
+ )))
+ paths.add(reverse('blog:archive_month', args=(
+ item.pubdate.year, item.pubdate.month
+ )))
+ paths.add(reverse('blog:archive_year', args=(item.pubdate.year,)))
+
+ # Cast back to a list since that's what we're expecting.
+ return list(paths)
+
+ renderers = [BlogPostsRenderer, ]
+
+## Renderer backends
+
+### Disk-based static site renderer
+
+Example settings:
+
+ INSTALLED_APPS = (
+ # ...
+ # ...
+ 'django_medusa',
+ )
+ # ...
+ MEDUSA_RENDERER_CLASS = "django_medusa.renderers.DiskStaticSiteRenderer"
+ MEDUSA_MULTITHREAD = True
+ MEDUSA_DEPLOY_DIR = os.path.abspath(os.path.join(
+ REPO_DIR,
+ 'var',
+ "html"
+ ))
+
+### S3-based site renderer
+
+Example settings:
+
+ INSTALLED_APPS = (
+ # ...
+ # ...
+ 'django_medusa',
+ )
+ # ...
+ MEDUSA_RENDERER_CLASS = "django_medusa.renderers.S3StaticSiteRenderer"
+ MEDUSA_MULTITHREAD = True
+ AWS_ACCESS_KEY = ""
+ AWS_SECRET_ACCESS_KEY = ""
+ AWS_STORAGE_BUCKET_NAME = ""
+
+Be aware that the S3 renderer will overwrite any existing files that match
+URL paths in your site.
No changes.
No changes.
No changes.
@@ -0,0 +1,14 @@
+from django.core.management.base import BaseCommand, CommandError
+from django_medusa.utils import get_static_renderers
+
+class Command(BaseCommand):
+ can_import_settings = True
+
+ help = 'Looks for \'renderers.py\' in each INSTALLED_APP, which defines '\
+ 'a class for processing one or more URL paths into static files.'
+
+ def handle(self, *args, **options):
+ for Renderer in get_static_renderers():
+ r = Renderer()
+ r.generate()
+
@@ -0,0 +1,24 @@
+from django.conf import settings
+import importlib
+from .base import BaseStaticSiteRenderer
+from .disk import DiskStaticSiteRenderer
+from .s3 import S3StaticSiteRenderer
+
+__all__ = ('BaseStaticSiteRenderer', 'DiskStaticSiteRenderer',
+ 'S3StaticSiteRenderer', 'StaticSiteRenderer')
+
+
+def get_cls(renderer_name):
+ mod_path, cls_name = renderer_name.rsplit('.', 1)
+ mod = importlib.import_module(mod_path)
+ return getattr(mod, cls_name)
+
+
+DEFAULT_RENDERER = 'medusa.renderers.BaseStaticSiteRenderer'
+
+# Define the default "django_medusa.renderers.StaticSiteRenderer" class as
+# whatever class we have chosen in settings (defaulting to Base which will
+# throw NotImplementedErrors when attempting to render).
+StaticSiteRenderer = get_cls(getattr(settings,
+ 'MEDUSA_RENDERER_CLASS', DEFAULT_RENDERER
+))
@@ -0,0 +1,44 @@
+__all__ = ['COMMON_MIME_MAPS', 'BaseStaticSiteRenderer']
+
+
+# Since mimetypes.get_extension() gets the "first known" (alphabetically),
+# we get supid behavior like "text/plain" mapping to ".bat". This list
+# overrides some file types we will surely use, to eliminate a call to
+# mimetypes.get_extension() except in unusual cases.
+COMMON_MIME_MAPS = {
+ "text/plain": ".txt",
+ "text/html": ".html",
+ "text/javascript": ".js",
+ "application/javascript": ".js",
+ "text/json": ".json",
+ "application/json": ".json",
+ "text/css": ".css",
+}
+
+
+class BaseStaticSiteRenderer(object):
+ """
+ This default renderer writes the given URLs (defined in get_paths())
+ into static files on the filesystem by getting the view's response
+ through the Django testclient.
+ """
+
+ def get_paths(self):
+ """ Override this in a subclass to define the URLs to process """
+ raise NotImplementedError
+
+ @property
+ def paths(self):
+ """ Property that memoizes get_paths. """
+ p = getattr(self, "_paths", None)
+ if not p:
+ p = self.get_paths()
+ self._paths = p
+ return p
+
+ def render_path(self, path=None, view=None):
+ raise NotImplementedError
+
+ def generate(self):
+ for path in self.paths:
+ self.render_path(path)
@@ -0,0 +1,82 @@
+from django.conf import settings
+from django.test.client import Client
+import mimetypes
+import os
+from .base import COMMON_MIME_MAPS, BaseStaticSiteRenderer
+
+__all__ = ('DiskStaticSiteRenderer', )
+
+
+# Unfortunately split out from the class at the moment to allow rendering with
+# several processes via `multiprocessing`.
+# TODO: re-implement within the class if possible?
+def _disk_render_path(args):
+ client, path, view = args
+ if not client:
+ client = Client()
+ if path:
+ DEPLOY_DIR = settings.MEDUSA_DEPLOY_DIR
+ realpath = path
+ if path.startswith("/"):
+ realpath = realpath[1:]
+
+ if path.endswith("/"):
+ needs_ext = True
+ else:
+ needs_ext = False
+
+ output_dir = os.path.abspath(os.path.join(
+ DEPLOY_DIR,
+ os.path.dirname(realpath)
+ ))
+ if not os.path.exists(output_dir):
+ os.makedirs(output_dir)
+ outpath = os.path.join(DEPLOY_DIR, realpath)
+
+ resp = client.get(path)
+ if resp.status_code != 200:
+ raise Exception
+ if needs_ext:
+ mime = resp['Content-Type']
+ mime = mime.split(';', 1)[0]
+
+ # Check our override list above first.
+ ext = COMMON_MIME_MAPS.get(
+ mime,
+ mimetypes.guess_extension(mime)
+ )
+ if ext:
+ outpath += "index" + ext
+ else:
+ # Default to ".html"
+ outpath += "index.html"
+ print outpath
+ with open(outpath, 'w') as f:
+ f.write(resp.content)
+
+
+class DiskStaticSiteRenderer(BaseStaticSiteRenderer):
+
+ def render_path(self, path=None, view=None):
+ _disk_render_path((self.client, path, view))
+
+ def generate(self):
+ if getattr(settings, "MEDUSA_MULTITHREAD", False):
+ # Upload up to ten items at once via `multiprocessing`.
+ from multiprocessing import Pool, cpu_count
+
+ print "Generating with up to %d processes..." % cpu_count()
+ pool = Pool(cpu_count())
+
+ pool.map_async(
+ _disk_render_path,
+ ((None, path, None) for path in self.paths),
+ chunksize=5
+ )
+ pool.close()
+ pool.join()
+ else:
+ # Use standard, serial upload.
+ self.client = Client()
+ for path in self.paths:
+ self.render_path(path=path)
Oops, something went wrong.

0 comments on commit 07510cc

Please sign in to comment.