Experimental new loader

New class based loader implementation

* Potentially Breaking Changes

    This commit introduces a new `CACHE` setting which when set to true makes the loader cache the contents of the stats files in memory. This means if set to True, the server will have to be restarted every time the stats file contents change or it'll keep serving old, cached URLs. `CACHE` defaults to `not DEBUG` by default.

setup.py

@@ -1,15 +1,31 @@
+import os
+import re
+
 from setuptools import setup
 
-version = '0.2.4'
+
+def rel(*parts):
+    '''returns the relative path to a file wrt to the current directory'''
+    return os.path.abspath(os.path.join(os.path.dirname(__file__), *parts))
+
+with open(rel('README.md')) as handler:
+    README = handler.read()
+
+with open(rel('webpack_loader', '__init__.py')) as handler:
+    INIT_PY = handler.read()
+
+
+VERSION = re.findall("__version__ = '([^']+)'", INIT_PY)[0]
 
 setup(
   name = 'django-webpack-loader',
   packages = ['webpack_loader', 'webpack_loader/templatetags', 'webpack_loader/contrib'],
-  version = version,
+  version = VERSION,
   description = 'Transparently use webpack with django',
+  long_description=README,
   author = 'Owais Lone',
   author_email = 'hello@owaislone.org',
-  download_url = 'https://github.com/owais/django-webpack-loader/tarball/{0}'.format(version),
+  download_url = 'https://github.com/owais/django-webpack-loader/tarball/{0}'.format(VERSION),
   url = 'https://github.com/owais/django-webpack-loader', # use the URL to the github repo
   keywords = ['django', 'webpack', 'assets'], # arbitrary keywords
   data_files = [("", ["LICENSE"])],

tests/app/settings.py

@@ -110,10 +110,12 @@
 
 WEBPACK_LOADER = {
     'DEFAULT': {
+        'CACHE': False,
         'BUNDLE_DIR_NAME': 'bundles/',
         'STATS_FILE': os.path.join(BASE_DIR, 'webpack-stats.json'),
     },
     'APP2': {
+        'CACHE': False,
         'BUNDLE_DIR_NAME': 'bundles/',
         'STATS_FILE': os.path.join(BASE_DIR, 'webpack-stats-app2.json'),
     }

tests/app/tests/test_webpack.py

@@ -10,12 +10,17 @@
 from django.views.generic.base import TemplateView
 from django_jinja.builtins import DEFAULT_EXTENSIONS
 from unittest2 import skipIf
-from webpack_loader.utils import (WebpackError, WebpackLoaderBadStatsError,
-                                  get_assets, get_bundle, get_config)
+from webpack_loader.exceptions import (
+    WebpackError,
+    WebpackLoaderBadStatsError
+)
+from webpack_loader.utils import get_loader
+
 
 BUNDLE_PATH = os.path.join(settings.BASE_DIR, 'assets/bundles/')
 DEFAULT_CONFIG = 'DEFAULT'
 
+
 class LoaderTestCase(TestCase):
     def setUp(self):
         self.factory = RequestFactory()
@@ -53,7 +58,7 @@ def test_config_check(self):
 
     def test_simple_and_css_extract(self):
         self.compile_bundles('webpack.config.simple.js')
-        assets = get_assets(get_config(DEFAULT_CONFIG))
+        assets = get_loader(DEFAULT_CONFIG).get_assets()
         self.assertEqual(assets['status'], 'done')
         self.assertIn('chunks', assets)
 
@@ -67,13 +72,13 @@ def test_simple_and_css_extract(self):
 
     def test_static_url(self):
         self.compile_bundles('webpack.config.publicPath.js')
-        assets = get_assets(get_config(DEFAULT_CONFIG))
+        assets = get_loader(DEFAULT_CONFIG).get_assets()
         self.assertEqual(assets['status'], 'done')
         self.assertEqual(assets['publicPath'], 'http://custom-static-host.com/')
 
     def test_code_spliting(self):
         self.compile_bundles('webpack.config.split.js')
-        assets = get_assets(get_config(DEFAULT_CONFIG))
+        assets = get_loader(DEFAULT_CONFIG).get_assets()
         self.assertEqual(assets['status'], 'done')
         self.assertIn('chunks', assets)
 
@@ -149,16 +154,21 @@ def test_reporting_errors(self):
         #TODO:
         self.compile_bundles('webpack.config.error.js')
         try:
-            get_bundle('main', get_config(DEFAULT_CONFIG))
+            get_loader(DEFAULT_CONFIG).get_bundle('main')
         except WebpackError as e:
             self.assertIn("Cannot resolve module 'the-library-that-did-not-exist'", str(e))
 
     def test_missing_stats_file(self):
-        os.remove(settings.WEBPACK_LOADER[DEFAULT_CONFIG]['STATS_FILE'])
+        stats_file = settings.WEBPACK_LOADER[DEFAULT_CONFIG]['STATS_FILE']
+        if os.path.exists(stats_file):
+            os.remove(stats_file)
         try:
-            get_assets(get_config(DEFAULT_CONFIG))
+            get_loader(DEFAULT_CONFIG).get_assets()
         except IOError as e:
-            expected = 'Error reading {0}. Are you sure webpack has generated the file and the path is correct?'.format(settings.WEBPACK_LOADER[DEFAULT_CONFIG]['STATS_FILE'])
+            expected = (
+                'Error reading {0}. Are you sure webpack has generated the '
+                'file and the path is correct?'
+            ).format(stats_file)
             self.assertIn(expected, str(e))
 
     def test_bad_status_in_production(self):
@@ -168,7 +178,7 @@ def test_bad_status_in_production(self):
         stats_file.write(json.dumps({'status': 'unexpected-status'}))
         stats_file.close()
         try:
-            get_bundle('main', get_config(DEFAULT_CONFIG))
+            get_loader(DEFAULT_CONFIG).get_bundle('main')
         except WebpackLoaderBadStatsError as e:
             self.assertIn((
                 "The stats file does not contain valid data. Make sure "
@@ -207,4 +217,3 @@ def test_request_blocking(self):
             result.rendered_content
             elapsed = time.time() - then
             self.assertTrue(elapsed < wait_for)
-

tests/tox.ini

@@ -25,4 +25,4 @@ deps =
     django18: django>=1.8.0,<1.9.0
     django19: django>=1.9.0,<1.10.0
 commands =
-    coverage run --source=webpack_loader manage.py test
+    coverage run --source=webpack_loader manage.py test {posargs}

webpack_loader/__init__.py

@@ -1 +1,4 @@
+__author__ = 'Owais Lone'
+__version__ = '0.3.0'
+
 default_app_config = 'webpack_loader.apps.WebpackLoaderConfig'

webpack_loader/config.py

@@ -0,0 +1,32 @@
+import re
+
+from django.conf import settings
+
+
+__all__ = ('load_config',)
+
+
+DEFAULT_CONFIG = {
+    'DEFAULT': {
+        'CACHE': not settings.DEBUG,
+        'BUNDLE_DIR_NAME': 'webpack_bundles/',
+        'STATS_FILE': 'webpack-stats.json',
+        # FIXME: Explore usage of fsnotify
+        'POLL_INTERVAL': 0.1,
+        'IGNORE': ['.+\.hot-update.js', '.+\.map']
+    }
+}
+
+user_config = getattr(settings, 'WEBPACK_LOADER', DEFAULT_CONFIG)
+
+user_config = dict(
+    (name, dict(DEFAULT_CONFIG['DEFAULT'], **cfg))
+    for name, cfg in user_config.items()
+)
+
+for entry in user_config.values():
+    entry['ignores'] = [re.compile(I) for I in entry['IGNORE']]
+
+
+def load_config(name):
+    return user_config[name]

webpack_loader/exceptions.py

@@ -0,0 +1,9 @@
+__all__ = ('WebpackError', 'WebpackLoaderBadStatsError')
+
+
+class WebpackError(Exception):
+    pass
+
+
+class WebpackLoaderBadStatsError(Exception):
+    pass

webpack_loader/loader.py

@@ -0,0 +1,79 @@
+import json
+import time
+
+from django.conf import settings
+from django.contrib.staticfiles.storage import staticfiles_storage
+
+from .exceptions import WebpackError, WebpackLoaderBadStatsError
+from .config import load_config
+
+
+class WebpackLoader(object):
+    _assets = {}
+
+    def __init__(self, name='DEFAULT'):
+        self.name = name
+        self.config = load_config(self.name)
+
+    def _load_assets(self):
+        try:
+            with open(self.config['STATS_FILE']) as f:
+                return json.load(f)
+        except IOError:
+            raise IOError(
+                'Error reading {0}. Are you sure webpack has generated '
+                'the file and the path is correct?'.format(
+                    self.config['STATS_FILE']))
+
+    def get_assets(self):
+        if self.config['CACHE']:
+            if self.name not in self._assets:
+                self._assets[self.name] = self._load_assets()
+            return self._assets[self.name]
+        return self._load_assets()
+
+    def filter_chunks(self, chunks):
+        for chunk in chunks:
+            ignore = any(regex.match(chunk['name'])
+                         for regex in self.config['ignores'])
+            if not ignore:
+                chunk['url'] = self.get_chunk_url(chunk)
+                yield chunk
+
+    def get_chunk_url(self, chunk):
+        public_path = chunk.get('publicPath')
+        if public_path:
+            return public_path
+
+        relpath = '{0}{1}'.format(
+            self.config['BUNDLE_DIR_NAME'], chunk['name']
+        )
+        return staticfiles_storage.url(relpath)
+
+    def get_bundle(self, bundle_name):
+        assets = self.get_assets()
+
+        if settings.DEBUG:
+            # poll when debugging and block request until bundle is compiled
+            # TODO: support timeouts
+            while assets['status'] == 'compiling':
+                time.sleep(self.config['POLL_INTERVAL'])
+                assets = self.get_assets()
+
+        if assets.get('status') == 'done':
+            chunks = assets['chunks'][bundle_name]
+            return self.filter_chunks(chunks)
+
+        elif assets.get('status') == 'error':
+            if 'file' not in assets:
+                assets['file'] = ''
+            error = u"""
+            {error} in {file}
+            {message}
+            """.format(**assets)
+            raise WebpackError(error)
+
+        raise WebpackLoaderBadStatsError(
+            "The stats file does not contain valid data. Make sure "
+            "webpack-bundle-tracker plugin is enabled and try to run "
+            "webpack again.")

webpack_loader/templatetags/webpack_loader.py

@@ -2,8 +2,7 @@
 from django.conf import settings
 from django.utils.safestring import mark_safe
 
-from ..utils import get_config, get_assets, get_bundle
-
+from ..utils import get_loader
 
 register = template.Library()
 
@@ -17,16 +16,19 @@ def filter_by_extension(bundle, extension):
 def render_as_tags(bundle):
     tags = []
     for chunk in bundle:
-        url = chunk.get('publicPath') or chunk['url']
         if chunk['name'].endswith('.js'):
-            tags.append('<script type="text/javascript" src="{0}"></script>'.format(url))
+            tags.append((
+                '<script type="text/javascript" src="{0}"></script>'
+            ).format(chunk['url']))
         elif chunk['name'].endswith('.css'):
-            tags.append('<link type="text/css" href="{0}" rel="stylesheet"/>'.format(url))
+            tags.append((
+                '<link type="text/css" href="{0}" rel="stylesheet"/>'
+            ).format(chunk['url']))
     return mark_safe('\n'.join(tags))
 
 
 def _get_bundle(bundle_name, extension, config):
-    bundle = get_bundle(bundle_name, get_config(config))
+    bundle = get_loader(config).get_bundle(bundle_name)
     if extension:
         bundle = filter_by_extension(bundle, extension)
     return bundle
@@ -40,7 +42,7 @@ def render_bundle(bundle_name, extension=None, config='DEFAULT'):
 @register.simple_tag
 def webpack_static(asset_name, config='DEFAULT'):
     return "{0}{1}".format(
-        get_assets(get_config(config)).get(
+        get_loader(config).get_assets().get(
             'publicPath', getattr(settings, 'STATIC_URL')
         ),
         asset_name