Merge pull request #4614 from smotornyuk/webassets

Webassets

ckan/cli/asset.py

@@ -0,0 +1,46 @@
+# encoding: utf-8
+
+import logging
+
+import click
+from webassets import script
+from webassets.exceptions import BundleError
+
+from ckan.lib import webassets_tools
+from ckan.cli import error_shout
+
+log = logging.getLogger(__name__)
+
+
+@click.group(name=u'asset', short_help=u'WebAssets commands')
+def asset():
+    pass
+
+
+@asset.command(u'build', short_help=u'Builds all bundles.')
+def build():
+    u'''Builds bundles, regardless of whether they are changed or not.'''
+    script.main(['build'], webassets_tools.env)
+    click.secho(u'Compile assets: SUCCESS', fg=u'green', bold=True)
+
+
+@asset.command(u'watch', short_help=u'Watch changes in source files.')
+def watch():
+    u'''Start a daemon which monitors source files, and rebuilds bundles.
+
+    This can be useful during development, if building is not
+    instantaneous, and you are losing valuable time waiting for the
+    build to finish while trying to access your site.
+
+    '''
+    script.main(['watch'], webassets_tools.env)
+
+
+@asset.command(u'clean', short_help=u'Clear cache.')
+def clean():
+    u'''Will clear out the cache, which after a while can grow quite large.'''
+    try:
+        script.main(['clean'], webassets_tools.env)
+    except BundleError as e:
+        return error_shout(e)
+    click.secho(u'Clear cache: SUCCESS', fg=u'green', bold=True)

ckan/cli/cli.py

@@ -7,6 +7,7 @@
 from ckan.cli import (
     datapusher,
     click_config_option, db, load_config, search_index, server,
+    asset,
     datastore,
     translation,
     dataset,
@@ -39,6 +40,7 @@ def ckan(ctx, config, *args, **kwargs):
 ckan.add_command(db.db)
 ckan.add_command(datapusher.datapusher)
 ckan.add_command(search_index.search_index)
+ckan.add_command(asset.asset)
 ckan.add_command(datastore.datastore)
 ckan.add_command(translation.translation)
 ckan.add_command(dataset.dataset)

ckan/config/deployment.ini_tmpl

@@ -156,6 +156,11 @@ ckan.feeds.author_link =
 #ckan.max_resource_size = 10
 #ckan.max_image_size = 2
 
+## Webassets Settings
+#ckan.webassets.use_x_sendfile = false
+#ckan.webassets.path = /var/lib/ckan/webassets
+
+
 ## Datapusher settings
 
 # Make sure you have set up the DataStore

ckan/config/environment.py

@@ -24,6 +24,7 @@
 import ckan.logic as logic
 import ckan.authz as authz
 import ckan.lib.jinja_extensions as jinja_extensions
+from ckan.lib.webassets_tools import webassets_init
 from ckan.lib.i18n import build_js_translations
 
 from ckan.common import _, ungettext, config
@@ -163,6 +164,8 @@ def update_config():
     plugin might have changed the config values (for instance it might
     change ckan.site_url) '''
 
+    webassets_init()
+
     for plugin in p.PluginImplementations(p.IConfigurer):
         # must do update in place as this does not work:
         # config = plugin.update_config(config)

ckan/config/middleware/flask_app.py

@@ -8,7 +8,7 @@
 import itertools
 import pkgutil
 
-from flask import Flask, Blueprint
+from flask import Flask, Blueprint, send_from_directory
 from flask.ctx import _AppCtxGlobals
 from flask.sessions import SessionInterface
 
@@ -23,14 +23,16 @@
 from repoze.who.config import WhoConfig
 from repoze.who.middleware import PluggableAuthenticationMiddleware
 
+import ckan
 import ckan.model as model
 from ckan.lib import base
 from ckan.lib import helpers
 from ckan.lib import jinja_extensions
 from ckan.common import config, g, request, ungettext
 import ckan.lib.app_globals as app_globals
 import ckan.lib.plugins as lib_plugins
-
+import ckan.plugins.toolkit as toolkit
+from ckan.lib.webassets_tools import get_webassets_path
 
 from ckan.plugins import PluginImplementations
 from ckan.plugins.interfaces import IBlueprint, IMiddleware, ITranslation
@@ -40,7 +42,6 @@
                         set_controller_and_action
                         )
 
-import ckan.lib.plugins as lib_plugins
 import logging
 from logging.handlers import SMTPHandler
 log = logging.getLogger(__name__)
@@ -183,6 +184,9 @@ def hello_world():
     def hello_world_post():
         return 'Hello World, this was posted to Flask'
 
+    # WebAssets
+    _setup_webassets(app)
+
     # Auto-register all blueprints defined in the `views` folder
     _register_core_blueprints(app)
     _register_error_handler(app)
@@ -493,3 +497,15 @@ def filter(self, log_record):
     context_provider = ContextualFilter()
     app.logger.addFilter(context_provider)
     app.logger.addHandler(mail_handler)
+
+
+def _setup_webassets(app):
+    app.use_x_sendfile = toolkit.asbool(
+        config.get('ckan.webassets.use_x_sendfile')
+    )
+
+    webassets_folder = get_webassets_path()
+
+    @app.route('/webassets/<path:path>')
+    def webassets(path):
+        return send_from_directory(webassets_folder, path)

ckan/lib/extract.py

@@ -11,6 +11,7 @@
                     ckan.lib.jinja_extensions.CkanExtend,
                     ckan.lib.jinja_extensions.LinkForExtension,
                     ckan.lib.jinja_extensions.ResourceExtension,
+                    ckan.lib.jinja_extensions.AssetExtension,
                     ckan.lib.jinja_extensions.UrlForStaticExtension,
                     ckan.lib.jinja_extensions.UrlForExtension
                    '''

ckan/lib/helpers.py

@@ -49,6 +49,7 @@
 import ckan
 
 from ckan.common import _, ungettext, c, g, request, session, json
+from ckan.lib.webassets_tools import include_asset, render_assets
 from markupsafe import Markup, escape
 
 
@@ -2651,6 +2652,8 @@ def clean_html(html):
 core_helper(converters.asbool)
 # Useful additions from the stdlib.
 core_helper(urlencode)
+core_helper(include_asset)
+core_helper(render_assets)
 
 
 def load_plugin_helpers():

ckan/lib/jinja_extensions.py

@@ -34,7 +34,8 @@ def get_jinja_env_options():
                     LinkForExtension,
                     ResourceExtension,
                     UrlForStaticExtension,
-                    UrlForExtension],
+                    UrlForExtension,
+                    AssetExtension],
     )
 
 
@@ -312,7 +313,7 @@ def _call(cls, args, kwargs):
         return h.nav_link(*args, **kwargs)
 
 class ResourceExtension(BaseExtension):
-    ''' Custom include_resource tag
+    ''' Deprecated. Custom include_resource tag.
 
     {% resource <resource_name> %}
 
@@ -329,6 +330,23 @@ def _call(cls, args, kwargs):
         return ''
 
 
+class AssetExtension(BaseExtension):
+    ''' Custom include_asset tag.
+
+    {% asset <bundle_name> %}
+
+    see lib.webassets_tools.include_asset() for more details.
+    '''
+
+    tags = set(['asset'])
+
+    @classmethod
+    def _call(cls, args, kwargs):
+        assert len(args) == 1
+        assert len(kwargs) == 0
+        h.include_asset(args[0])
+        return ''
+
 
 '''
 The following function is based on jinja2 code

ckan/lib/webassets_tools.py

@@ -0,0 +1,161 @@
+# encoding: utf-8
+
+import logging
+import os
+import tempfile
+
+from markupsafe import Markup
+from webassets import Environment
+from webassets.loaders import YAMLLoader
+
+from ckan.common import config, g
+
+
+logger = logging.getLogger(__name__)
+env = None
+
+
+def create_library(name, path):
+    """Create WebAssets library(set of Bundles).
+    """
+    config_path = os.path.join(path, u'webassets.yml')
+    if not os.path.exists(config_path):
+        return
+
+    library = YAMLLoader(config_path).load_bundles()
+    bundles = {
+        u'/'.join([name, key]): bundle
+        for key, bundle
+        in library.items()
+    }
+
+    # Unfortunately, you'll get an error attempting to register bundle
+    # with the same name twice. For now, let's just pop existing
+    # bundle and avoid name-conflicts
+    # TODO: make PR into webassets with preferable solution
+    # Issue: https://github.com/miracle2k/webassets/issues/519
+    for name, bundle in bundles.items():
+        env._named_bundles.pop(name, None)
+        env.register(name, bundle)
+
+    env.append_path(path)
+
+
+def webassets_init():
+    global env
+
+    static_path = get_webassets_path()
+
+    public = config.get(u'ckan.base_public_folder')
+
+    public_folder = os.path.abspath(os.path.join(
+        os.path.dirname(__file__), u'..', public))
+
+    base_path = os.path.join(public_folder, u'base')
+
+    env = Environment()
+    env.directory = static_path
+    env.debug = config.get(u'debug', False)
+    env.url = u'/webassets/'
+
+    env.append_path(base_path, u'/base/')
+
+    logger.debug(u'Base path {0}'.format(base_path))
+    create_library(u'vendor', os.path.join(
+        base_path, u'vendor'))
+
+    create_library(u'base', os.path.join(base_path, u'javascript'))
+
+    create_library(u'datapreview', os.path.join(base_path, u'datapreview'))
+
+    create_library(u'css', os.path.join(base_path, u'css'))
+
+
+def _make_asset_collection():
+    return {u'style': [], u'script': [], u'included': set()}
+
+
+def include_asset(name):
+    from ckan.lib.helpers import url_for_static_or_external
+    try:
+        if not g.webassets:
+            raise AttributeError(u'WebAssets not initialized yet')
+    except AttributeError:
+        g.webassets = _make_asset_collection()
+    if name in g.webassets[u'included']:
+        return
+
+    try:
+        bundle = env[name]
+    except KeyError:
+        logger.error(u'Trying to include unknown asset: <{}>'.format(name))
+        return
+
+    deps = bundle.extra.get(u'preload', [])
+
+    # Using DFS may lead to infinite recursion(unlikely, because
+    # extensions rarely depends on each other), so there is a sense to
+    # memoize visited routes.
+
+    # TODO: consider infinite loop prevention for assets that depends
+    # on each other
+    for dep in deps:
+        include_asset(dep)
+
+    # Add `site_root` if configured
+    urls = [url_for_static_or_external(url) for url in bundle.urls()]
+    type_ = None
+    for url in urls:
+        link = url.split(u'?')[0]
+        if link.endswith(u'.css'):
+            type_ = u'style'
+            break
+        elif link.endswith(u'.js'):
+            type_ = u'script'
+            break
+    else:
+        logger.warn(u'Undefined asset type: {}'.format(urls))
+        return
+    g.webassets[type_].extend(urls)
+    g.webassets[u'included'].add(name)
+
+
+def _to_tag(url, type_):
+    if type_ == u'style':
+        return u'<link href="{}" rel="stylesheet"/>'.format(url)
+    elif type_ == u'script':
+        return u'<script src="{}" type="text/javascript"></script>'.format(url)
+    return u''
+
+
+def render_assets(type_):
+    try:
+        assets = g.webassets
+    except AttributeError:
+        return u''
+
+    if not assets:
+        return u''
+    collection = assets[type_]
+    tags = u'\n'.join([_to_tag(asset, type_) for asset in assets[type_]])
+    collection[:] = []
+    return Markup(tags)
+
+
+def get_webassets_path():
+    webassets_path = config.get(u'ckan.webassets.path')
+
+    if not webassets_path:
+        storage_path = config.get(
+            u'ckan.storage_path'
+        ) or tempfile.gettempdir()
+
+        if storage_path:
+            webassets_path = os.path.join(storage_path, u'webassets')
+
+    if not webassets_path:
+        raise RuntimeError(
+            u'Either `ckan.webassets.path` or `ckan.storage_path` '
+            u'must be specified'
+        )
+    return webassets_path