diff --git a/ckan/cli/asset.py b/ckan/cli/asset.py new file mode 100644 index 00000000000..e10136de8db --- /dev/null +++ b/ckan/cli/asset.py @@ -0,0 +1,46 @@ +# encoding: utf-8 + +import logging + +import click +from webassets import script +from webassets.exceptions import BundleError + +from ckan.lib import webassets_tools +from ckan.cli import error_shout + +log = logging.getLogger(__name__) + + +@click.group(name=u'asset', short_help=u'WebAssets commands') +def asset(): + pass + + +@asset.command(u'build', short_help=u'Builds all bundles.') +def build(): + u'''Builds bundles, regardless of whether they are changed or not.''' + script.main(['build'], webassets_tools.env) + click.secho(u'Compile assets: SUCCESS', fg=u'green', bold=True) + + +@asset.command(u'watch', short_help=u'Watch changes in source files.') +def watch(): + u'''Start a daemon which monitors source files, and rebuilds bundles. + + This can be useful during development, if building is not + instantaneous, and you are losing valuable time waiting for the + build to finish while trying to access your site. + + ''' + script.main(['watch'], webassets_tools.env) + + +@asset.command(u'clean', short_help=u'Clear cache.') +def clean(): + u'''Will clear out the cache, which after a while can grow quite large.''' + try: + script.main(['clean'], webassets_tools.env) + except BundleError as e: + return error_shout(e) + click.secho(u'Clear cache: SUCCESS', fg=u'green', bold=True) diff --git a/ckan/cli/cli.py b/ckan/cli/cli.py index 12e53f9a4aa..61a63f6aa18 100644 --- a/ckan/cli/cli.py +++ b/ckan/cli/cli.py @@ -7,6 +7,7 @@ from ckan.cli import ( datapusher, click_config_option, db, load_config, search_index, server, + asset, datastore, translation, dataset, @@ -39,6 +40,7 @@ def ckan(ctx, config, *args, **kwargs): ckan.add_command(db.db) ckan.add_command(datapusher.datapusher) ckan.add_command(search_index.search_index) +ckan.add_command(asset.asset) ckan.add_command(datastore.datastore) ckan.add_command(translation.translation) ckan.add_command(dataset.dataset) diff --git a/ckan/config/deployment.ini_tmpl b/ckan/config/deployment.ini_tmpl index 767d81e86a2..a07e481b739 100644 --- a/ckan/config/deployment.ini_tmpl +++ b/ckan/config/deployment.ini_tmpl @@ -156,6 +156,11 @@ ckan.feeds.author_link = #ckan.max_resource_size = 10 #ckan.max_image_size = 2 +## Webassets Settings +#ckan.webassets.use_x_sendfile = false +#ckan.webassets.path = /var/lib/ckan/webassets + + ## Datapusher settings # Make sure you have set up the DataStore diff --git a/ckan/config/environment.py b/ckan/config/environment.py index 2beaed1fad1..6eaf499193d 100644 --- a/ckan/config/environment.py +++ b/ckan/config/environment.py @@ -24,6 +24,7 @@ import ckan.logic as logic import ckan.authz as authz import ckan.lib.jinja_extensions as jinja_extensions +from ckan.lib.webassets_tools import webassets_init from ckan.lib.i18n import build_js_translations from ckan.common import _, ungettext, config @@ -163,6 +164,8 @@ def update_config(): plugin might have changed the config values (for instance it might change ckan.site_url) ''' + webassets_init() + for plugin in p.PluginImplementations(p.IConfigurer): # must do update in place as this does not work: # config = plugin.update_config(config) diff --git a/ckan/config/middleware/flask_app.py b/ckan/config/middleware/flask_app.py index c6d437a7221..46e0bcda90b 100644 --- a/ckan/config/middleware/flask_app.py +++ b/ckan/config/middleware/flask_app.py @@ -8,7 +8,7 @@ import itertools import pkgutil -from flask import Flask, Blueprint +from flask import Flask, Blueprint, send_from_directory from flask.ctx import _AppCtxGlobals from flask.sessions import SessionInterface @@ -23,6 +23,7 @@ from repoze.who.config import WhoConfig from repoze.who.middleware import PluggableAuthenticationMiddleware +import ckan import ckan.model as model from ckan.lib import base from ckan.lib import helpers @@ -30,7 +31,8 @@ from ckan.common import config, g, request, ungettext import ckan.lib.app_globals as app_globals import ckan.lib.plugins as lib_plugins - +import ckan.plugins.toolkit as toolkit +from ckan.lib.webassets_tools import get_webassets_path from ckan.plugins import PluginImplementations from ckan.plugins.interfaces import IBlueprint, IMiddleware, ITranslation @@ -40,7 +42,6 @@ set_controller_and_action ) -import ckan.lib.plugins as lib_plugins import logging from logging.handlers import SMTPHandler log = logging.getLogger(__name__) @@ -183,6 +184,9 @@ def hello_world(): def hello_world_post(): return 'Hello World, this was posted to Flask' + # WebAssets + _setup_webassets(app) + # Auto-register all blueprints defined in the `views` folder _register_core_blueprints(app) _register_error_handler(app) @@ -493,3 +497,15 @@ def filter(self, log_record): context_provider = ContextualFilter() app.logger.addFilter(context_provider) app.logger.addHandler(mail_handler) + + +def _setup_webassets(app): + app.use_x_sendfile = toolkit.asbool( + config.get('ckan.webassets.use_x_sendfile') + ) + + webassets_folder = get_webassets_path() + + @app.route('/webassets/') + def webassets(path): + return send_from_directory(webassets_folder, path) diff --git a/ckan/lib/extract.py b/ckan/lib/extract.py index 6a57ce664ac..6c67d1d067d 100644 --- a/ckan/lib/extract.py +++ b/ckan/lib/extract.py @@ -11,6 +11,7 @@ ckan.lib.jinja_extensions.CkanExtend, ckan.lib.jinja_extensions.LinkForExtension, ckan.lib.jinja_extensions.ResourceExtension, + ckan.lib.jinja_extensions.AssetExtension, ckan.lib.jinja_extensions.UrlForStaticExtension, ckan.lib.jinja_extensions.UrlForExtension ''' diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index 27af0df6325..d97982ddcc9 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -49,6 +49,7 @@ import ckan from ckan.common import _, ungettext, c, g, request, session, json +from ckan.lib.webassets_tools import include_asset, render_assets from markupsafe import Markup, escape @@ -2651,6 +2652,8 @@ def clean_html(html): core_helper(converters.asbool) # Useful additions from the stdlib. core_helper(urlencode) +core_helper(include_asset) +core_helper(render_assets) def load_plugin_helpers(): diff --git a/ckan/lib/jinja_extensions.py b/ckan/lib/jinja_extensions.py index 8520e7bd862..7221dbe1ae5 100644 --- a/ckan/lib/jinja_extensions.py +++ b/ckan/lib/jinja_extensions.py @@ -34,7 +34,8 @@ def get_jinja_env_options(): LinkForExtension, ResourceExtension, UrlForStaticExtension, - UrlForExtension], + UrlForExtension, + AssetExtension], ) @@ -312,7 +313,7 @@ def _call(cls, args, kwargs): return h.nav_link(*args, **kwargs) class ResourceExtension(BaseExtension): - ''' Custom include_resource tag + ''' Deprecated. Custom include_resource tag. {% resource %} @@ -329,6 +330,23 @@ def _call(cls, args, kwargs): return '' +class AssetExtension(BaseExtension): + ''' Custom include_asset tag. + + {% asset %} + + see lib.webassets_tools.include_asset() for more details. + ''' + + tags = set(['asset']) + + @classmethod + def _call(cls, args, kwargs): + assert len(args) == 1 + assert len(kwargs) == 0 + h.include_asset(args[0]) + return '' + ''' The following function is based on jinja2 code diff --git a/ckan/lib/webassets_tools.py b/ckan/lib/webassets_tools.py new file mode 100644 index 00000000000..1110f54ad8e --- /dev/null +++ b/ckan/lib/webassets_tools.py @@ -0,0 +1,161 @@ +# encoding: utf-8 + +import logging +import os +import tempfile + +from markupsafe import Markup +from webassets import Environment +from webassets.loaders import YAMLLoader + +from ckan.common import config, g + + +logger = logging.getLogger(__name__) +env = None + + +def create_library(name, path): + """Create WebAssets library(set of Bundles). + """ + config_path = os.path.join(path, u'webassets.yml') + if not os.path.exists(config_path): + return + + library = YAMLLoader(config_path).load_bundles() + bundles = { + u'/'.join([name, key]): bundle + for key, bundle + in library.items() + } + + # Unfortunately, you'll get an error attempting to register bundle + # with the same name twice. For now, let's just pop existing + # bundle and avoid name-conflicts + # TODO: make PR into webassets with preferable solution + # Issue: https://github.com/miracle2k/webassets/issues/519 + for name, bundle in bundles.items(): + env._named_bundles.pop(name, None) + env.register(name, bundle) + + env.append_path(path) + + +def webassets_init(): + global env + + static_path = get_webassets_path() + + public = config.get(u'ckan.base_public_folder') + + public_folder = os.path.abspath(os.path.join( + os.path.dirname(__file__), u'..', public)) + + base_path = os.path.join(public_folder, u'base') + + env = Environment() + env.directory = static_path + env.debug = config.get(u'debug', False) + env.url = u'/webassets/' + + env.append_path(base_path, u'/base/') + + logger.debug(u'Base path {0}'.format(base_path)) + create_library(u'vendor', os.path.join( + base_path, u'vendor')) + + create_library(u'base', os.path.join(base_path, u'javascript')) + + create_library(u'datapreview', os.path.join(base_path, u'datapreview')) + + create_library(u'css', os.path.join(base_path, u'css')) + + +def _make_asset_collection(): + return {u'style': [], u'script': [], u'included': set()} + + +def include_asset(name): + from ckan.lib.helpers import url_for_static_or_external + try: + if not g.webassets: + raise AttributeError(u'WebAssets not initialized yet') + except AttributeError: + g.webassets = _make_asset_collection() + if name in g.webassets[u'included']: + return + + try: + bundle = env[name] + except KeyError: + logger.error(u'Trying to include unknown asset: <{}>'.format(name)) + return + + deps = bundle.extra.get(u'preload', []) + + # Using DFS may lead to infinite recursion(unlikely, because + # extensions rarely depends on each other), so there is a sense to + # memoize visited routes. + + # TODO: consider infinite loop prevention for assets that depends + # on each other + for dep in deps: + include_asset(dep) + + # Add `site_root` if configured + urls = [url_for_static_or_external(url) for url in bundle.urls()] + type_ = None + for url in urls: + link = url.split(u'?')[0] + if link.endswith(u'.css'): + type_ = u'style' + break + elif link.endswith(u'.js'): + type_ = u'script' + break + else: + logger.warn(u'Undefined asset type: {}'.format(urls)) + return + g.webassets[type_].extend(urls) + g.webassets[u'included'].add(name) + + +def _to_tag(url, type_): + if type_ == u'style': + return u''.format(url) + elif type_ == u'script': + return u''.format(url) + return u'' + + +def render_assets(type_): + try: + assets = g.webassets + except AttributeError: + return u'' + + if not assets: + return u'' + collection = assets[type_] + tags = u'\n'.join([_to_tag(asset, type_) for asset in assets[type_]]) + collection[:] = [] + return Markup(tags) + + +def get_webassets_path(): + webassets_path = config.get(u'ckan.webassets.path') + + if not webassets_path: + storage_path = config.get( + u'ckan.storage_path' + ) or tempfile.gettempdir() + + if storage_path: + webassets_path = os.path.join(storage_path, u'webassets') + + if not webassets_path: + raise RuntimeError( + u'Either `ckan.webassets.path` or `ckan.storage_path` ' + u'must be specified' + ) + return webassets_path diff --git a/ckan/plugins/toolkit.py b/ckan/plugins/toolkit.py index ce107f0f20f..b6b641f31ed 100644 --- a/ckan/plugins/toolkit.py +++ b/ckan/plugins/toolkit.py @@ -337,8 +337,10 @@ def _add_served_directory(cls, config, relative_path, config_var): assert config_var in ('extra_template_paths', 'extra_public_paths') # we want the filename that of the function caller but they will # have used one of the available helper functions - frame, filename, line_number, function_name, lines, index =\ - inspect.getouterframes(inspect.currentframe())[2] + # TODO: starting from python 3.5, `inspect.stack` returns list + # of named tuples `FrameInfo`. Don't forget to remove + # `getframeinfo` wrapper after migration. + filename = inspect.getframeinfo(inspect.stack()[2][0]).filename this_dir = os.path.dirname(filename) absolute_path = os.path.join(this_dir, relative_path) @@ -350,24 +352,31 @@ def _add_served_directory(cls, config, relative_path, config_var): @classmethod def _add_resource(cls, path, name): - '''Add a Fanstatic resource library to CKAN. + '''Add a WebAssets library to CKAN. - Fanstatic libraries are directories containing static resource files - (e.g. CSS, JavaScript or image files) that can be accessed from CKAN. + WebAssets libraries are directories containing static resource + files (e.g. CSS, JavaScript or image files) that can be + compiled into WebAsset Bundles. See :doc:`/theming/index` for more details. ''' import inspect import os + from ckan.lib.webassets_tools import create_library - # we want the filename that of the function caller but they will - # have used one of the available helper functions - frame, filename, line_number, function_name, lines, index =\ - inspect.getouterframes(inspect.currentframe())[1] + # we want the filename that of the function caller but they + # will have used one of the available helper functions + # TODO: starting from python 3.5, `inspect.stack` returns list + # of named tuples `FrameInfo`. Don't forget to remove + # `getframeinfo` wrapper after migration. + filename = inspect.getframeinfo(inspect.stack()[1][0]).filename this_dir = os.path.dirname(filename) absolute_path = os.path.join(this_dir, path) + create_library(name, absolute_path) + + # TODO: remove next two lines after dropping Fanstatic support import ckan.lib.fanstatic_resources ckan.lib.fanstatic_resources.create_library(name, absolute_path) diff --git a/ckan/public/base/css/webassets.yml b/ckan/public/base/css/webassets.yml new file mode 100644 index 00000000000..2aa2e470041 --- /dev/null +++ b/ckan/public/base/css/webassets.yml @@ -0,0 +1,18 @@ +fuchsia: + output: base/%(version)s_fuchsia.css + contents: fuchsia.css +green: + output: base/%(version)s_green.css + contents: green.css +main: + output: base/%(version)s_main.css + contents: main.css +main-rtl: + output: base/%(version)s_main-rtl.css + contents: main-rtl.css +maroon: + output: base/%(version)s_maroon.css + contents: maroon.css +red: + output: base/%(version)s_red.css + contents: red.css diff --git a/ckan/public/base/javascript/apply-html-class.js b/ckan/public/base/javascript/apply-html-class.js new file mode 100644 index 00000000000..8a5d1265edc --- /dev/null +++ b/ckan/public/base/javascript/apply-html-class.js @@ -0,0 +1 @@ +document.getElementsByTagName('html')[0].className += ' js'; diff --git a/ckan/public/base/javascript/modules/resource-reorder.js b/ckan/public/base/javascript/modules/resource-reorder.js index 789b123d1a7..466e2cf025e 100644 --- a/ckan/public/base/javascript/modules/resource-reorder.js +++ b/ckan/public/base/javascript/modules/resource-reorder.js @@ -1,5 +1,6 @@ /* Module for reordering resources */ + this.ckan.module('resource-reorder', function($) { return { options: { diff --git a/ckan/public/base/javascript/resource.config b/ckan/public/base/javascript/resource.config index 8037ce0bded..b39433efcab 100644 --- a/ckan/public/base/javascript/resource.config +++ b/ckan/public/base/javascript/resource.config @@ -6,7 +6,6 @@ dont_bundle = tracking.js main = vendor/vendor ckan = vendor/bootstrap -tracking.js = vendor/jquery.js [custom render order] @@ -20,49 +19,4 @@ apply_html_class = [groups] ckan = - modules/select-switch.js - modules/slug-preview.js - modules/basic-form.js - modules/confirm-action.js - modules/api-info.js - modules/autocomplete.js - modules/custom-fields.js - modules/data-viewer.js - modules/table-selectable-rows.js - modules/resource-form.js - modules/resource-upload-field.js - modules/resource-reorder.js - modules/resource-view-reorder.js - modules/follow.js - modules/activity-stream.js - modules/dashboard.js - modules/resource-view-embed.js - view-filters.js - modules/resource-view-filters-form.js - modules/resource-view-filters.js - modules/table-toggle-more.js - modules/dataset-visibility.js - modules/media-grid.js - modules/image-upload.js - modules/followers-counter.js - main = - apply_html_class - plugins/jquery.inherit.js - plugins/jquery.proxy-all.js - plugins/jquery.url-helpers.js - plugins/jquery.date-helpers.js - plugins/jquery.slug.js - plugins/jquery.slug-preview.js - plugins/jquery.truncator.js - plugins/jquery.masonry.js - plugins/jquery.form-warning.js - plugins/jquery.images-loaded.js - - sandbox.js - module.js - pubsub.js - client.js - notify.js - i18n.js - main.js diff --git a/ckan/public/base/javascript/webassets.yml b/ckan/public/base/javascript/webassets.yml new file mode 100644 index 00000000000..88b2db83bc3 --- /dev/null +++ b/ckan/public/base/javascript/webassets.yml @@ -0,0 +1,66 @@ +main: + filters: rjsmin + output: base/%(version)s_main.js + extra: + preload: + - vendor/vendor + contents: + - apply-html-class.js + - plugins/jquery.inherit.js + - plugins/jquery.proxy-all.js + - plugins/jquery.url-helpers.js + - plugins/jquery.date-helpers.js + - plugins/jquery.slug.js + - plugins/jquery.slug-preview.js + - plugins/jquery.truncator.js + - plugins/jquery.masonry.js + - plugins/jquery.form-warning.js + - plugins/jquery.images-loaded.js + - sandbox.js + - module.js + - pubsub.js + - client.js + - notify.js + - i18n.js + - main.js + +ckan: + filters: rjsmin + output: base/%(version)s_ckan.js + extra: + preload: + - vendor/bootstrap + contents: + - modules/select-switch.js + - modules/slug-preview.js + - modules/basic-form.js + - modules/confirm-action.js + - modules/api-info.js + - modules/autocomplete.js + - modules/custom-fields.js + - modules/data-viewer.js + - modules/table-selectable-rows.js + - modules/resource-form.js + - modules/resource-upload-field.js + - modules/resource-reorder.js + - modules/resource-view-reorder.js + - modules/follow.js + - modules/activity-stream.js + - modules/dashboard.js + - modules/resource-view-embed.js + - view-filters.js + - modules/resource-view-filters-form.js + - modules/resource-view-filters.js + - modules/table-toggle-more.js + - modules/dataset-visibility.js + - modules/media-grid.js + - modules/image-upload.js + - modules/followers-counter.js + +tracking: + output: base/%(version)s_tracking.js + extra: + preload: + - vendor/jquery + contents: + - tracking.js diff --git a/ckan/public/base/vendor/resource.config b/ckan/public/base/vendor/resource.config index 1354199b6c3..b674a8ee1b5 100644 --- a/ckan/public/base/vendor/resource.config +++ b/ckan/public/base/vendor/resource.config @@ -10,30 +10,9 @@ select2/select2.css = 1 [depends] -vendor = jquery.js -bootstrap = jquery.js -fileupload = jquery.js -reorder = jquery.js - [groups] vendor = - jed.js - moment-with-locales.js - select2/select2.js - select2/select2.css - bootstrap = - bootstrap/js/bootstrap.js - font-awesome/css/font-awesome.css - fileupload = - jquery.ui.widget.js - jquery-fileupload/jquery.iframe-transport.js - jquery-fileupload/jquery.fileupload.js - reorder = - jquery.ui.core.js - jquery.ui.widget.js - jquery.ui.mouse.js - jquery.ui.sortable.js diff --git a/ckan/public/base/vendor/webassets.yml b/ckan/public/base/vendor/webassets.yml new file mode 100644 index 00000000000..3af9ccede43 --- /dev/null +++ b/ckan/public/base/vendor/webassets.yml @@ -0,0 +1,62 @@ +select2-css: + output: vendor/%(version)s_select2.css + filters: cssrewrite + contents: + - select2/select2.css + +font-awesome: + output: vendor/%(version)s_font-awesome.css + filters: cssrewrite + contents: + - font-awesome/css/font-awesome.css + +jquery: + filters: rjsmin + output: vendor/%(version)s_jquery.js + contents: + - jquery.js + +vendor: + filters: rjsmin + output: vendor/%(version)s_vendor.js + extra: + preload: + - vendor/select2-css + - vendor/jquery + contents: + - jed.js + - moment-with-locales.js + - select2/select2.js + +bootstrap: + filters: rjsmin + output: vendor/%(version)s_bootstrap.js + extra: + preload: + - vendor/font-awesome + - vendor/jquery + contents: + - bootstrap/js/bootstrap.js + +fileupload: + filters: rjsmin + output: vendor/%(version)s_fileupload.js + extra: + preload: + - vendor/jquery + contents: + - jquery.ui.widget.js + - jquery-fileupload/jquery.iframe-transport.js + - jquery-fileupload/jquery.fileupload.js + +reorder: + filters: rjsmin + output: vendor/%(version)s_reorder.js + extra: + preload: + - vendor/jquery + contents: + - jquery.ui.core.js + - jquery.ui.widget.js + - jquery.ui.mouse.js + - jquery.ui.sortable.js diff --git a/ckan/templates/base.html b/ckan/templates/base.html index 58e36311803..e3c9bf54a6e 100644 --- a/ckan/templates/base.html +++ b/ckan/templates/base.html @@ -66,7 +66,10 @@ {% endblock %} #} {%- block styles %} - {% if h.is_rtl_language() %}{% resource h.get_rtl_css()[6:] %}{% else %}{% resource g.main_css[6:] %}{% endif %} + {# TODO: store just name of asset instead of path to it. #} + {% set main_css = h.get_rtl_css() if h.is_rtl_language() else g.main_css %} + {# strip '/base/' prefix and '.css' suffix #} + {% asset main_css[6:-4] %} {% endblock %} {% block head_extras %} @@ -74,6 +77,8 @@ {{ g.template_head_end | safe }} {% endblock %} + {# render all assets included in styles block #} + {{ h.render_assets('style') }} {%- block custom_styles %} {%- if g.site_custom_css -%}