From 645828f1385f282e8c6ff782fa1bcbec75c51447 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Tue, 18 Dec 2018 20:12:14 +0200 Subject: [PATCH 01/21] At least it works --- .gitignore | 3 + ckan/cli/asset.py | 46 ++++++ ckan/cli/cli.py | 5 +- ckan/config/environment.py | 3 + ckan/config/middleware/flask_app.py | 13 +- ckan/lib/extract.py | 1 + ckan/lib/helpers.py | 3 + ckan/lib/jinja_extensions.py | 26 +++- ckan/lib/webassets_tools.py | 138 ++++++++++++++++++ ckan/plugins/toolkit.py | 27 ++-- ckan/public/base/css/webassets.yml | 18 +++ .../base/javascript/apply-html-class.js | 1 + ckan/public/base/javascript/webassets.yml | 66 +++++++++ ckan/public/base/vendor/webassets.yml | 62 ++++++++ ckan/templates/base.html | 13 +- ckan/templates/package/new_resource.html | 3 +- .../templates/package/resource_edit_base.html | 3 +- ckan/templates/package/resource_views.html | 3 +- ckan/templates/package/resources.html | 3 +- ckan/templates/page.html | 15 +- ckanext/datatablesview/public/webassets.yml | 27 ++++ .../templates/datatables/datatables_view.html | 3 +- .../v15_fanstatic/fanstatic/webassets.yml | 4 + .../v15_fanstatic/templates/base.html | 12 +- .../fanstatic/webassets.yml | 4 + .../templates/snippets/package_item.html | 15 +- .../v17_popover/fanstatic/webassets.yml | 4 + .../templates/snippets/package_item.html | 7 +- .../v18_snippet_api/fanstatic/webassets.yml | 12 ++ .../templates/snippets/package_item.html | 5 +- .../v19_01_error/fanstatic/webassets.yml | 12 ++ .../fanstatic/webassets.yml | 12 ++ .../v20_pubsub/fanstatic/webassets.yml | 12 ++ .../fanstatic/webassets.yml | 13 ++ .../templates/snippets/package_item.html | 8 +- .../reclineview/theme/public/webassets.yml | 45 ++++++ .../theme/templates/recline_view.html | 3 +- .../stats/public/ckanext/stats/webassets.yml | 8 + .../stats/templates/ckanext/stats/index.html | 3 +- ckanext/textview/theme/public/webassets.yml | 15 ++ .../textview/theme/templates/text_view.html | 3 +- requirements.txt | 2 + 42 files changed, 634 insertions(+), 47 deletions(-) create mode 100644 ckan/cli/asset.py create mode 100644 ckan/lib/webassets_tools.py create mode 100644 ckan/public/base/css/webassets.yml create mode 100644 ckan/public/base/javascript/apply-html-class.js create mode 100644 ckan/public/base/javascript/webassets.yml create mode 100644 ckan/public/base/vendor/webassets.yml create mode 100644 ckanext/datatablesview/public/webassets.yml create mode 100644 ckanext/example_theme_docs/v15_fanstatic/fanstatic/webassets.yml create mode 100644 ckanext/example_theme_docs/v16_initialize_a_javascript_module/fanstatic/webassets.yml create mode 100644 ckanext/example_theme_docs/v17_popover/fanstatic/webassets.yml create mode 100644 ckanext/example_theme_docs/v18_snippet_api/fanstatic/webassets.yml create mode 100644 ckanext/example_theme_docs/v19_01_error/fanstatic/webassets.yml create mode 100644 ckanext/example_theme_docs/v19_02_error_handling/fanstatic/webassets.yml create mode 100644 ckanext/example_theme_docs/v20_pubsub/fanstatic/webassets.yml create mode 100644 ckanext/example_theme_docs/v21_custom_jquery_plugin/fanstatic/webassets.yml create mode 100644 ckanext/reclineview/theme/public/webassets.yml create mode 100644 ckanext/stats/public/ckanext/stats/webassets.yml create mode 100644 ckanext/textview/theme/public/webassets.yml diff --git a/.gitignore b/.gitignore index 39267c75f22..dbc4ad040ac 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,9 @@ fl_notes.txt # custom style ckan/public/base/less/custom.less +# webassets +ckan/public/webassets + # nosetest coverage output .coverage htmlcov/* diff --git a/ckan/cli/asset.py b/ckan/cli/asset.py new file mode 100644 index 00000000000..e10136de8db --- /dev/null +++ b/ckan/cli/asset.py @@ -0,0 +1,46 @@ +# encoding: utf-8 + +import logging + +import click +from webassets import script +from webassets.exceptions import BundleError + +from ckan.lib import webassets_tools +from ckan.cli import error_shout + +log = logging.getLogger(__name__) + + +@click.group(name=u'asset', short_help=u'WebAssets commands') +def asset(): + pass + + +@asset.command(u'build', short_help=u'Builds all bundles.') +def build(): + u'''Builds bundles, regardless of whether they are changed or not.''' + script.main(['build'], webassets_tools.env) + click.secho(u'Compile assets: SUCCESS', fg=u'green', bold=True) + + +@asset.command(u'watch', short_help=u'Watch changes in source files.') +def watch(): + u'''Start a daemon which monitors source files, and rebuilds bundles. + + This can be useful during development, if building is not + instantaneous, and you are losing valuable time waiting for the + build to finish while trying to access your site. + + ''' + script.main(['watch'], webassets_tools.env) + + +@asset.command(u'clean', short_help=u'Clear cache.') +def clean(): + u'''Will clear out the cache, which after a while can grow quite large.''' + try: + script.main(['clean'], webassets_tools.env) + except BundleError as e: + return error_shout(e) + click.secho(u'Clear cache: SUCCESS', fg=u'green', bold=True) diff --git a/ckan/cli/cli.py b/ckan/cli/cli.py index 236196c88c6..fe4e99684b9 100644 --- a/ckan/cli/cli.py +++ b/ckan/cli/cli.py @@ -4,7 +4,9 @@ import click -from ckan.cli import click_config_option, db, load_config, search_index, server +from ckan.cli import ( + click_config_option, db, load_config, search_index, server, asset +) from ckan.config.middleware import make_app log = logging.getLogger(__name__) @@ -28,3 +30,4 @@ def ckan(ctx, config, *args, **kwargs): ckan.add_command(server.run) ckan.add_command(db.db) ckan.add_command(search_index.search_index) +ckan.add_command(asset.asset) diff --git a/ckan/config/environment.py b/ckan/config/environment.py index 963d7e5d726..e99c8884c42 100644 --- a/ckan/config/environment.py +++ b/ckan/config/environment.py @@ -24,6 +24,7 @@ import ckan.logic as logic import ckan.authz as authz import ckan.lib.jinja_extensions as jinja_extensions +from ckan.lib.webassets_tools import webassets_init from ckan.lib.i18n import build_js_translations from ckan.common import _, ungettext, config @@ -163,6 +164,8 @@ def update_config(): plugin might have changed the config values (for instance it might change ckan.site_url) ''' + webassets_init() + for plugin in p.PluginImplementations(p.IConfigurer): # must do update in place as this does not work: # config = plugin.update_config(config) diff --git a/ckan/config/middleware/flask_app.py b/ckan/config/middleware/flask_app.py index 3593ff1c937..c58baed57b9 100644 --- a/ckan/config/middleware/flask_app.py +++ b/ckan/config/middleware/flask_app.py @@ -7,7 +7,7 @@ import itertools import pkgutil -from flask import Flask, Blueprint +from flask import Flask, Blueprint, send_from_directory from flask.ctx import _AppCtxGlobals from flask.sessions import SessionInterface @@ -22,6 +22,7 @@ from repoze.who.config import WhoConfig from repoze.who.middleware import PluggableAuthenticationMiddleware +import ckan import ckan.model as model from ckan.lib import base from ckan.lib import helpers @@ -181,6 +182,16 @@ def hello_world(): def hello_world_post(): return 'Hello World, this was posted to Flask' + # WebAssets + public_folder = config.get('ckan.base_public_folder') + webassets_folder = os.path.join( + os.path.dirname(ckan.__file__), public_folder, 'webassets' + ) + + @app.route('/webassets/') + def webassets(path): + return send_from_directory(webassets_folder, path) + # Auto-register all blueprints defined in the `views` folder _register_core_blueprints(app) _register_error_handler(app) diff --git a/ckan/lib/extract.py b/ckan/lib/extract.py index 6a57ce664ac..6c67d1d067d 100644 --- a/ckan/lib/extract.py +++ b/ckan/lib/extract.py @@ -11,6 +11,7 @@ ckan.lib.jinja_extensions.CkanExtend, ckan.lib.jinja_extensions.LinkForExtension, ckan.lib.jinja_extensions.ResourceExtension, + ckan.lib.jinja_extensions.AssetExtension, ckan.lib.jinja_extensions.UrlForStaticExtension, ckan.lib.jinja_extensions.UrlForExtension ''' diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index 076c07de844..9caecca9fee 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -49,6 +49,7 @@ import ckan from ckan.common import _, ungettext, c, g, request, session, json +from ckan.lib.webassets_tools import include_asset, render_assets from markupsafe import Markup, escape @@ -2645,6 +2646,8 @@ def clean_html(html): core_helper(converters.asbool) # Useful additions from the stdlib. core_helper(urlencode) +core_helper(include_asset) +core_helper(render_assets) def load_plugin_helpers(): diff --git a/ckan/lib/jinja_extensions.py b/ckan/lib/jinja_extensions.py index 82851fd7919..678d7a1cf57 100644 --- a/ckan/lib/jinja_extensions.py +++ b/ckan/lib/jinja_extensions.py @@ -34,7 +34,8 @@ def get_jinja_env_options(): LinkForExtension, ResourceExtension, UrlForStaticExtension, - UrlForExtension], + UrlForExtension, + AssetExtension], ) @@ -312,7 +313,7 @@ def _call(cls, args, kwargs): return h.nav_link(*args, **kwargs) class ResourceExtension(BaseExtension): - ''' Custom include_resource tag + ''' Deprecated. Custom include_resource tag. {% resource %} @@ -323,12 +324,33 @@ class ResourceExtension(BaseExtension): @classmethod def _call(cls, args, kwargs): + log.warn( + '`resource` tag is deprecated. ' + 'Use `assets`' + ' instead') assert len(args) == 1 assert len(kwargs) == 0 h.include_resource(args[0], **kwargs) return '' +class AssetExtension(BaseExtension): + ''' Custom include_asset tag. + + {% asset %} + + see lib.webassets_tools.include_asset() for more details. + ''' + + tags = set(['asset']) + + @classmethod + def _call(cls, args, kwargs): + assert len(args) == 1 + assert len(kwargs) == 0 + h.include_asset(args[0]) + return '' + ''' The following function is based on jinja2 code diff --git a/ckan/lib/webassets_tools.py b/ckan/lib/webassets_tools.py new file mode 100644 index 00000000000..96d2cada7b0 --- /dev/null +++ b/ckan/lib/webassets_tools.py @@ -0,0 +1,138 @@ +# encoding: utf-8 + +import logging +import os + +from markupsafe import Markup + +from webassets import Environment +from webassets.loaders import YAMLLoader + +from ckan.common import config, g + +logger = logging.getLogger(__name__) +env = None + + +def create_library(name, path): + """Create WebAssets library(set of Bundles). + """ + config_path = os.path.join(path, u'webassets.yml') + if not os.path.exists(config_path): + return + + library = YAMLLoader(config_path).load_bundles() + bundles = { + '/'.join([name, key]): bundle + for key, bundle + in library.items() + } + + # Unfortunately, you'll get an error attempting to register bundle + # with the same name twice. For now, let's just pop existing + # bundle and avoid name-conflicts + # TODO: make PR into webassets with preferable solution + for name, bundle in bundles.items(): + env._named_bundles.pop(name, None) + env.register(name, bundle) + + env.append_path(path) + + +def webassets_init(): + global env + + public = config.get(u'ckan.base_public_folder') + + public_folder = os.path.abspath(os.path.join( + os.path.dirname(__file__), '..', public)) + + base_path = os.path.join(public_folder, u'base') + static_path = os.path.join(public_folder, u'webassets') + + env = Environment() + env.directory = static_path + logger.warn(u'DEBUG: %s', config.get(u'debug', False)) + env.debug = True + env.url = u'/webassets/' + + env.append_path(base_path, u'/base/') + + logger.debug(u'Base path {0}'.format(base_path)) + create_library(u'vendor', os.path.join( + base_path, u'vendor')) + + create_library(u'base', os.path.join(base_path, u'javascript')) + + create_library(u'datapreview', os.path.join(base_path, u'datapreview')) + + create_library(u'css', os.path.join(base_path, u'css')) + + +def _make_asset_collection(): + return {u'style': [], u'script': [], u'included': set()} + + +def include_asset(name): + try: + if not g.webassets: + raise AttributeError(u'WebAssets not initialized yet') + except AttributeError: + g.webassets = _make_asset_collection() + if name in g.webassets[u'included']: + return + + try: + bundle = env[name] + except KeyError: + logger.error(u'Trying to include unknown asset: <{}>'.format(name)) + return + + deps = bundle.extra.get(u'preload', []) + + # Using DFS may lead to infinite recursion(unlikely, because + # extensions rarely depends on each other), so there is a sense to + # memoize visited routes. + + # TODO: consider infinite loop prevention for assets that depends + # on each other + for dep in deps: + include_asset(dep) + + urls = bundle.urls() + type_ = None + for url in urls: + link = url.split(u'?')[0] + if link.endswith(u'.css'): + type_ = u'style' + break + elif link.endswith(u'.js'): + type_ = u'script' + break + else: + logger.warn(u'Undefined asset type: {}'.format(urls)) + return + g.webassets[type_].extend(urls) + g.webassets[u'included'].add(name) + + +def _to_tag(url, type_): + if type_ == u'style': + return u''.format(url) + elif type_ == u'script': + return u''.format(url) + return u'' + + +def render_assets(type_): + try: + assets = g.webassets + except AttributeError: + return u'' + + if not assets: + return u'' + collection = assets[type_] + tags = u'\n'.join([_to_tag(asset, type_) for asset in assets[type_]]) + collection[:] = [] + return Markup(tags) diff --git a/ckan/plugins/toolkit.py b/ckan/plugins/toolkit.py index da2fbcd8cca..858b0b43ace 100644 --- a/ckan/plugins/toolkit.py +++ b/ckan/plugins/toolkit.py @@ -337,8 +337,10 @@ def _add_served_directory(cls, config, relative_path, config_var): assert config_var in ('extra_template_paths', 'extra_public_paths') # we want the filename that of the function caller but they will # have used one of the available helper functions - frame, filename, line_number, function_name, lines, index =\ - inspect.getouterframes(inspect.currentframe())[2] + # TODO: starting from python 3.5, `inspect.stack` returns list + # of named tuples `FrameInfo`. Don't forget to remove + # `getframeinfo` wrapper after migration. + filename = inspect.getframeinfo(inspect.stack()[2][0]).filename this_dir = os.path.dirname(filename) absolute_path = os.path.join(this_dir, relative_path) @@ -350,24 +352,31 @@ def _add_served_directory(cls, config, relative_path, config_var): @classmethod def _add_resource(cls, path, name): - '''Add a Fanstatic resource library to CKAN. + '''Add a WebAssets library to CKAN. - Fanstatic libraries are directories containing static resource files - (e.g. CSS, JavaScript or image files) that can be accessed from CKAN. + WebAssets libraries are directories containing static resource + files (e.g. CSS, JavaScript or image files) that can be + compiled into WebAsset Bundles. See :doc:`/theming/index` for more details. ''' import inspect import os + from ckan.lib.webassets_tools import create_library - # we want the filename that of the function caller but they will - # have used one of the available helper functions - frame, filename, line_number, function_name, lines, index =\ - inspect.getouterframes(inspect.currentframe())[1] + # we want the filename that of the function caller but they + # will have used one of the available helper functions + # TODO: starting from python 3.5, `inspect.stack` returns list + # of named tuples `FrameInfo`. Don't forget to remove + # `getframeinfo` wrapper after migration. + filename = inspect.getframeinfo(inspect.stack()[1][0]).filename this_dir = os.path.dirname(filename) absolute_path = os.path.join(this_dir, path) + create_library(name, absolute_path) + + # TODO: remove next two lines after dropping Fanstatic support import ckan.lib.fanstatic_resources ckan.lib.fanstatic_resources.create_library(name, absolute_path) diff --git a/ckan/public/base/css/webassets.yml b/ckan/public/base/css/webassets.yml new file mode 100644 index 00000000000..2aa2e470041 --- /dev/null +++ b/ckan/public/base/css/webassets.yml @@ -0,0 +1,18 @@ +fuchsia: + output: base/%(version)s_fuchsia.css + contents: fuchsia.css +green: + output: base/%(version)s_green.css + contents: green.css +main: + output: base/%(version)s_main.css + contents: main.css +main-rtl: + output: base/%(version)s_main-rtl.css + contents: main-rtl.css +maroon: + output: base/%(version)s_maroon.css + contents: maroon.css +red: + output: base/%(version)s_red.css + contents: red.css diff --git a/ckan/public/base/javascript/apply-html-class.js b/ckan/public/base/javascript/apply-html-class.js new file mode 100644 index 00000000000..8a5d1265edc --- /dev/null +++ b/ckan/public/base/javascript/apply-html-class.js @@ -0,0 +1 @@ +document.getElementsByTagName('html')[0].className += ' js'; diff --git a/ckan/public/base/javascript/webassets.yml b/ckan/public/base/javascript/webassets.yml new file mode 100644 index 00000000000..88b2db83bc3 --- /dev/null +++ b/ckan/public/base/javascript/webassets.yml @@ -0,0 +1,66 @@ +main: + filters: rjsmin + output: base/%(version)s_main.js + extra: + preload: + - vendor/vendor + contents: + - apply-html-class.js + - plugins/jquery.inherit.js + - plugins/jquery.proxy-all.js + - plugins/jquery.url-helpers.js + - plugins/jquery.date-helpers.js + - plugins/jquery.slug.js + - plugins/jquery.slug-preview.js + - plugins/jquery.truncator.js + - plugins/jquery.masonry.js + - plugins/jquery.form-warning.js + - plugins/jquery.images-loaded.js + - sandbox.js + - module.js + - pubsub.js + - client.js + - notify.js + - i18n.js + - main.js + +ckan: + filters: rjsmin + output: base/%(version)s_ckan.js + extra: + preload: + - vendor/bootstrap + contents: + - modules/select-switch.js + - modules/slug-preview.js + - modules/basic-form.js + - modules/confirm-action.js + - modules/api-info.js + - modules/autocomplete.js + - modules/custom-fields.js + - modules/data-viewer.js + - modules/table-selectable-rows.js + - modules/resource-form.js + - modules/resource-upload-field.js + - modules/resource-reorder.js + - modules/resource-view-reorder.js + - modules/follow.js + - modules/activity-stream.js + - modules/dashboard.js + - modules/resource-view-embed.js + - view-filters.js + - modules/resource-view-filters-form.js + - modules/resource-view-filters.js + - modules/table-toggle-more.js + - modules/dataset-visibility.js + - modules/media-grid.js + - modules/image-upload.js + - modules/followers-counter.js + +tracking: + output: base/%(version)s_tracking.js + extra: + preload: + - vendor/jquery + contents: + - tracking.js diff --git a/ckan/public/base/vendor/webassets.yml b/ckan/public/base/vendor/webassets.yml new file mode 100644 index 00000000000..3af9ccede43 --- /dev/null +++ b/ckan/public/base/vendor/webassets.yml @@ -0,0 +1,62 @@ +select2-css: + output: vendor/%(version)s_select2.css + filters: cssrewrite + contents: + - select2/select2.css + +font-awesome: + output: vendor/%(version)s_font-awesome.css + filters: cssrewrite + contents: + - font-awesome/css/font-awesome.css + +jquery: + filters: rjsmin + output: vendor/%(version)s_jquery.js + contents: + - jquery.js + +vendor: + filters: rjsmin + output: vendor/%(version)s_vendor.js + extra: + preload: + - vendor/select2-css + - vendor/jquery + contents: + - jed.js + - moment-with-locales.js + - select2/select2.js + +bootstrap: + filters: rjsmin + output: vendor/%(version)s_bootstrap.js + extra: + preload: + - vendor/font-awesome + - vendor/jquery + contents: + - bootstrap/js/bootstrap.js + +fileupload: + filters: rjsmin + output: vendor/%(version)s_fileupload.js + extra: + preload: + - vendor/jquery + contents: + - jquery.ui.widget.js + - jquery-fileupload/jquery.iframe-transport.js + - jquery-fileupload/jquery.fileupload.js + +reorder: + filters: rjsmin + output: vendor/%(version)s_reorder.js + extra: + preload: + - vendor/jquery + contents: + - jquery.ui.core.js + - jquery.ui.widget.js + - jquery.ui.mouse.js + - jquery.ui.sortable.js diff --git a/ckan/templates/base.html b/ckan/templates/base.html index 58e36311803..eddeff8a894 100644 --- a/ckan/templates/base.html +++ b/ckan/templates/base.html @@ -66,7 +66,10 @@ {% endblock %} #} {%- block styles %} - {% if h.is_rtl_language() %}{% resource h.get_rtl_css()[6:] %}{% else %}{% resource g.main_css[6:] %}{% endif %} + {# TODO: store just name of asset instead of path to it. #} + {% set main_css = h.get_rtl_css() if h.is_rtl_language() else g.main_css %} + {# strip '/base/' prefix and '.css' suffix #} + {% asset main_css[6:-4] %} {% endblock %} {% block head_extras %} @@ -74,6 +77,8 @@ {{ g.template_head_end | safe }} {% endblock %} + {# render all assets included in styles block #} + {{ h.render_assets('style') }} {%- block custom_styles %} {%- if g.site_custom_css -%}