Merge branch 'loader_module'

.travis.yml

@@ -37,15 +37,19 @@ matrix:
     - language: node_js
       node_js: 4.9
       os: osx
-      env: TRAVIS_PYTHON_VERSION=3.4.5
+      env: TRAVIS_PYTHON_VERSION=3.4.8
     - language: node_js
       node_js: 6.14
       os: osx
-      env: TRAVIS_PYTHON_VERSION=3.5.3
+      env: TRAVIS_PYTHON_VERSION=3.5.5
     - language: node_js
       node_js: 8.11
       os: osx
-      env: TRAVIS_PYTHON_VERSION=3.6.0
+      env: TRAVIS_PYTHON_VERSION=3.6.6
+    - language: node_js
+      node_js: 10
+      os: osx
+      env: TRAVIS_PYTHON_VERSION=3.7.0
 
 before_install:
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
@@ -82,7 +86,7 @@ script:
   # run the first test without calmjs.dev.
   - python -m unittest calmjs.webpack.tests.make_suite
   # run the real test with all optional dependencies.
-  - pip install calmjs.dev
+  - pip install -e .[dev]
   - coverage run --include=src/* -m unittest calmjs.webpack.tests.make_suite
   - coverage report -m
 after_success:

CHANGES.rst

@@ -4,6 +4,19 @@ Changelog
 1.1.0 (Unreleased)
 ------------------
 
+- Provide support of prefix-free loaders through a customized webpack
+  loader module registry; this one also works in tandem with the
+  ``calmjs.module`` registry.  [
+  `#5 <https://github.com/calmjs/calmjs.webpack/issues/5>`_
+  ]
+
+  - Integrate the support of the package resource loader registry
+    introduced by ``calmjs-3.3.0``.
+  - Bumped supported ``calmjs.dev`` to version 2.2.0 such that the
+    correct test loader registries can be automatically acquired.  This
+    also makes it possible to declare test data files as resources for
+    JavaScript tests in the most straightforward manner as possible
+
 - The base loader plugin handler will also generate a modname prefixed
   with ``./``, in an attempt to provide a more natural include mechanism
   from within certain webpack loader contexts, as a number of them will

README.rst

@@ -536,26 +536,166 @@ feature may be disabled by the ``--disable-calmjs-compat`` flag.
 Handling of Webpack loaders
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-If a provided JavaScript module imports a target using the inline loader
-syntax, the default registry ``calmjs.webpack.loaderplugins`` will
-resolve a generic handler to copy the target files.  This generic
-handler supports the chaining of loaders.  If this behavior is unwanted,
-a static registry is defined at ``calmjs.webpack.static.loaderplugins``
-for this purpose.  If a mix of the two is needed (e.g. where some
-specific loader require special handling), it is also possible to
-register the specific handler to override the generic handler for that
-specific loader.
-
-So if some JavaScript code contain a require statement like:
+Webpack provides the support of loaders to handle the importing of
+files that are not JavaScript sources, such that the importing of files
+such as text of JSON file can be done seamlessly with their import
+system.  Python packages may export these resource files to make them
+available for the webpack environment through a subsidiary loader
+registry ``calmjs.module.loader`` that will expose the relevant resource
+files for namespaces defined in ``calmjs.module`` registry for the given
+package.
+
+As for the integration of the loaders themselves with the Calmjs
+toolchain system, a separate loaderplugin registry must be specified.
+For the webpack toolchain, the ``calmjs.webpack.loaderplugins`` registry
+is assigned as the default, which will resolve a generic handler to
+handle the target resource files.  This generic handler supports the
+chaining of loaders.  Specific handlers for the resource types may be
+assigned directly to that registry, for example if an alternative
+Node.js package is required to override the default automatically
+generated handler.
+
+If this automatic resolution behavior is unwanted, a static registry is
+defined at ``calmjs.webpack.static.loaderplugins`` for this purpose,
+however, to enable the usage of this registry at this time require the
+usage of the toolchain API directly.
+
+As for specifying which resource files contained in Python packages are
+to be made available for the webpack environment, the subsidiary module
+loader registry ``calmjs.module.loader`` should be used.
+
+Putting all this together, the ``example`` package defined earlier is
+now extended to expose various types of resource files for usage:
+
+.. code:: ini
+
+    [calmjs.module]
+    example.lib = example.lib
+    example.app = example.app
+
+    [calmjs.module.loader]
+    json = json[json]
+    text = text[txt,json]
+
+This would make the relevant resource files under both the
+``example/lib`` and ``example/app`` namespace available under the
+relevant loaders, such that if some JavaScript code contain a require
+statement like either of the following:
 
 .. code:: JavaScript
 
-    var readme = require('text!readme.txt');
+    var readme_lib = require('text!example/lib/readme.txt');
+    var data_txt = require('text!example/lib/data.json');
+    var data = require('json!example/lib/data.json');
+
+Please do note that this will make available these full names for
+dynamic module imports as the full names are exposed out of the
+generated artifact.
+
+The default loaderplugin handler registry will provide the standard
+handler as none are defined, such that those require statements with an
+explicit loader prefix will be resolved correctly.  However, webpack has
+largely deprecated the usage of explicit loader prefixes, and prefers a
+syntax that imports without the prefix specified.  This requires a
+different handling method, documented in the next section.
+
+Handling of ``require()`` without explicit ``loader!`` prefixes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If the support of the RequireJS/AMD tooling is ignored (note that this
+will affect any dependent Python packages that make use of this code
+directly, as it does not currently support this import method yet), the
+bare import syntax may be used, for example:
+
+.. code:: JavaScript
+
+    var readme_app = require('example/app/readme.txt');
+    var style = require('example/app/style.css');
+
+One reason why compatibility across all toolchains, especially with
+loaders, is difficult if not impossible to implement is due to the many
+variations with there are a variety of methods implemented by the
+different Node.js tooling.  For instance, importing stylesheets from
+within webpack is usually done by chaining the ``style-loader`` with
+whatever specific stylesheet loader, such as ``css-loader`` or
+``sass-loader`` down the chain.  While it is possible to define the
+following entry point on the ``calmjs.module.loader`` registry::
+
+    [calmjs.module.loader]
+    style!css = css[css]
+
+With the above definition, importing stylesheet resources using the
+complete syntax (i.e. ``require('style!css!example/app/style.css');``
+will work, but it is incompatible with the ``require-css`` loader as
+it does not necessarily support the chaining of the ``style!`` loader
+prefix as the RequireJS version of the plugin will apply the styles
+immediately without that (this is why the loader-prefixes are considered
+non-portable).
+
+So to better support this more agnostic use case, |webpack| provides a
+``module.rules`` section that dictates how the specific module is to be
+loaded, so that the loaderprefix-free loading can be achieved (i.e. the
+previous JavaScript fragment).  To specifically support this through
+Calmjs, the resources entry points should be defined under the
+``calmjs.module.webpackloader`` registry instead of the common
+``calmjs.module.loader`` registry.  For example:
+
+.. code:: ini
+
+    [calmjs.module]
+    example.lib = example.lib
+    example.app = example.app
+
+    [calmjs.module.webpackloader]
+    style-loader!css-loader = stylesheet[css]
+    text-loader = txt[txt]
+
+Please note that while it is possible to also define the entry point
+like the following:
+
+.. code:: ini
 
-And there exists a custom Calmjs module registry that provide those
-sources, the default loaderplugin handler registry will provide a
-standard handler that will process this, provided the loader package is
-available along with webpack on the working Node.js environment.
+    [calmjs.module.webpackloader]
+    style!css = stylesheet[css]
+
+Previously this relies on a legacy behavior which |webpack| removed, but
+it is still supported by |calmjs| and |calmjs.webpack| simply due to the
+generic support of this format, but given that this registry is
+specifically for webpack, there is should be no issue if the webpack
+specific syntax is used, if the following caveats are addressed.
+
+Please note that if a given file name extension is defined on multiple
+webpackloaders (note that the text loader has removed json as a file
+name extension), the resulting behavior is undefined as the generated
+configuration will not guarantee that the loaders are chained together
+in the expected manner, as both loaders will be applied to the selected
+files under an undefined ordering.
+
+Module names exported by the ``calmjs.module.webpackloader`` will not be
+made available the gathered module or import names for the dynamic
+import module when processed by the default loader plugin handlers, as
+there exists a number of subtle complexities that severely complicates
+exposing these names in a meaningful manner for usage within the calmjs
+system.  In effect, no dynamic imports will be possible after the
+construction of the artifact.
+
+If a file name extension defined in this is also defined in the
+``calmjs.module.loader`` registry, it will also cause complications if
+the dynamic import module was also generated.  This issue is related to
+the issue outlined by the previous paragraph.
+
+If multiple loaders are required (such as for the case of stylesheets),
+use the chained syntax as in the ``style-loader!css-loader`` definition
+to ensure that they are applied correctly, much like they would have
+been if they were prefixed on the imports directly for this particular
+Python package (i.e. ``style!css!``).
+
+Much like the standard ``calmjs.module.loader`` registry, the
+definitions for any given filename extensions declared under the
+``calmjs.module.webpackloader`` registry are local to the package, so
+that definitions that make use of a different set of loaders from an
+upstream or downstream package will not cause interference with how they
+are applied.
 
 Testing standalone, finalized webpack artifacts
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

appveyor.yml

@@ -25,7 +25,7 @@ test_script:
   # verify tests working without optional dependencies
   - python -m unittest calmjs.webpack.tests.make_suite
   # run the full test suite using all optional dependencies
-  - pip install calmjs.dev
+  - pip install -e .[dev]
   - coverage run --include=src/* -m unittest calmjs.webpack.tests.make_suite
   - coverage report -m
 

setup.py

@@ -69,20 +69,26 @@
     include_package_data=True,
     zip_safe=False,
     install_requires=[
-        'calmjs>=3.0.0dev',
-        'calmjs.parse',
+        'calmjs>=3.3.0dev',
     ],
     extras_require={
         'dev': [
-            'calmjs.dev>=2.0.0,<3',
+            'calmjs.dev>=2.2.0,<3',
         ],
     },
     entry_points={
         'calmjs.registry': [
             'calmjs.webpack.loaderplugins = '
             'calmjs.webpack.loaderplugin:AutogenWebpackLoaderPluginRegistry',
+
             'calmjs.webpack.static.loaderplugins = '
             'calmjs.loaderplugin:LoaderPluginRegistry',
+
+            'calmjs.module.webpackloader = '
+            'calmjs.webpack.loaderplugin:WebpackModuleLoaderRegistry',
+
+            'calmjs.module.tests.webpackloader = '
+            'calmjs.webpack.loaderplugin:WebpackModuleLoaderRegistry',
         ],
         'calmjs.webpack.static.loaderplugins': [
             'text = calmjs.webpack.loaderplugin:WebpackLoaderHandler',

src/calmjs/webpack/README.rst

@@ -1,9 +1,9 @@
 Module layout
 =============
 
-This module, ``calmjs.webpack``, also follows the ``calmjs`` module
-layout order, but for clarity sake the modules defined here are included
-in the descriptions.
+This module, ``calmjs.webpack``, also mostly follows the ``calmjs``
+module layout order, but for clarity sake the modules defined here are
+included in the descriptions.
 
 interrogation
     Helpers for interrogating a webpack artifact file.
@@ -26,7 +26,9 @@ env
 
 loaderplugin
     Integration with the loader plugin system; include the base loader
-    plugins and an automatic registry system.
+    plugins and an automatic registry system.  Note that for the webpack
+    integration, loaderplugins have a much higher precedence as both the
+    toolchain and cli will require this module.
 
 registry
     Currently contain just one registry implementation, which is for

src/calmjs/webpack/base.py

@@ -5,10 +5,17 @@
 
 from __future__ import unicode_literals
 
+from collections import namedtuple
+
 # keys
 
 # enable calmjs compatibility - i.e. the dynamic import feature
 CALMJS_COMPAT = 'calmjs_compat'
+# the map from a module name to the loader needed; used by the various
+# functions and methods in the loaderplugin module
+# see definition of WebpackModuleLoaderRegistryKey later
+CALMJS_WEBPACK_MODNAME_LOADER_MAP = 'calmjs_webpack_modname_loader_map'
+
 # The spec key for storing the base webpack configuration.
 WEBPACK_CONFIG = 'webpack_config'
 # The key for the webpack.output.library
@@ -23,6 +30,8 @@
 # For webpack loaderplugin integration - this is the spec key - note that
 # this is NOT for webpack plugins which are a separate type of things
 WEBPACK_LOADERPLUGINS = 'webpack_loaderplugins'
+# for the module.rules section; used by loaderplugin module
+WEBPACK_MODULE_RULES = 'webpack_module_rules'
 # for the configuration in webpack config
 WEBPACK_RESOLVELOADER_ALIAS = 'webpack_resolveloader_alias'
 
@@ -67,3 +76,11 @@
     "root": DEFAULT_BOOTSTRAP_EXPORT,
     "amd": DEFAULT_BOOTSTRAP_EXPORT,
 }
+
+
+# due to webpack specific requirements, a special type for the key is
+# needed for the WebpackModuleLoaderRegistry such that the correct
+# handling mechanism may be done.
+CALMJS_WEBPACK_MODULE_LOADER_SUFFIX = '.webpackloader'
+WebpackModuleLoaderRegistryKey = namedtuple(
+    'WebpackModuleLoaderRegistryKey', ['loader', 'modname'])

src/calmjs/webpack/cli.py

@@ -27,10 +27,12 @@
 from calmjs.webpack.base import VERIFY_IMPORTS
 
 from calmjs.webpack.base import CALMJS_WEBPACK_LOADERPLUGINS
+
 from calmjs.webpack.base import DEFAULT_BOOTSTRAP_EXPORT
 from calmjs.webpack.base import DEFAULT_BOOTSTRAP_EXPORT_CONFIG
 
 from calmjs.webpack.toolchain import WebpackToolchain
+from calmjs.webpack.loaderplugin import normalize_and_register_webpackloaders
 
 from calmjs.webpack.dist import generate_transpile_sourcepaths
 from calmjs.webpack.dist import generate_bundle_sourcepaths
@@ -245,14 +247,21 @@ def create_spec(
     spec[VERIFY_IMPORTS] = verify_imports
     spec[CALMJS_LOADERPLUGIN_REGISTRY_NAME] = calmjs_loaderplugin_registry_name
 
-    spec_update_sourcepath_filter_loaderplugins(
-        spec, generate_transpile_sourcepaths(
-            package_names=package_names,
-            registries=source_registries,
-            method=sourcepath_method,
-        ), 'transpile_sourcepath',
+    raw_transpile_sourcepaths = generate_transpile_sourcepaths(
+        package_names=package_names,
+        registries=source_registries,
+        method=sourcepath_method,
     )
 
+    # filter out the WebpackModuleLoaderRegistryKey so that they are
+    # registered properly onto the spec for further processing using the
+    # helpers in the loaderplugin module for this package.
+    transpile_sourcepaths = normalize_and_register_webpackloaders(
+        spec, raw_transpile_sourcepaths)
+
+    spec_update_sourcepath_filter_loaderplugins(
+        spec, transpile_sourcepaths, 'transpile_sourcepath')
+
     spec_update_sourcepath_filter_loaderplugins(
         spec, generate_bundle_sourcepaths(
             package_names=package_names,