feat: Serve theme static files from VAADIN/static (#9451)

Static files in META-INF/VAADIN/static will now be served
on request to VAADIN/static.
Added new webpack loader that changes app theme
css urls targeting ./ and ../ to be VAADIN/static/ instead.
Where ../ can not go outside of the application theme folder.

Fixes #9405
# Conflicts:
#	flow-server/src/main/java/com/vaadin/flow/server/StaticFileServer.java
#	flow-server/src/main/java/com/vaadin/flow/server/frontend/NodeUpdater.java
#	flow-server/src/main/java/com/vaadin/flow/server/frontend/TaskUpdateWebpack.java

flow-server/src/main/java/com/vaadin/flow/server/StaticFileServer.java

@@ -110,7 +110,7 @@ public boolean serveStaticResource(HttpServletRequest request,
         }
 
         URL resourceUrl = null;
-        if (isAllowedVAADINBuildUrl(filenameWithPath)) {
+        if (isAllowedVAADINBuildOrStaticUrl(filenameWithPath)) {
             resourceUrl = servletService.getClassLoader()
                     .getResource("META-INF" + filenameWithPath);
         }
@@ -194,7 +194,7 @@ private String fixIncorrectWebjarPath(String requestFilename) {
      *            requested filename containing path
      * @return true if we are ok to try serving the file
      */
-    private boolean isAllowedVAADINBuildUrl(String filenameWithPath) {
+    private boolean isAllowedVAADINBuildOrStaticUrl(String filenameWithPath) {
         if (deploymentConfiguration.isCompatibilityMode()) {
             getLogger().trace(
                     "Serving from the classpath in legacy "
@@ -205,7 +205,8 @@ private boolean isAllowedVAADINBuildUrl(String filenameWithPath) {
         }
 
         // Check that we target VAADIN/build
-        return filenameWithPath.startsWith("/" + VAADIN_BUILD_FILES_PATH);
+        return filenameWithPath.startsWith("/" + VAADIN_BUILD_FILES_PATH)
+            || filenameWithPath.startsWith("/" + VAADIN_STATIC_FILES_PATH);
     }
 
     /**

flow-server/src/main/java/com/vaadin/flow/server/frontend/TaskUpdateWebpack.java

@@ -30,6 +30,7 @@
 import static com.vaadin.flow.server.frontend.FrontendUtils.DEFAULT_GENERATED_DIR;
 import static com.vaadin.flow.server.frontend.FrontendUtils.WEBPACK_CONFIG;
 import static com.vaadin.flow.server.frontend.FrontendUtils.WEBPACK_GENERATED;
+import static com.vaadin.flow.shared.ApplicationConstants.VAADIN_STATIC_FILES_PATH;
 
 /**
  * Updates the webpack config file according with current project settings.
@@ -80,7 +81,7 @@ public class TaskUpdateWebpack implements FallibleCommand {
         this.flowResourcesFolder = new File(webpackConfigFolder,
             DEFAULT_GENERATED_DIR).toPath();
         this.resourceFolder = new File(webpackOutputDirectory.getParentFile(),
-            "resources").toPath();
+            VAADIN_STATIC_FILES_PATH).toPath();
     }
 
     @Override

flow-server/src/main/resources/plugins/application-theme-plugin/application-theme-plugin.js

@@ -110,7 +110,7 @@ function handleThemes(themeName, themesFolder, projectStaticAssetsOutputFolder)
   if (fs.existsSync(themeFolder)) {
     logger.debug("Found theme ", themeName, " in folder ", themeFolder);
 
-    copyThemeResources(themeName, themeFolder, projectStaticAssetsOutputFolder);
+    copyThemeResources(themeFolder, projectStaticAssetsOutputFolder);
 
     const themeFile = generateThemeFile(themeFolder, themeName);
 

flow-server/src/main/resources/plugins/application-theme-plugin/package.json

@@ -1,11 +1,12 @@
 {
   "description": "application-theme-plugin",
   "keywords": [
-    "plugin"
+    "plugin",
+    "application theme"
   ],
   "repository": "vaadin/flow",
   "name": "@vaadin/application-theme-plugin",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "main": "application-theme-plugin.js",
   "author": "Vaadin Ltd",
   "license": "Apache-2.0",

flow-server/src/main/resources/plugins/application-theme-plugin/theme-copy.js

@@ -16,30 +16,24 @@
 
 /**
  * This file handles copying of theme files to
- * [staticResourcesFolder]/theme/[theme-name]
+ * [staticResourcesFolder]
  */
 
 const fs = require('fs');
 const path = require('path');
 
 /**
- * create theme/themeName folders and copy theme files there.
+ * Copy theme files to static assets folder. All files in the theme folder will be copied excluding
+ * css, js and json files that will be handled by webpack and not be shared as static files.
  *
- * @param {String} themeName name of theme we are handling
  * @param {string} themeFolder Folder with theme file
  * @param {string} projectStaticAssetsOutputFolder resources output folder
  */
-function copyThemeResources(themeName, themeFolder, projectStaticAssetsOutputFolder) {
+function copyThemeResources(themeFolder, projectStaticAssetsOutputFolder) {
   if (!fs.existsSync(path.resolve(projectStaticAssetsOutputFolder))) {
     require('mkdirp')(path.resolve(projectStaticAssetsOutputFolder));
   }
-  if (!fs.existsSync(path.resolve(projectStaticAssetsOutputFolder, "theme"))) {
-    fs.mkdirSync(path.resolve(projectStaticAssetsOutputFolder, "theme"));
-  }
-  if (!fs.existsSync(path.resolve(projectStaticAssetsOutputFolder, "theme", themeName))) {
-    fs.mkdirSync(path.resolve(projectStaticAssetsOutputFolder, "theme", themeName));
-  }
-  copyThemeFiles(themeFolder, path.resolve(projectStaticAssetsOutputFolder, "theme", themeName));
+  copyThemeFiles(themeFolder, projectStaticAssetsOutputFolder);
 }
 
 const ignoredFileExtensions = [".css", ".js", ".json"];

flow-server/src/main/resources/plugins/theme-loader/package.json

@@ -0,0 +1,19 @@
+{
+  "description": "theme-loader updates css urls targeting './' to instead be 'VAADIN/static/' as used by app theme",
+  "keywords": [
+    "plugin",
+    "application theme"
+  ],
+  "repository": "vaadin/flow",
+  "name": "@vaadin/theme-loader",
+  "version": "0.0.1",
+  "main": "theme-loader.js",
+  "author": "Vaadin Ltd",
+  "license": "Apache-2.0",
+  "bugs": {
+    "url": "https://github.com/vaadin/flow/issues"
+  },
+  "files": [
+    "theme-loader.js"
+  ]
+}

flow-server/src/main/resources/plugins/theme-loader/theme-loader.js

@@ -0,0 +1,45 @@
+const loaderUtils = require("loader-utils");
+const fs = require('fs');
+const path = require('path');
+
+// Collect groups [url(] [ |'|"]optional './', file part and end of url
+const urlMatcher = /(url\()(\'|\")?(\.\/|\.\.\/)(\S*)(\2\))/g;
+
+/**
+ * This custom loader handles rewriting urls for the application theme css files.
+ * URLs starting with ./ or ../ are checked against the filesystem and converted if a file exists.
+ * URLs going outside of the application theme folder are not accepted and will not be rewritten.
+ *
+ * @param source file contents to handle
+ * @param map source map for file
+ */
+module.exports = function (source, map) {
+  const options = loaderUtils.getOptions(this);
+  const handledResourceFolder = path.dirname(this._module.resource);
+  const logger = this.getLogger("theme-loader");
+
+  let themeFolder = handledResourceFolder;
+  while (themeFolder && path.basename(path.resolve(themeFolder, "..")) !== "theme") {
+    themeFolder = path.resolve(themeFolder, "..");
+  }
+  logger.log("Using '", themeFolder, "' for the application theme base folder.");
+
+  source = source.replace(urlMatcher, function (match, url, quoteMark, replace, fileUrl, endString) {
+    let absolutePath = path.resolve(handledResourceFolder, replace, fileUrl);
+    if (fs.existsSync(absolutePath) && absolutePath.startsWith(themeFolder)) {
+      logger.debug("Updating url for file '", replace, fileUrl, "' to use 'VAADIN/static'");
+      const pathResolved = absolutePath.substring(themeFolder.length).replace(/\\/g, '/');
+
+      // keep the url the same except replace the ./ to VAADIN/static
+      if (quoteMark) {
+        return url + quoteMark + 'VAADIN/static' + pathResolved + endString;
+      }
+      return url + 'VAADIN/static' + pathResolved + endString;
+    } else if (options.devMode) {
+      logger.info("No rewrite for '", match, "' as the file was not found.");
+    }
+    return match;
+  });
+
+  this.callback(null, source, map);
+}

flow-server/src/main/resources/plugins/webpack-plugins.json

@@ -1,6 +1,7 @@
 {
   "plugins": [
     "stats-plugin",
-    "application-theme-plugin"
+    "application-theme-plugin",
+    "theme-loader"
   ]
 }

flow-server/src/main/resources/webpack.generated.js

@@ -159,14 +159,21 @@ module.exports = {
             loader: 'css-loader',
             options: {
               url: (url, resourcePath) => {
-                // do not handle theme folder url files
-                if(url.startsWith("theme")) {
-                  return false;
-                }
-                return true;
+                // Only translate files from node_modules
+                return resourcePath.includes('/node_modules/');
               },
+              // use theme-loader to also handle any imports in css files
+              importLoaders: 1
             },
           },
+          {
+            // theme-loader will change any url starting with './' to start with 'VAADIN/static' instead
+            // NOTE! this loader should be here so it's run before css-loader as loaders are applied Right-To-Left
+            loader: '@vaadin/theme-loader',
+            options: {
+              devMode: devMode
+            }
+          }
         ],
       },
     ]

flow-server/src/test/java/com/vaadin/flow/server/StaticFileServerTest.java

@@ -509,6 +509,37 @@ public void serveStaticBundleBuildResource() throws IOException {
         assertBundleBuildResource(pathInfo);
     }
 
+    @Test
+    public void contextAndServletPath_serveStaticFileResource()
+            throws IOException {
+        String pathInfo = "/VAADIN/static/img/bg.jpg";
+        setupRequestURI("/context", "/servlet", pathInfo);
+        assertBundleBuildResource(pathInfo);
+    }
+
+    @Test
+    public void ServletPath_serveStaticFileResource()
+            throws IOException {
+        String pathInfo = "/VAADIN/static/img/bg.jpg";
+        setupRequestURI("", "/servlet", pathInfo);
+        assertBundleBuildResource(pathInfo);
+    }
+
+    @Test
+    public void contextPath_serveStaticFileResource()
+            throws IOException {
+        String pathInfo = "/VAADIN/static/img/bg.jpg";
+        setupRequestURI("/context", "", pathInfo);
+        assertBundleBuildResource(pathInfo);
+    }
+
+    @Test
+    public void serveStaticFileResource() throws IOException {
+        String pathInfo = "/VAADIN/static/img/bg.jpg";
+        setupRequestURI("", "", pathInfo);
+        assertBundleBuildResource(pathInfo);
+    }
+
     public void assertBundleBuildResource(String pathInfo) throws IOException {
         byte[] fileData = "function() {eval('foo');};"
                 .getBytes(StandardCharsets.UTF_8);

flow-server/src/test/java/com/vaadin/flow/server/frontend/TaskInstallWebpackPluginsTest.java

@@ -53,7 +53,7 @@ public void init() throws IOException {
 
     @Test
     public void getPluginsReturnsExpectedList() {
-        String[] expectedPlugins = new String[] { "stats-plugin", "application-theme-plugin" };
+        String[] expectedPlugins = new String[] { "stats-plugin", "application-theme-plugin", "theme-loader" };
         final List<String> plugins = task.getPlugins();
         Assert.assertEquals(
             "Unexpected amount of plugins in 'webpack-plugins.json'",

flow-tests/test-themes-legacy/pom-npm.xml

@@ -51,10 +51,6 @@
             <plugin>
                 <groupId>org.eclipse.jetty</groupId>
                 <artifactId>jetty-maven-plugin</artifactId>
-                <configuration>
-                    <!-- Use war output directory instead of default src/main/webapp to get the webpack bundles  -->
-                    <webAppSourceDirectory>${project.build.directory}/${project.build.finalName}</webAppSourceDirectory>
-                </configuration>
             </plugin>
         </plugins>
     </build>

flow-tests/test-themes-legacy/src/main/java/com/vaadin/flow/uitest/ui/theme/MyTheme.java

@@ -28,7 +28,7 @@ public String getBaseUrl() {
 
     @Override
     public String getThemeUrl() {
-        return "legacyTheme/myTheme/";
+        return "theme/myTheme/";
     }
 
     @Override

flow-tests/test-themes-legacy/src/main/webapp/frontend/bower_components/my-component/src/my-component.html

@@ -0,0 +1,13 @@
+<link rel="import" href="../../../bower_components/polymer/polymer-element.html">
+
+<dom-module id="my-component">
+    <template>
+        <div id="component">My component</div>
+    </template>
+    <script>
+        class MyComponent extends Polymer.Element {
+            static get is() { return 'my-component' }
+        }
+        customElements.define(MyComponent.is, MyComponent);
+    </script>
+</dom-module>

flow-tests/test-themes-legacy/src/main/webapp/frontend/bower_components/my-component/theme/myTheme/my-component.html

@@ -0,0 +1,22 @@
+<link rel="import" href="../../../../bower_components/polymer/polymer-element.html">
+<link rel="import" href="../../src/my-component.html">
+
+<dom-module id="my-component">
+    <script>
+        customElements.whenDefined('my-component').then(() => {
+            let element = document.createElement("div");
+            element.setAttribute("id", "theme-component");
+            element.innerHTML = "My themed component";
+
+            function _append(){
+                if ( document.body ){
+                    document.body.appendChild(element);
+                }   
+                else {
+                    setTimeout(_append);
+                }
+            }
+            _append();
+        });
+    </script>
+</dom-module>

flow-tests/test-themes-legacy/src/main/webapp/frontend/bower_components/themed-template/src/com/ThemedTemplate.html

@@ -0,0 +1,35 @@
+<!--
+  ~ Copyright 2000-2019 Vaadin Ltd.
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License"); you may not
+  ~ use this file except in compliance with the License. You may obtain a copy of
+  ~ the License at
+  ~
+  ~ http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  ~ WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  ~ License for the specific language governing permissions and limitations under
+  ~ the License.
+  -->
+<!-- Using an absolute URL is a bad practice, but the nesting depth here makes it incovenient traverse up with ../.. -->
+<link rel="import" href="/frontend/bower_components/polymer/polymer-element.html">
+<link rel="import" href="relative1.html">
+<link rel="import" href="../relative2.html">
+<link rel="import" href="/frontend/absolute.html">
+
+<dom-module id="themed-template">
+    <template>
+      <div id='div'>Themed Template</div>
+    </template>
+
+    <script>
+        class ThemedTemplate extends Polymer.Element {
+            static get is() {
+                return 'themed-template'
+            }
+        }
+        customElements.define(ThemedTemplate.is, ThemedTemplate);
+    </script>
+</dom-module>

flow-tests/test-themes-legacy/src/main/webapp/frontend/bower_components/themed-template/theme/myTheme/com/ThemedTemplate.html

@@ -0,0 +1,35 @@
+<!--
+  ~ Copyright 2000-2019 Vaadin Ltd.
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License"); you may not
+  ~ use this file except in compliance with the License. You may obtain a copy of
+  ~ the License at
+  ~
+  ~ http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  ~ WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  ~ License for the specific language governing permissions and limitations under
+  ~ the License.
+  -->
+<!-- Using an absolute URL is a bad practice, but the nesting depth here makes it incovenient traverse up with ../.. -->
+<link rel="import" href="/frontend/bower_components/polymer/polymer-element.html">
+
+<dom-module id="themed-template">
+    <template>
+      <div id='div'>Themed Template</div>
+      <themed-relative1 id='relative1'></themed-relative1>
+      <themed-relative2 id='relative2'></themed-relative2>
+      <themed-absolute id='absolute'></themed-absolute>
+    </template>
+
+    <script>
+        class ThemedTemplate extends Polymer.Element {
+            static get is() {
+                return 'themed-template'
+            }
+        }
+        customElements.define(ThemedTemplate.is, ThemedTemplate);
+    </script>
+</dom-module>