Skip to content

Commit

Permalink
chore: cache JS imported paths during chunk generation (#17672) (#17684)
Browse files Browse the repository at this point in the history 
* chore: cache JS imported paths during chunk generation

In a production build, for every chunk the JS files are parsed to extract the
imported paths. There are checks to ensure that a module is visited only once,
but this works on a chunk level.
This change caches the JS imports extraction results, so that a single JS file
is visited only once during the build.

Part of #17234

* cleanup

* improve caching of imported paths

Co-authored-by: Marco Collovati <marco@vaadin.com>
  • Loading branch information
@vaadin-bot @mcollovati
vaadin-bot and mcollovati committed Sep 21, 2023
1 parent 4b2217a commit d2d83de
Showing 1 changed file with 74 additions and 53 deletions.
127 changes: 74 additions & 53 deletions flow-server/src/main/java/com/vaadin/flow/server/frontend/AbstractUpdateImports.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.function.UnaryOperator;
Expand Down Expand Up @@ -94,6 +95,10 @@ abstract class AbstractUpdateImports implements Runnable {
private static final Pattern STARTING_DOT_SLASH = Pattern.compile("^\\./+");
final Options options;

private final UnaryOperator<String> themeToLocalPathConverter;

private final Map<Path, List<String>> resolvedImportPathsCache = new HashMap<>();

private FrontendDependenciesScanner scanner;

private ClassFinder classFinder;
Expand All @@ -107,6 +112,8 @@ abstract class AbstractUpdateImports implements Runnable {
this.options = options;
this.scanner = scanner;
this.classFinder = classFinder;
this.themeToLocalPathConverter = createThemeToLocalPathConverter(
scanner.getTheme());

generatedFlowImports = FrontendUtils
.getFlowGeneratedImports(options.getFrontendDirectory());
Expand Down Expand Up @@ -219,7 +226,6 @@ private Map<File, List<String>> process(Map<ChunkInfo, List<CssData>> css,
mergedChunkKeys.size());

for (ChunkInfo chunkInfo : mergedChunkKeys) {

List<String> chunkLines = new ArrayList<>();
if (lazyJavascript.containsKey(chunkInfo)) {
chunkLines.addAll(
Expand Down Expand Up @@ -470,21 +476,6 @@ private Set<String> getUniqueEs6ImportPaths(Collection<String> modules) {
Set<String> es6ImportPaths = new LinkedHashSet<>();
AbstractTheme theme = scanner.getTheme();

UnaryOperator<String> convertToLocalPath;
if (theme != null) {
// (#5964) Allows:
// - custom @Theme with files placed in /frontend
// - customize an already themed component
// @vaadin/vaadin-grid/theme/lumo/vaadin-grid.js ->
// theme/lumo/vaadin-grid.js
String themePath = theme.getThemeUrl();
Pattern themePattern = Pattern.compile("@.+" + themePath);
convertToLocalPath = path -> themePattern.matcher(path)
.replaceFirst(themePath);
} else {
convertToLocalPath = UnaryOperator.identity();
}

Set<String> visited = new HashSet<>();

for (String originalModulePath : modules) {
Expand All @@ -493,7 +484,7 @@ private Set<String> getUniqueEs6ImportPaths(Collection<String> modules) {
if (theme != null
&& translatedModulePath.contains(theme.getBaseUrl())) {
translatedModulePath = theme.translateUrl(translatedModulePath);
localModulePath = convertToLocalPath
localModulePath = themeToLocalPathConverter
.apply(translatedModulePath);
}

Expand Down Expand Up @@ -556,10 +547,28 @@ && frontendFileExists(localModulePath)) {
"Failed to find the following imports in the `node_modules` tree:",
getImportsNotFoundMessage()));
}

return es6ImportPaths;
}

private static UnaryOperator<String> createThemeToLocalPathConverter(
AbstractTheme theme) {
UnaryOperator<String> convertToLocalPath;
if (theme != null) {
// (#5964) Allows:
// - custom @Theme with files placed in /frontend
// - customize an already themed component
// @vaadin/vaadin-grid/theme/lumo/vaadin-grid.js ->
// theme/lumo/vaadin-grid.js
String themePath = theme.getThemeUrl();
Pattern themePattern = Pattern.compile("@.+" + themePath);
convertToLocalPath = path -> themePattern.matcher(path)
.replaceFirst(themePath);
} else {
convertToLocalPath = UnaryOperator.identity();
}
return convertToLocalPath;
}

private boolean isGeneratedFlowFile(String localModulePath) {
return localModulePath
.startsWith(FrontendUtils.FRONTEND_GENERATED_FLOW_IMPORT_PATH);
Expand Down Expand Up @@ -759,43 +768,55 @@ private void visitImportsRecursively(Path filePath, String path,
AbstractTheme theme, Collection<String> imports,
Set<String> visitedImports) throws IOException {

String content = null;
try (final Stream<String> contentStream = Files.lines(filePath,
StandardCharsets.UTF_8)) {
content = contentStream.collect(Collectors.joining("\n"));
} catch (UncheckedIOException ioe) {
if (ioe.getCause() instanceof MalformedInputException) {
getLogger().trace(
"Failed to read file '{}' found from Es6 import statements. "
+ "This is probably due to it being a binary file, "
+ "in which case it doesn't matter as imports are only in js/ts files.",
filePath.toString(), ioe);
return;
}
throw ioe;
}
ImportExtractor extractor = new ImportExtractor(content);
List<String> importedPaths = extractor.getImportedPaths();
for (String importedPath : importedPaths) {
// try to resolve path relatively to original filePath (inside user
// frontend folder)
importedPath = StringUtil.stripSuffix(importedPath, "?inline");
String resolvedPath = resolve(importedPath, filePath, path);
File file = getImportedFrontendFile(resolvedPath);
if (file == null && !importedPath.startsWith("./")) {
// In case such file doesn't exist it may be external: inside
// node_modules folder
file = getFile(options.getNodeModulesFolder(), importedPath);
if (!file.exists()) {
file = null;
if (!resolvedImportPathsCache.containsKey(filePath)) {
String content = null;
try (final Stream<String> contentStream = Files.lines(filePath,
StandardCharsets.UTF_8)) {
content = contentStream.collect(Collectors.joining("\n"));
} catch (UncheckedIOException ioe) {
if (ioe.getCause() instanceof MalformedInputException) {
getLogger().trace(
"Failed to read file '{}' found from Es6 import statements. "
+ "This is probably due to it being a binary file, "
+ "in which case it doesn't matter as imports are only in js/ts files.",
filePath.toString(), ioe);
return;
}
resolvedPath = importedPath;
}
if (file == null) {
// don't do anything if such file doesn't exist at all
continue;
throw ioe;
}
resolvedPath = normalizePath(resolvedPath);
ImportExtractor extractor = new ImportExtractor(content);
resolvedImportPathsCache.put(filePath,
extractor.getImportedPaths().stream().map(importedPath -> {
// try to resolve path relatively to original filePath
// (inside user
// frontend folder)
importedPath = StringUtil.stripSuffix(importedPath,
"?inline");
String resolvedPath = resolve(importedPath, filePath,
path);
File file = getImportedFrontendFile(resolvedPath);
if (file == null && !importedPath.startsWith("./")) {
// In case such file doesn't exist it may be
// external: inside
// node_modules folder
file = getFile(options.getNodeModulesFolder(),
importedPath);
if (!file.exists()) {
file = null;
}
resolvedPath = importedPath;
}
if (file == null) {
// don't do anything if such file doesn't exist at
// all
return null;
}
return normalizePath(resolvedPath);
}).filter(Objects::nonNull).collect(Collectors.toList()));
}
List<String> resolvedPaths = resolvedImportPathsCache.get(filePath);

for (String resolvedPath : resolvedPaths) {
if (resolvedPath.contains(theme.getBaseUrl())) {
String translatedPath = theme.translateUrl(resolvedPath);
if (!visitedImports.contains(translatedPath)
Expand Down

0 comments on commit d2d83de

Please sign in to comment.