Skip to content

Commit

Permalink
If pruning is enabled then prune any files that are weakly reachable …
Browse files Browse the repository at this point in the history 
…from entry points.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=221851333
  • Loading branch information
@brad4d
johnplaisted authored and brad4d committed Nov 19, 2018
1 parent 6078b7f commit 6ac58c9
Show file tree
Hide file tree
Showing 7 changed files with 912 additions and 209 deletions.
62 changes: 14 additions & 48 deletions src/com/google/javascript/jscomp/Compiler.java
View file
Expand Up @@ -483,8 +483,6 @@ public <T extends SourceFile> void initModules(
initOptions(options);

checkFirstModule(modules);
modules = moveWeakSources(modules);
fillEmptyModules(modules);

this.externs = makeExternInputs(externs);

Expand All @@ -499,6 +497,9 @@ public <T extends SourceFile> void initModules(
return;
}

// Creating the module graph can move weak source around, and end up with empty modules.
fillEmptyModules(getModules());

this.commentsPerFile = new ConcurrentHashMap<>(moduleGraph.getInputCount());
initBasedOnOptions();

Expand Down Expand Up @@ -567,52 +568,6 @@ private void checkFirstModule(List<JSModule> modules) {
}
}

/** Moves all weak sources into a separate weak module that depends on every other module. */
private List<JSModule> moveWeakSources(List<JSModule> modules) {
// Collect weak sources.
List<CompilerInput> weakInputs = new ArrayList<>();
for (JSModule module : modules) {
if (module.getName().equals(JSModule.WEAK_MODULE_NAME)) {
// Skip an already existing weak module - see below.
continue;
}
for (int i = 0; i < module.getInputCount(); ) {
CompilerInput input = module.getInput(i);
if (input.getSourceFile().isWeak()) {
module.remove(input);
weakInputs.add(input);
} else {
i++;
}
}
}

// If a weak module already exists (e.g. in a stage 2 compilation), make sure it contains all
// weak sources, but leave the module graph otherwise untouched.
if (moduleGraph != null
&& moduleGraph.getModuleByName(JSModule.WEAK_MODULE_NAME) != null
&& !weakInputs.isEmpty()) {
throw new RuntimeException(
"A weak module already exists but weak sources were found in other modules.");
}

// Create the weak module and make it depend on every other module.
JSModule weakModule = new JSModule(JSModule.WEAK_MODULE_NAME);
for (JSModule module : modules) {
weakModule.addDependency(module);
}

// Move the weak sources.
for (CompilerInput input : weakInputs) {
weakModule.add(input);
}

// Make a copy in case the original list is immutable.
modules = ImmutableList.<JSModule>builder().addAll(modules).add(weakModule).build();

return modules;
}

/**
* Empty modules get an empty "fill" file, so that we can move code into
* an empty module.
Expand Down Expand Up @@ -823,6 +778,15 @@ public void stage2Passes() {
checkState(moduleGraph != null, "No inputs. Did you call init() or initModules()?");
checkState(!hasErrors());
checkState(!options.getInstrumentForCoverageOnly());
JSModule weakModule = moduleGraph.getModuleByName(JSModule.WEAK_MODULE_NAME);
if (weakModule != null) {
for (CompilerInput i : moduleGraph.getAllInputs()) {
if (i.getSourceFile().isWeak()) {
checkState(
i.getModule() == weakModule, "Expected all weak files to be in the weak module.");
}
}
}
runInCompilerThread(
() -> {
if (options.shouldOptimize()) {
Expand Down Expand Up @@ -1840,6 +1804,8 @@ void orderInputs() {
}
}

// Manage dependencies may move weak sources around, and end up with empty modules.
fillEmptyModules(getModules());
hoistNoCompileFiles();

if (staleInputs) {
Expand Down
111 changes: 103 additions & 8 deletions src/com/google/javascript/jscomp/JSModuleGraph.java
View file
Expand Up @@ -35,6 +35,7 @@
import com.google.javascript.jscomp.deps.SortedDependencies.MissingProvideException;
import com.google.javascript.jscomp.graph.LinkedDirectedGraph;
import com.google.javascript.jscomp.parsing.parser.util.format.SimpleFormat;
import com.google.javascript.rhino.StaticSourceFile.SourceKind;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
Expand Down Expand Up @@ -96,6 +97,7 @@ public JSModuleGraph(JSModule[] modulesInDepOrder) {
/** Creates a module graph from a list of modules in dependency order. */
public JSModuleGraph(List<JSModule> modulesInDepOrder) {
Preconditions.checkState(!modulesInDepOrder.isEmpty());
modulesInDepOrder = makeWeakModule(modulesInDepOrder);
modules = new JSModule[modulesInDepOrder.size()];

// n = number of modules
Expand All @@ -118,6 +120,9 @@ public JSModuleGraph(List<JSModule> modulesInDepOrder) {

// O(n*m)
subtreeSize = initSubtreeSize();

// Move all sources marked as weak by outside sources (e.g. flags) into the weak module.
moveMarkedWeakSources(getModuleByName(JSModule.WEAK_MODULE_NAME), getAllInputs());
}

private List<List<JSModule>> initModulesByDepth() {
Expand Down Expand Up @@ -146,6 +151,52 @@ private List<List<JSModule>> initModulesByDepth() {
return tmpModulesByDepth;
}

/**
* If a weak module doesn't already exist, creates a weak module depending on every other module.
*
* <p>Does not move any sources into the weak module.
*
* @return a new list of modules that includes the weak module, if it was newly created, or the
* same list if the weak module already existed
* @throws IllegalStateException if a weak module already exists but doesn't fulfill the above
* conditions
*/
private List<JSModule> makeWeakModule(List<JSModule> modulesInDepOrder) {
boolean hasWeakModule = false;
for (JSModule module : modulesInDepOrder) {
if (module.getName().equals(JSModule.WEAK_MODULE_NAME)) {
hasWeakModule = true;
Set<JSModule> allOtherModules = new HashSet<>(modulesInDepOrder);
allOtherModules.remove(module);
checkState(
module.getAllDependencies().containsAll(allOtherModules),
"The weak module must depend on all other modules.");
checkState(
module.getAllDependencies().size() == allOtherModules.size(),
"The weak module cannot have extra dependencies.");
break;
}
}
if (hasWeakModule) {
// All weak files (and only weak files) should be in the weak module.
for (JSModule module : modulesInDepOrder) {
for (CompilerInput input : module.getInputs()) {
checkState(
input.getSourceFile().isWeak() == module.getName().equals(JSModule.WEAK_MODULE_NAME),
"Weak sources must be in the weak module.");
}
}
} else {
JSModule weakModule = new JSModule(JSModule.WEAK_MODULE_NAME);
for (JSModule module : modulesInDepOrder) {
weakModule.addDependency(module);
}
modulesInDepOrder = new ArrayList<>(modulesInDepOrder);
modulesInDepOrder.add(weakModule);
}
return modulesInDepOrder;
}

private BitSet[] initTransitiveDepsBitSets() {
BitSet[] array = new BitSet[modules.length];
for (int moduleIndex = 0; moduleIndex < modules.length; ++moduleIndex) {
Expand Down Expand Up @@ -426,6 +477,27 @@ private Set<JSModule> getTransitiveDeps(JSModule m) {
return deps;
}

/**
* Moves all sources that have {@link SourceKind#WEAK} into the weak module so that they may be
* pruned later.
*/
private static void moveMarkedWeakSources(JSModule weakModule, Iterable<CompilerInput> inputs) {
checkNotNull(weakModule);
ImmutableList<CompilerInput> allInputs = ImmutableList.copyOf(inputs);
for (CompilerInput i : allInputs) {
if (i.getSourceFile().isWeak()) {
JSModule existingModule = i.getModule();
if (existingModule == weakModule) {
continue;
}
if (existingModule != null) {
existingModule.remove(i);
}
weakModule.add(i);
}
}
}

/**
* Apply the dependency options to the list of sources, returning a new source list re-ordering
* and dropping files as necessary. This module graph will be updated to reflect the new list.
Expand Down Expand Up @@ -467,7 +539,7 @@ public ImmutableList<CompilerInput> manageDependencies(DependencyOptions depende

// The order of inputs, sorted independently of modules.
List<CompilerInput> absoluteOrder =
sorter.getDependenciesOf(originalInputs, dependencyOptions.shouldSort());
sorter.getStrongDependenciesOf(originalInputs, dependencyOptions.shouldSort());

// Figure out which sources *must* be in each module.
ListMultimap<JSModule, CompilerInput> entryPointInputsPerModule =
Expand All @@ -478,8 +550,7 @@ public ImmutableList<CompilerInput> manageDependencies(DependencyOptions depende
entryPointInputsPerModule.put(module, input);
}

// Clear the modules of their inputs. This also nulls out
// the input's reference to its module.
// Clear the modules of their inputs. This also nulls out the input's reference to its module.
for (JSModule module : getAllModules()) {
module.removeAll();
}
Expand All @@ -488,6 +559,7 @@ public ImmutableList<CompilerInput> manageDependencies(DependencyOptions depende
// of that module's dependencies.
List<CompilerInput> orderedInputs = new ArrayList<>();
Set<CompilerInput> reachedInputs = new HashSet<>();

for (JSModule module : entryPointInputsPerModule.keySet()) {
List<CompilerInput> transitiveClosure;
// Prefer a depth first ordering of dependencies from entry points.
Expand All @@ -511,10 +583,14 @@ public ImmutableList<CompilerInput> manageDependencies(DependencyOptions depende
// Simply order inputs so that any required namespace comes before it's usage.
// Ordered result varies based on the original order of inputs.
transitiveClosure =
sorter.getDependenciesOf(
sorter.getStrongDependenciesOf(
entryPointInputsPerModule.get(module), dependencyOptions.shouldSort());
}
for (CompilerInput input : transitiveClosure) {
if (dependencyOptions.shouldPrune() && input.getSourceFile().isWeak()) {
throw new IllegalStateException(
"A file that is reachable via an entry point cannot be marked as weak.");
}
JSModule oldModule = input.getModule();
if (oldModule == null) {
input.setModule(module);
Expand All @@ -530,11 +606,31 @@ public ImmutableList<CompilerInput> manageDependencies(DependencyOptions depende
orderedInputs = absoluteOrder;
}

JSModule weakModule = getModuleByName(JSModule.WEAK_MODULE_NAME);
checkNotNull(weakModule);
// Mark all sources that are detected as weak.
if (dependencyOptions.shouldPrune()) {
List<CompilerInput> weakInputs = sorter.getSortedWeakDependenciesOf(orderedInputs);
for (CompilerInput i : weakInputs) {
// Have a separate loop here to add all these dependencies rather than rely on
// moveMarkedWeakSources below. moveMarkedWeakSources will be in user order, while this
// loop is in dependency order. For sources detected as weak we want dependency order,
// for pre-marked as weak we want user order.
checkState(i.getModule() == null);
i.getSourceFile().setKind(SourceKind.WEAK);
i.setModule(weakModule);
weakModule.add(i);
}
}

// Move all sources marked as weak by outside sources (e.g. flags) into the weak module.
moveMarkedWeakSources(weakModule, originalInputs);

// All the inputs are pointing to the modules that own them. Yeah!
// Update the modules to reflect this.
for (CompilerInput input : orderedInputs) {
JSModule module = input.getModule();
if (module != null) {
if (module != null && !module.getInputs().contains(input)) {
module.add(input);
}
}
Expand All @@ -549,7 +645,7 @@ public ImmutableList<CompilerInput> manageDependencies(DependencyOptions depende
}

/**
* Given an input and set of unprocessed inputs, return the input and it's dependencies by
* Given an input and set of unprocessed inputs, return the input and it's strong dependencies by
* performing a recursive, depth-first traversal.
*/
private List<CompilerInput> getDepthFirstDependenciesOf(
Expand All @@ -561,8 +657,7 @@ private List<CompilerInput> getDepthFirstDependenciesOf(
return orderedInputs;
}

for (String importedNamespace :
Iterables.concat(rootInput.getRequiredSymbols(), rootInput.getTypeRequires())) {
for (String importedNamespace : rootInput.getRequiredSymbols()) {
CompilerInput dependency = null;
if (inputsByProvide.containsKey(importedNamespace)
&& unreachedInputs.contains(inputsByProvide.get(importedNamespace))) {
Expand Down
47 changes: 43 additions & 4 deletions src/com/google/javascript/jscomp/deps/Es6SortedDependencies.java
View file
Expand Up @@ -68,14 +68,15 @@ public Es6SortedDependencies(List<INPUT> userOrderedInputs) {
}

@Override
public ImmutableList<INPUT> getDependenciesOf(List<INPUT> rootInputs, boolean sorted) {
public ImmutableList<INPUT> getStrongDependenciesOf(List<INPUT> rootInputs, boolean sorted) {
checkArgument(userOrderedInputs.containsAll(rootInputs));

Set<INPUT> includedInputs = new HashSet<>();
Deque<INPUT> worklist = new ArrayDeque<>(rootInputs);
while (!worklist.isEmpty()) {
INPUT input = worklist.pop();
if (includedInputs.add(input)) {

for (String symbolName : input.getRequiredSymbols()) {
INPUT importedSymbolName = exportingInputBySymbolName.get(symbolName);
if (importedSymbolName != null) {
Expand Down Expand Up @@ -110,8 +111,44 @@ public ImmutableList<INPUT> getInputsWithoutProvides() {
}

@Override
public ImmutableList<INPUT> getSortedDependenciesOf(List<INPUT> roots) {
return getDependenciesOf(roots, true);
public ImmutableList<INPUT> getSortedStrongDependenciesOf(List<INPUT> roots) {
return getStrongDependenciesOf(roots, true);
}

@Override
public List<INPUT> getSortedWeakDependenciesOf(List<INPUT> rootInputs) {
Set<INPUT> strongInputs = new HashSet<>(getSortedStrongDependenciesOf(rootInputs));
Set<INPUT> weakInputs = new HashSet<>();
Deque<INPUT> worklist = new ArrayDeque<>(strongInputs);
while (!worklist.isEmpty()) {
INPUT input = worklist.pop();
boolean isStrong = strongInputs.contains(input);

Iterable<String> edges =
isStrong
? input.getTypeRequires()
: Iterables.concat(input.getRequiredSymbols(), input.getTypeRequires());

if (!isStrong) {
weakInputs.add(input);
}

for (String symbolName : edges) {
INPUT importedSymbolName = exportingInputBySymbolName.get(symbolName);
if (importedSymbolName != null) {
worklist.add(importedSymbolName);
}
}
}

ImmutableList.Builder<INPUT> builder = ImmutableList.builder();
for (INPUT input : importOrderedInputs) {
if (weakInputs.contains(input)) {
builder.add(input);
}
}

return builder.build();
}

@Override
Expand Down Expand Up @@ -163,7 +200,9 @@ private void processInputs() {
}
}
for (INPUT userOrderedInput : userOrderedInputs) {
for (String symbolName : userOrderedInput.getRequiredSymbols()) {
for (String symbolName :
Iterables.concat(
userOrderedInput.getRequiredSymbols(), userOrderedInput.getTypeRequires())) {
INPUT importedInput = exportingInputBySymbolName.get(symbolName);
if (importedInput != null) {
importedInputByImportingInput.put(userOrderedInput, importedInput);
Expand Down

0 comments on commit 6ac58c9

Please sign in to comment.