Skip to content

Commit

Permalink
Rollback "If pruning is enabled then prune any..."
Browse files Browse the repository at this point in the history 
Broke a Google project.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=222139351
  • Loading branch information
@lauraharker @brad4d
lauraharker authored and brad4d committed Nov 20, 2018
1 parent bd009e7 commit 654e00d
Show file tree
Hide file tree
Showing 7 changed files with 209 additions and 912 deletions.
62 changes: 48 additions & 14 deletions src/com/google/javascript/jscomp/Compiler.java
View file
Expand Up @@ -483,6 +483,8 @@ public <T extends SourceFile> void initModules(
initOptions(options); initOptions(options);


checkFirstModule(modules); checkFirstModule(modules);
modules = moveWeakSources(modules);
fillEmptyModules(modules);


this.externs = makeExternInputs(externs); this.externs = makeExternInputs(externs);


Expand All @@ -497,9 +499,6 @@ public <T extends SourceFile> void initModules(
return; return;
} }


// Creating the module graph can move weak source around, and end up with empty modules.
fillEmptyModules(getModules());

this.commentsPerFile = new ConcurrentHashMap<>(moduleGraph.getInputCount()); this.commentsPerFile = new ConcurrentHashMap<>(moduleGraph.getInputCount());
initBasedOnOptions(); initBasedOnOptions();


Expand Down Expand Up @@ -568,6 +567,52 @@ private void checkFirstModule(List<JSModule> modules) {
} }
} }


/** Moves all weak sources into a separate weak module that depends on every other module. */
private List<JSModule> moveWeakSources(List<JSModule> modules) {
// Collect weak sources.
List<CompilerInput> weakInputs = new ArrayList<>();
for (JSModule module : modules) {
if (module.getName().equals(JSModule.WEAK_MODULE_NAME)) {
// Skip an already existing weak module - see below.
continue;
}
for (int i = 0; i < module.getInputCount(); ) {
CompilerInput input = module.getInput(i);
if (input.getSourceFile().isWeak()) {
module.remove(input);
weakInputs.add(input);
} else {
i++;
}
}
}

// If a weak module already exists (e.g. in a stage 2 compilation), make sure it contains all
// weak sources, but leave the module graph otherwise untouched.
if (moduleGraph != null
&& moduleGraph.getModuleByName(JSModule.WEAK_MODULE_NAME) != null
&& !weakInputs.isEmpty()) {
throw new RuntimeException(
"A weak module already exists but weak sources were found in other modules.");
}

// Create the weak module and make it depend on every other module.
JSModule weakModule = new JSModule(JSModule.WEAK_MODULE_NAME);
for (JSModule module : modules) {
weakModule.addDependency(module);
}

// Move the weak sources.
for (CompilerInput input : weakInputs) {
weakModule.add(input);
}

// Make a copy in case the original list is immutable.
modules = ImmutableList.<JSModule>builder().addAll(modules).add(weakModule).build();

return modules;
}

/** /**
* Empty modules get an empty "fill" file, so that we can move code into * Empty modules get an empty "fill" file, so that we can move code into
* an empty module. * an empty module.
Expand Down Expand Up @@ -778,15 +823,6 @@ public void stage2Passes() {
checkState(moduleGraph != null, "No inputs. Did you call init() or initModules()?"); checkState(moduleGraph != null, "No inputs. Did you call init() or initModules()?");
checkState(!hasErrors()); checkState(!hasErrors());
checkState(!options.getInstrumentForCoverageOnly()); checkState(!options.getInstrumentForCoverageOnly());
JSModule weakModule = moduleGraph.getModuleByName(JSModule.WEAK_MODULE_NAME);
if (weakModule != null) {
for (CompilerInput i : moduleGraph.getAllInputs()) {
if (i.getSourceFile().isWeak()) {
checkState(
i.getModule() == weakModule, "Expected all weak files to be in the weak module.");
}
}
}
runInCompilerThread( runInCompilerThread(
() -> { () -> {
if (options.shouldOptimize()) { if (options.shouldOptimize()) {
Expand Down Expand Up @@ -1804,8 +1840,6 @@ void orderInputs() {
} }
} }


// Manage dependencies may move weak sources around, and end up with empty modules.
fillEmptyModules(getModules());
hoistNoCompileFiles(); hoistNoCompileFiles();


if (staleInputs) { if (staleInputs) {
Expand Down
111 changes: 8 additions & 103 deletions src/com/google/javascript/jscomp/JSModuleGraph.java
View file
Expand Up @@ -35,7 +35,6 @@
import com.google.javascript.jscomp.deps.SortedDependencies.MissingProvideException; import com.google.javascript.jscomp.deps.SortedDependencies.MissingProvideException;
import com.google.javascript.jscomp.graph.LinkedDirectedGraph; import com.google.javascript.jscomp.graph.LinkedDirectedGraph;
import com.google.javascript.jscomp.parsing.parser.util.format.SimpleFormat; import com.google.javascript.jscomp.parsing.parser.util.format.SimpleFormat;
import com.google.javascript.rhino.StaticSourceFile.SourceKind;
import java.io.Serializable; import java.io.Serializable;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
Expand Down Expand Up @@ -97,7 +96,6 @@ public JSModuleGraph(JSModule[] modulesInDepOrder) {
/** Creates a module graph from a list of modules in dependency order. */ /** Creates a module graph from a list of modules in dependency order. */
public JSModuleGraph(List<JSModule> modulesInDepOrder) { public JSModuleGraph(List<JSModule> modulesInDepOrder) {
Preconditions.checkState(!modulesInDepOrder.isEmpty()); Preconditions.checkState(!modulesInDepOrder.isEmpty());
modulesInDepOrder = makeWeakModule(modulesInDepOrder);
modules = new JSModule[modulesInDepOrder.size()]; modules = new JSModule[modulesInDepOrder.size()];


// n = number of modules // n = number of modules
Expand All @@ -120,9 +118,6 @@ public JSModuleGraph(List<JSModule> modulesInDepOrder) {


// O(n*m) // O(n*m)
subtreeSize = initSubtreeSize(); subtreeSize = initSubtreeSize();

// Move all sources marked as weak by outside sources (e.g. flags) into the weak module.
moveMarkedWeakSources(getModuleByName(JSModule.WEAK_MODULE_NAME), getAllInputs());
} }


private List<List<JSModule>> initModulesByDepth() { private List<List<JSModule>> initModulesByDepth() {
Expand Down Expand Up @@ -151,52 +146,6 @@ private List<List<JSModule>> initModulesByDepth() {
return tmpModulesByDepth; return tmpModulesByDepth;
} }


/**
* If a weak module doesn't already exist, creates a weak module depending on every other module.
*
* <p>Does not move any sources into the weak module.
*
* @return a new list of modules that includes the weak module, if it was newly created, or the
* same list if the weak module already existed
* @throws IllegalStateException if a weak module already exists but doesn't fulfill the above
* conditions
*/
private List<JSModule> makeWeakModule(List<JSModule> modulesInDepOrder) {
boolean hasWeakModule = false;
for (JSModule module : modulesInDepOrder) {
if (module.getName().equals(JSModule.WEAK_MODULE_NAME)) {
hasWeakModule = true;
Set<JSModule> allOtherModules = new HashSet<>(modulesInDepOrder);
allOtherModules.remove(module);
checkState(
module.getAllDependencies().containsAll(allOtherModules),
"The weak module must depend on all other modules.");
checkState(
module.getAllDependencies().size() == allOtherModules.size(),
"The weak module cannot have extra dependencies.");
break;
}
}
if (hasWeakModule) {
// All weak files (and only weak files) should be in the weak module.
for (JSModule module : modulesInDepOrder) {
for (CompilerInput input : module.getInputs()) {
checkState(
input.getSourceFile().isWeak() == module.getName().equals(JSModule.WEAK_MODULE_NAME),
"Weak sources must be in the weak module.");
}
}
} else {
JSModule weakModule = new JSModule(JSModule.WEAK_MODULE_NAME);
for (JSModule module : modulesInDepOrder) {
weakModule.addDependency(module);
}
modulesInDepOrder = new ArrayList<>(modulesInDepOrder);
modulesInDepOrder.add(weakModule);
}
return modulesInDepOrder;
}

private BitSet[] initTransitiveDepsBitSets() { private BitSet[] initTransitiveDepsBitSets() {
BitSet[] array = new BitSet[modules.length]; BitSet[] array = new BitSet[modules.length];
for (int moduleIndex = 0; moduleIndex < modules.length; ++moduleIndex) { for (int moduleIndex = 0; moduleIndex < modules.length; ++moduleIndex) {
Expand Down Expand Up @@ -477,27 +426,6 @@ private Set<JSModule> getTransitiveDeps(JSModule m) {
return deps; return deps;
} }


/**
* Moves all sources that have {@link SourceKind#WEAK} into the weak module so that they may be
* pruned later.
*/
private static void moveMarkedWeakSources(JSModule weakModule, Iterable<CompilerInput> inputs) {
checkNotNull(weakModule);
ImmutableList<CompilerInput> allInputs = ImmutableList.copyOf(inputs);
for (CompilerInput i : allInputs) {
if (i.getSourceFile().isWeak()) {
JSModule existingModule = i.getModule();
if (existingModule == weakModule) {
continue;
}
if (existingModule != null) {
existingModule.remove(i);
}
weakModule.add(i);
}
}
}

/** /**
* Apply the dependency options to the list of sources, returning a new source list re-ordering * Apply the dependency options to the list of sources, returning a new source list re-ordering
* and dropping files as necessary. This module graph will be updated to reflect the new list. * and dropping files as necessary. This module graph will be updated to reflect the new list.
Expand Down Expand Up @@ -539,7 +467,7 @@ public ImmutableList<CompilerInput> manageDependencies(DependencyOptions depende


// The order of inputs, sorted independently of modules. // The order of inputs, sorted independently of modules.
List<CompilerInput> absoluteOrder = List<CompilerInput> absoluteOrder =
sorter.getStrongDependenciesOf(originalInputs, dependencyOptions.shouldSort()); sorter.getDependenciesOf(originalInputs, dependencyOptions.shouldSort());


// Figure out which sources *must* be in each module. // Figure out which sources *must* be in each module.
ListMultimap<JSModule, CompilerInput> entryPointInputsPerModule = ListMultimap<JSModule, CompilerInput> entryPointInputsPerModule =
Expand All @@ -550,7 +478,8 @@ public ImmutableList<CompilerInput> manageDependencies(DependencyOptions depende
entryPointInputsPerModule.put(module, input); entryPointInputsPerModule.put(module, input);
} }


// Clear the modules of their inputs. This also nulls out the input's reference to its module. // Clear the modules of their inputs. This also nulls out
// the input's reference to its module.
for (JSModule module : getAllModules()) { for (JSModule module : getAllModules()) {
module.removeAll(); module.removeAll();
} }
Expand All @@ -559,7 +488,6 @@ public ImmutableList<CompilerInput> manageDependencies(DependencyOptions depende
// of that module's dependencies. // of that module's dependencies.
List<CompilerInput> orderedInputs = new ArrayList<>(); List<CompilerInput> orderedInputs = new ArrayList<>();
Set<CompilerInput> reachedInputs = new HashSet<>(); Set<CompilerInput> reachedInputs = new HashSet<>();

for (JSModule module : entryPointInputsPerModule.keySet()) { for (JSModule module : entryPointInputsPerModule.keySet()) {
List<CompilerInput> transitiveClosure; List<CompilerInput> transitiveClosure;
// Prefer a depth first ordering of dependencies from entry points. // Prefer a depth first ordering of dependencies from entry points.
Expand All @@ -583,14 +511,10 @@ public ImmutableList<CompilerInput> manageDependencies(DependencyOptions depende
// Simply order inputs so that any required namespace comes before it's usage. // Simply order inputs so that any required namespace comes before it's usage.
// Ordered result varies based on the original order of inputs. // Ordered result varies based on the original order of inputs.
transitiveClosure = transitiveClosure =
sorter.getStrongDependenciesOf( sorter.getDependenciesOf(
entryPointInputsPerModule.get(module), dependencyOptions.shouldSort()); entryPointInputsPerModule.get(module), dependencyOptions.shouldSort());
} }
for (CompilerInput input : transitiveClosure) { for (CompilerInput input : transitiveClosure) {
if (dependencyOptions.shouldPrune() && input.getSourceFile().isWeak()) {
throw new IllegalStateException(
"A file that is reachable via an entry point cannot be marked as weak.");
}
JSModule oldModule = input.getModule(); JSModule oldModule = input.getModule();
if (oldModule == null) { if (oldModule == null) {
input.setModule(module); input.setModule(module);
Expand All @@ -606,31 +530,11 @@ public ImmutableList<CompilerInput> manageDependencies(DependencyOptions depende
orderedInputs = absoluteOrder; orderedInputs = absoluteOrder;
} }


JSModule weakModule = getModuleByName(JSModule.WEAK_MODULE_NAME);
checkNotNull(weakModule);
// Mark all sources that are detected as weak.
if (dependencyOptions.shouldPrune()) {
List<CompilerInput> weakInputs = sorter.getSortedWeakDependenciesOf(orderedInputs);
for (CompilerInput i : weakInputs) {
// Have a separate loop here to add all these dependencies rather than rely on
// moveMarkedWeakSources below. moveMarkedWeakSources will be in user order, while this
// loop is in dependency order. For sources detected as weak we want dependency order,
// for pre-marked as weak we want user order.
checkState(i.getModule() == null);
i.getSourceFile().setKind(SourceKind.WEAK);
i.setModule(weakModule);
weakModule.add(i);
}
}

// Move all sources marked as weak by outside sources (e.g. flags) into the weak module.
moveMarkedWeakSources(weakModule, originalInputs);

// All the inputs are pointing to the modules that own them. Yeah! // All the inputs are pointing to the modules that own them. Yeah!
// Update the modules to reflect this. // Update the modules to reflect this.
for (CompilerInput input : orderedInputs) { for (CompilerInput input : orderedInputs) {
JSModule module = input.getModule(); JSModule module = input.getModule();
if (module != null && !module.getInputs().contains(input)) { if (module != null) {
module.add(input); module.add(input);
} }
} }
Expand All @@ -645,7 +549,7 @@ public ImmutableList<CompilerInput> manageDependencies(DependencyOptions depende
} }


/** /**
* Given an input and set of unprocessed inputs, return the input and it's strong dependencies by * Given an input and set of unprocessed inputs, return the input and it's dependencies by
* performing a recursive, depth-first traversal. * performing a recursive, depth-first traversal.
*/ */
private List<CompilerInput> getDepthFirstDependenciesOf( private List<CompilerInput> getDepthFirstDependenciesOf(
Expand All @@ -657,7 +561,8 @@ private List<CompilerInput> getDepthFirstDependenciesOf(
return orderedInputs; return orderedInputs;
} }


for (String importedNamespace : rootInput.getRequiredSymbols()) { for (String importedNamespace :
Iterables.concat(rootInput.getRequiredSymbols(), rootInput.getTypeRequires())) {
CompilerInput dependency = null; CompilerInput dependency = null;
if (inputsByProvide.containsKey(importedNamespace) if (inputsByProvide.containsKey(importedNamespace)
&& unreachedInputs.contains(inputsByProvide.get(importedNamespace))) { && unreachedInputs.contains(inputsByProvide.get(importedNamespace))) {
Expand Down
47 changes: 4 additions & 43 deletions src/com/google/javascript/jscomp/deps/Es6SortedDependencies.java
View file
Expand Up @@ -68,15 +68,14 @@ public Es6SortedDependencies(List<INPUT> userOrderedInputs) {
} }


@Override @Override
public ImmutableList<INPUT> getStrongDependenciesOf(List<INPUT> rootInputs, boolean sorted) { public ImmutableList<INPUT> getDependenciesOf(List<INPUT> rootInputs, boolean sorted) {
checkArgument(userOrderedInputs.containsAll(rootInputs)); checkArgument(userOrderedInputs.containsAll(rootInputs));


Set<INPUT> includedInputs = new HashSet<>(); Set<INPUT> includedInputs = new HashSet<>();
Deque<INPUT> worklist = new ArrayDeque<>(rootInputs); Deque<INPUT> worklist = new ArrayDeque<>(rootInputs);
while (!worklist.isEmpty()) { while (!worklist.isEmpty()) {
INPUT input = worklist.pop(); INPUT input = worklist.pop();
if (includedInputs.add(input)) { if (includedInputs.add(input)) {

for (String symbolName : input.getRequiredSymbols()) { for (String symbolName : input.getRequiredSymbols()) {
INPUT importedSymbolName = exportingInputBySymbolName.get(symbolName); INPUT importedSymbolName = exportingInputBySymbolName.get(symbolName);
if (importedSymbolName != null) { if (importedSymbolName != null) {
Expand Down Expand Up @@ -111,44 +110,8 @@ public ImmutableList<INPUT> getInputsWithoutProvides() {
} }


@Override @Override
public ImmutableList<INPUT> getSortedStrongDependenciesOf(List<INPUT> roots) { public ImmutableList<INPUT> getSortedDependenciesOf(List<INPUT> roots) {
return getStrongDependenciesOf(roots, true); return getDependenciesOf(roots, true);
}

@Override
public List<INPUT> getSortedWeakDependenciesOf(List<INPUT> rootInputs) {
Set<INPUT> strongInputs = new HashSet<>(getSortedStrongDependenciesOf(rootInputs));
Set<INPUT> weakInputs = new HashSet<>();
Deque<INPUT> worklist = new ArrayDeque<>(strongInputs);
while (!worklist.isEmpty()) {
INPUT input = worklist.pop();
boolean isStrong = strongInputs.contains(input);

Iterable<String> edges =
isStrong
? input.getTypeRequires()
: Iterables.concat(input.getRequiredSymbols(), input.getTypeRequires());

if (!isStrong) {
weakInputs.add(input);
}

for (String symbolName : edges) {
INPUT importedSymbolName = exportingInputBySymbolName.get(symbolName);
if (importedSymbolName != null) {
worklist.add(importedSymbolName);
}
}
}

ImmutableList.Builder<INPUT> builder = ImmutableList.builder();
for (INPUT input : importOrderedInputs) {
if (weakInputs.contains(input)) {
builder.add(input);
}
}

return builder.build();
} }


@Override @Override
Expand Down Expand Up @@ -200,9 +163,7 @@ private void processInputs() {
} }
} }
for (INPUT userOrderedInput : userOrderedInputs) { for (INPUT userOrderedInput : userOrderedInputs) {
for (String symbolName : for (String symbolName : userOrderedInput.getRequiredSymbols()) {
Iterables.concat(
userOrderedInput.getRequiredSymbols(), userOrderedInput.getTypeRequires())) {
INPUT importedInput = exportingInputBySymbolName.get(symbolName); INPUT importedInput = exportingInputBySymbolName.get(symbolName);
if (importedInput != null) { if (importedInput != null) {
importedInputByImportingInput.put(userOrderedInput, importedInput); importedInputByImportingInput.put(userOrderedInput, importedInput);
Expand Down

0 comments on commit 654e00d

Please sign in to comment.