Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

#1336 running EO optimization in parallel #1345

Merged
merged 2 commits into from
Oct 19, 2022
Merged

#1336 running EO optimization in parallel #1345

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dac18b8
#1336 running EO optimization in parallel
mximp Oct 19, 2022
2bd67e6
#1336 correct puzzle description.
mximp Oct 19, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
146 changes: 111 additions & 35 deletions eo-maven-plugin/src/main/java/org/eolang/maven/OptimizeMojo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import com.jcabi.xml.XML;
import com.jcabi.xml.XMLDocument;
import com.yegor256.tojos.Tojo;
import com.yegor256.tojos.Tojos;
import com.yegor256.xsline.Shift;
import com.yegor256.xsline.StClasspath;
import com.yegor256.xsline.TrClasspath;
Expand All @@ -38,16 +39,26 @@
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.maven.plugins.annotations.LifecyclePhase;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;
import org.cactoos.scalar.Unchecked;
import org.eolang.parser.ParsingTrain;

/**
* Optimize XML files.
*
* @since 0.1
*
* @todo #1336:30min Make a number of threads in `exec()` method configurable
* via mojo parameter `threads`. Default value should be set to 4.
*/
@Mojo(
name = "optimize",
Expand All @@ -68,21 +79,30 @@ public final class OptimizeMojo extends SafeMojo {

/**
* Parsing train with XSLs.
* @implNote The list of applied XSLs is adjusted during execution
* @implNote The list of applied XSLs is adjusted during execution.
* <br>Separate instance of the train is used of each optimization
* thread since {@link com.jcabi.xml.XSLDocument}, which is used under
* the hood in {@link TrClasspath}, is not thread-safe.
* @todo #1336:30min Replace creation of new `Train` instances for each
* parsing task to a single `Train&gtShift&lt TRAIN`, once `TrClasspath`
* is thread-safe (solved by
* <a href="https://github.com/jcabi/jcabi-xml/issues/185"/>).
*/
private static final Train<Shift> TRAIN = new TrFast(
new TrClasspath<>(
new ParsingTrain(),
"/org/eolang/parser/optimize/globals-to-abstracts.xsl",
"/org/eolang/parser/optimize/remove-refs.xsl",
"/org/eolang/parser/optimize/abstracts-float-up.xsl",
"/org/eolang/parser/optimize/remove-levels.xsl",
"/org/eolang/parser/add-refs.xsl",
"/org/eolang/parser/optimize/fix-missed-names.xsl",
"/org/eolang/parser/add-refs.xsl",
"/org/eolang/parser/errors/broken-refs.xsl",
"/org/eolang/parser/optimize/constant-folding.xsl"
).back()
private static final Unchecked<Train<Shift>> TRAIN = new Unchecked<>(
() -> new TrFast(
new TrClasspath<>(
new ParsingTrain(),
"/org/eolang/parser/optimize/globals-to-abstracts.xsl",
"/org/eolang/parser/optimize/remove-refs.xsl",
"/org/eolang/parser/optimize/abstracts-float-up.xsl",
"/org/eolang/parser/optimize/remove-levels.xsl",
"/org/eolang/parser/add-refs.xsl",
"/org/eolang/parser/optimize/fix-missed-names.xsl",
"/org/eolang/parser/add-refs.xsl",
"/org/eolang/parser/errors/broken-refs.xsl",
"/org/eolang/parser/optimize/constant-folding.xsl"
).back()
)
);

/**
Expand Down Expand Up @@ -122,30 +142,86 @@ public void exec() throws IOException {
final Collection<Tojo> sources = this.scopedTojos().select(
row -> row.exists(AssembleMojo.ATTR_XMIR)
);
int done = 0;
for (final Tojo tojo : sources) {
final Path src = Paths.get(tojo.get(AssembleMojo.ATTR_XMIR));
if (tojo.exists(AssembleMojo.ATTR_XMIR2)) {
final Path tgt = Paths.get(tojo.get(AssembleMojo.ATTR_XMIR2));
if (tgt.toFile().lastModified() >= src.toFile().lastModified()) {
Logger.debug(
this, "Already optimized %s to %s",
new Home().rel(src), new Home().rel(tgt)
final Set<Callable<Object>> tasks = new HashSet<>(0);
final AtomicInteger done = new AtomicInteger(0);
sources.stream()
.map(SynchronizedTojo::new)
.forEach(
tojo -> {
final Path src = Paths.get(tojo.get(AssembleMojo.ATTR_XMIR));
if (tojo.exists(AssembleMojo.ATTR_XMIR2)) {
final Path tgt = Paths.get(tojo.get(AssembleMojo.ATTR_XMIR2));
if (tgt.toFile().lastModified() >= src.toFile().lastModified()) {
Logger.debug(
this, "Already optimized %s to %s",
new Home().rel(src), new Home().rel(tgt)
);
return;
}
}
Logger.info(
this, "Adding optimization task for %s",
src
);
tasks.add(
Executors.callable(
() -> {
try {
final XML optimized = this.optimize(src);
done.incrementAndGet();
if (this.shouldPass(optimized)) {
tojo.set(
AssembleMojo.ATTR_XMIR2,
this.make(optimized, src).toAbsolutePath().toString()
);
}
} catch (final IOException exception) {
throw new IllegalStateException(
String.format(
"Unable to optimize %s",
tojo.get(Tojos.KEY)
),
exception
);
}
}
)
);
continue;
}
}
++done;
final XML optimized = this.optimize(src);
if (this.shouldPass(optimized)) {
tojo.set(
AssembleMojo.ATTR_XMIR2,
this.make(optimized, src).toAbsolutePath().toString()
);
try {
Logger.info(
this, "Running %s optimizations in parallel",
tasks.size()
);
Executors.newFixedThreadPool(4)
.invokeAll(tasks)
.forEach(
completed -> {
try {
completed.get();
} catch (final InterruptedException ex) {
Thread.currentThread().interrupt();
} catch (final ExecutionException ex) {
throw new IllegalArgumentException(
ex.getCause().getMessage(),
ex
);
}
}
);
}
} catch (final InterruptedException ex) {
Thread.currentThread().interrupt();
throw new IllegalStateException(
String.format(
"Interrupted while waiting for %d optimizations to finish",
done.get()
),
ex
);
}
if (done > 0) {
Logger.info(this, "Optimized %d out of %d XMIR program(s)", done, sources.size());
if (done.get() > 0) {
Logger.info(this, "Optimized %d out of %d XMIR program(s)", done.get(), sources.size());
} else {
Logger.debug(this, "No XMIR programs out of %d optimized", sources.size());
}
Expand All @@ -162,7 +238,7 @@ public void exec() throws IOException {
*/
private XML optimize(final Path file) throws FileNotFoundException {
final String name = new XMLDocument(file).xpath("/program/@name").get(0);
Train<Shift> trn = OptimizeMojo.TRAIN;
Train<Shift> trn = OptimizeMojo.TRAIN.value();
if (this.failOnWarning) {
trn = trn.with(new StClasspath("/org/eolang/parser/errors/fail-on-warnings.xsl"));
}
Expand Down